package build
// Attribution:
// JLPT word lists are based on data from Jonathan Waller's JLPT resources
// (http://www.tanos.co.uk/jlpt/), licensed under Creative Commons Attribution (CC BY),
// curated in github.com/jamsinclair/open-anki-jlpt-decks.
import (
"bufio"
"bytes"
"compress/gzip"
"encoding/binary"
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"slices"
"sort"
"strconv"
"strings"
"unicode/utf8"
"chirami/internal/dict"
"chirami/internal/jp"
)
type BuildOption func(*buildOptions)
type buildOptions struct {
prioritiesReader io.Reader
furiganaReader io.Reader
jlptReaders map[string]io.Reader
namesReader io.Reader
kradfileReader io.Reader
radkfileReader io.Reader
}
// WithKradfile supplies an io.Reader of kradfile JSON to embed kanji component decompositions.
func WithKradfile(r io.Reader) BuildOption {
return func(o *buildOptions) {
o.kradfileReader = r
}
}
// WithRadkfile supplies an io.Reader of radkfile JSON to embed radical to kanji mappings.
func WithRadkfile(r io.Reader) BuildOption {
return func(o *buildOptions) {
o.radkfileReader = r
}
}
// WithPriorities supplies an io.Reader of EDRDG JMdict XML (plain or gzip-compressed)
// to extract frequency scores for ranking.
func WithPriorities(r io.Reader) BuildOption {
return func(o *buildOptions) {
o.prioritiesReader = r
}
}
// WithFurigana supplies an io.Reader of JmdictFurigana JSON to embed per-kanji furigana.
func WithFurigana(r io.Reader) BuildOption {
return func(o *buildOptions) {
o.furiganaReader = r
}
}
// WithJLPT supplies readers for JLPT deck CSVs mapped by level ("N5".."N1").
func WithJLPT(levels map[string]io.Reader) BuildOption {
return func(o *buildOptions) {
o.jlptReaders = levels
}
}
// WithNames supplies an io.Reader of jmnedict JSON to append proper name entries.
func WithNames(r io.Reader) BuildOption {
return func(o *buildOptions) {
o.namesReader = r
}
}
type stringInterner struct {
m map[string]uint32
list []string
}
func newStringInterner() *stringInterner {
return &stringInterner{
m: make(map[string]uint32),
list: make([]string, 0, 1024),
}
}
func (si *stringInterner) Intern(s string) uint32 {
if id, ok := si.m[s]; ok {
return id
}
id := uint32(len(si.list))
si.m[s] = id
si.list = append(si.list, s)
return id
}
type postingItem struct {
key string
priScore uint16 // lower is better / higher frequency
isCommon bool // true before false
isRegular bool // true before false (not search-only/irregular)
entryIndex uint32 // JMdict order
}
type rawKanjiForm struct {
Common bool `json:"common"`
Text string `json:"text"`
Tags []string `json:"tags"`
}
type rawKanaForm struct {
Common bool `json:"common"`
Text string `json:"text"`
Tags []string `json:"tags"`
AppliesToKanji []string `json:"appliesToKanji"`
}
type rawGloss struct {
Lang string `json:"lang"`
Text string `json:"text"`
}
type rawSense struct {
PartOfSpeech []string `json:"partOfSpeech"`
AppliesToKanji []string `json:"appliesToKanji"`
AppliesToKana []string `json:"appliesToKana"`
Field []string `json:"field"`
Dialect []string `json:"dialect"`
Misc []string `json:"misc"`
Info []string `json:"info"`
Gloss []rawGloss `json:"gloss"`
}
type rawWord struct {
ID string `json:"id"`
Kanji []rawKanjiForm `json:"kanji"`
Kana []rawKanaForm `json:"kana"`
Sense []rawSense `json:"sense"`
}
func (w *rawWord) Common() bool {
for _, k := range w.Kanji {
if k.Common {
return true
}
}
for _, r := range w.Kana {
if r.Common {
return true
}
}
return false
}
func isIrregularOrSearchOnly(tags []string) bool {
for _, t := range tags {
switch t {
case "sK", "sk", "ik", "iK", "rK", "rk", "io":
return true
}
}
return false
}
type entFormKey struct {
id uint32
form string
}
func jlptRank(lvl string) int {
switch strings.ToUpper(lvl) {
case "N5":
return 5
case "N4":
return 4
case "N3":
return 3
case "N2":
return 2
case "N1":
return 1
default:
return 0
}
}
// Build compiles JMdict, KANJIDIC2, and kanjium pitch accent data into a single
// memory-mapped dictionary file at out, optionally attaching EDRDG priority tags, furigana,
// JLPT levels, and JMnedict names.
func Build(jmdictReader, kanjidicReader, accentsReader io.Reader, out string, opts ...BuildOption) error {
var bo buildOptions
for _, opt := range opts {
opt(&bo)
}
if err := os.MkdirAll(filepath.Dir(out), 0755); err != nil {
return err
}
// 1. Parse pitch accents
accentsMap, err := parseAccents(accentsReader)
if err != nil {
return fmt.Errorf("parse accents: %w", err)
}
// 2. Parse kanjidic
kanjiList, err := parseKanjidic(kanjidicReader)
if err != nil {
return fmt.Errorf("parse kanjidic: %w", err)
}
sort.Slice(kanjiList, func(i, j int) bool {
rI, _ := utf8.DecodeRuneInString(kanjiList[i].Literal)
rJ, _ := utf8.DecodeRuneInString(kanjiList[j].Literal)
return rI < rJ
})
type kanjiMeta struct {
freq int
strokes int
}
kanjiMetaMap := make(map[rune]kanjiMeta, len(kanjiList))
for _, k := range kanjiList {
r, _ := utf8.DecodeRuneInString(k.Literal)
kanjiMetaMap[r] = kanjiMeta{freq: k.Freq, strokes: k.Strokes}
}
// 3. Parse EDRDG priority tags if provided
var formPris map[entFormKey][]string
var entryPris map[uint32][]string
if bo.prioritiesReader != nil {
fp, ep, err := parseJMdictXML(bo.prioritiesReader)
if err != nil {
return fmt.Errorf("parse priorities xml: %w", err)
}
formPris = fp
entryPris = ep
}
// 4. Parse JLPT CSVs if provided
var jlptMap map[string]string // key: expr + "\x00" + readingNorm -> level
if len(bo.jlptReaders) > 0 {
jlptMap, err = parseJLPTDecks(bo.jlptReaders)
if err != nil {
return fmt.Errorf("parse jlpt decks: %w", err)
}
}
// 4b. Parse KRADFILE / RADKFILE if provided
var kradMap map[rune][]rune
if bo.kradfileReader != nil {
km, err := parseKradfile(bo.kradfileReader)
if err != nil {
return fmt.Errorf("parse kradfile: %w", err)
}
kradMap = km
}
var radkMap map[rune][]rune
if bo.radkfileReader != nil {
rm, err := parseRadkfile(bo.radkfileReader)
if err != nil {
return fmt.Errorf("parse radkfile: %w", err)
}
radkMap = rm
} else if kradMap != nil {
radkMap = make(map[rune][]rune)
for k, comps := range kradMap {
for _, c := range comps {
radkMap[c] = append(radkMap[c], k)
}
}
}
// Ensure both display form and placeholder form exist in radkMap
for r, kList := range radkMap {
disp := dict.DisplayComponent(r)
if disp != r && len(radkMap[disp]) == 0 {
radkMap[disp] = append([]rune(nil), kList...)
}
if orig, ok := dict.ToPlaceholderForm[r]; ok && orig != r && len(radkMap[orig]) == 0 {
radkMap[orig] = append([]rune(nil), kList...)
}
}
for rad, kList := range radkMap {
sort.Slice(kList, func(i, j int) bool {
rI, rJ := kList[i], kList[j]
mI, mJ := kanjiMetaMap[rI], kanjiMetaMap[rJ]
if mI.freq > 0 && mJ.freq > 0 {
if mI.freq != mJ.freq {
return mI.freq < mJ.freq
}
if mI.strokes != mJ.strokes {
return mI.strokes < mJ.strokes
}
return rI < rJ
}
if mI.freq > 0 && mJ.freq == 0 {
return true
}
if mI.freq == 0 && mJ.freq > 0 {
return false
}
if mI.strokes != mJ.strokes {
return mI.strokes < mJ.strokes
}
return rI < rJ
})
radkMap[rad] = kList
}
// 5. Stream JMdict words into temporary entries file
entryTempFile, err := os.CreateTemp("", "chirami-dict-entries-*.bin")
if err != nil {
return fmt.Errorf("create temp entries file: %w", err)
}
defer func() {
entryTempFile.Close()
os.Remove(entryTempFile.Name())
}()
interner := newStringInterner()
interner.Intern("*")
type kanjiWordItem struct {
priScore uint16
isCommon bool
isRegular bool
entryIndex uint32
}
kanjiWordsMap := make(map[rune][]kanjiWordItem)
entryOffsets := make([]uint32, 0, 300000)
postings := make([]postingItem, 0, 700000)
tagsMap := make(map[string]string)
validPairs := make(map[string]bool)
entryBufWriter := bufio.NewWriterSize(entryTempFile, 1<<20)
entryWriter := dict.NewByteWriter(4096)
var currentEntryOffset uint32
dec := json.NewDecoder(jmdictReader)
tok, err := dec.Token()
if err != nil {
return fmt.Errorf("invalid jmdict json: %w", err)
}
if delim, ok := tok.(json.Delim); !ok || delim != '{' {
return errors.New("expected { at root of jmdict json")
}
curEntryIndex := uint32(0)
for dec.More() {
tok, err := dec.Token()
if err != nil {
return err
}
key, ok := tok.(string)
if !ok {
continue
}
switch key {
case "tags":
if err := dec.Decode(&tagsMap); err != nil {
return fmt.Errorf("decode tags: %w", err)
}
case "words":
arrTok, err := dec.Token()
if err != nil {
return err
}
if delim, ok := arrTok.(json.Delim); !ok || delim != '[' {
return errors.New("expected [ for words")
}
for dec.More() {
var raw rawWord
if err := dec.Decode(&raw); err != nil {
return fmt.Errorf("decode word %d: %w", curEntryIndex, err)
}
e := dict.Entry{}
if idNum, err := strconv.ParseUint(raw.ID, 10, 32); err == nil {
e.ID = uint32(idNum)
}
isCommon := raw.Common()
if len(raw.Kanji) > 0 {
e.Kanji = make([]dict.Form, len(raw.Kanji))
for i, k := range raw.Kanji {
e.Kanji[i].Text = k.Text
e.Kanji[i].Common = k.Common
if len(k.Tags) > 0 {
e.Kanji[i].Tags = make([]string, len(k.Tags))
for t, tag := range k.Tags {
interner.Intern(tag)
e.Kanji[i].Tags[t] = tag
}
}
}
}
if len(raw.Kana) > 0 {
e.Kana = make([]dict.Form, len(raw.Kana))
for i, r := range raw.Kana {
e.Kana[i].Text = r.Text
e.Kana[i].Common = r.Common
if len(r.Tags) > 0 {
e.Kana[i].Tags = make([]string, len(r.Tags))
for t, tag := range r.Tags {
interner.Intern(tag)
e.Kana[i].Tags[t] = tag
}
}
}
}
if len(raw.Sense) > 0 {
e.Senses = make([]dict.Sense, len(raw.Sense))
for i, s := range raw.Sense {
if len(s.PartOfSpeech) > 0 {
e.Senses[i].POS = make([]string, len(s.PartOfSpeech))
for p, pos := range s.PartOfSpeech {
interner.Intern(pos)
e.Senses[i].POS[p] = pos
}
}
if len(s.Misc) > 0 {
e.Senses[i].Misc = make([]string, len(s.Misc))
for m, misc := range s.Misc {
interner.Intern(misc)
e.Senses[i].Misc[m] = misc
}
}
if len(s.Field) > 0 {
e.Senses[i].Field = make([]string, len(s.Field))
for f, fld := range s.Field {
interner.Intern(fld)
e.Senses[i].Field[f] = fld
}
}
if len(s.Dialect) > 0 {
e.Senses[i].Dialect = make([]string, len(s.Dialect))
for d, dia := range s.Dialect {
interner.Intern(dia)
e.Senses[i].Dialect[d] = dia
}
}
e.Senses[i].Info = s.Info
if len(s.Gloss) > 0 {
e.Senses[i].Gloss = make([]string, 0, len(s.Gloss))
for _, g := range s.Gloss {
if g.Text != "" {
e.Senses[i].Gloss = append(e.Senses[i].Gloss, g.Text)
}
}
}
if len(s.AppliesToKanji) > 0 {
e.Senses[i].AppliesToKanji = make([]string, len(s.AppliesToKanji))
for a, atk := range s.AppliesToKanji {
interner.Intern(atk)
e.Senses[i].AppliesToKanji[a] = atk
}
}
if len(s.AppliesToKana) > 0 {
e.Senses[i].AppliesToKana = make([]string, len(s.AppliesToKana))
for a, atk := range s.AppliesToKana {
interner.Intern(atk)
e.Senses[i].AppliesToKana[a] = atk
}
}
}
}
attachPitch(&e, accentsMap)
// Match JLPT level if map provided
if len(jlptMap) > 0 {
var bestLevel string
for _, k := range raw.Kanji {
for _, r := range raw.Kana {
key := k.Text + "\x00" + jp.ToHiragana(r.Text)
if lvl, ok := jlptMap[key]; ok {
if jlptRank(lvl) > jlptRank(bestLevel) {
bestLevel = lvl
}
}
}
}
// Also check kana-only keys
for _, r := range raw.Kana {
norm := jp.ToHiragana(r.Text)
key := norm + "\x00" + norm
if lvl, ok := jlptMap[key]; ok {
if jlptRank(lvl) > jlptRank(bestLevel) {
bestLevel = lvl
}
}
}
e.JLPT = bestLevel
}
// Record valid (word, reading) pairs for furigana filtering
if bo.furiganaReader != nil {
for _, k := range raw.Kanji {
for _, r := range raw.Kana {
validPairs[k.Text+"\x00"+jp.ToHiragana(r.Text)] = true
}
}
}
entryWriter.Reset()
dict.EncodeEntry(entryWriter, &e, interner.m)
n, err := entryBufWriter.Write(entryWriter.Buf)
if err != nil {
return fmt.Errorf("write entry %d: %w", curEntryIndex, err)
}
entryOffsets = append(entryOffsets, currentEntryOffset)
currentEntryOffset += uint32(n)
// Calculate priorities & post keys
entryPrisList := entryPris[e.ID]
inKana := dict.WrittenInKana(e)
type keyMeta struct {
priScore uint16
isCommon bool
isRegular bool
}
entryKeys := make(map[string]keyMeta)
addKey := func(k string, formText string, isKanaKey, formCommon, irregular bool) {
if k == "" {
return
}
var fPris []string
if formPris != nil {
fPris = formPris[entFormKey{id: e.ID, form: formText}]
}
ePris := entryPrisList
if isCommon && !formCommon {
ePris = nil
}
score := calcPriScore(fPris, ePris)
if isKanaKey && inKana {
if score > 30 {
score -= 30
} else {
score = 0
}
}
reg := !irregular
if existing, ok := entryKeys[k]; !ok {
entryKeys[k] = keyMeta{priScore: score, isCommon: formCommon, isRegular: reg}
} else {
newScore := existing.priScore
if score < newScore {
newScore = score
}
entryKeys[k] = keyMeta{
priScore: newScore,
isCommon: existing.isCommon || formCommon,
isRegular: existing.isRegular || reg,
}
}
}
for _, k := range raw.Kanji {
addKey(k.Text, k.Text, false, k.Common, isIrregularOrSearchOnly(k.Tags))
}
for _, r := range raw.Kana {
norm := jp.ToHiragana(r.Text)
addKey(norm, r.Text, true, r.Common, isIrregularOrSearchOnly(r.Tags))
}
for k, meta := range entryKeys {
postings = append(postings, postingItem{
key: k,
priScore: meta.priScore,
isCommon: meta.isCommon,
isRegular: meta.isRegular,
entryIndex: curEntryIndex,
})
}
// Index entries by kanji for WordsWith
seenKanjiInEntry := make(map[rune]bool)
for _, kForm := range raw.Kanji {
for _, r := range kForm.Text {
if jp.IsKanji(r) && !seenKanjiInEntry[r] {
seenKanjiInEntry[r] = true
}
}
}
for r := range seenKanjiInEntry {
allSearchOnly := true
for _, kForm := range raw.Kanji {
if strings.ContainsRune(kForm.Text, r) {
if !slices.Contains(kForm.Tags, "sK") {
allSearchOnly = false
break
}
}
}
if allSearchOnly {
continue
}
bestScore := uint16(1000)
hasRegular := false
hasCommonForm := false
for _, kForm := range raw.Kanji {
if strings.ContainsRune(kForm.Text, r) && !slices.Contains(kForm.Tags, "sK") {
var fPris []string
if formPris != nil {
fPris = formPris[entFormKey{id: e.ID, form: kForm.Text}]
}
score := calcPriScore(fPris, entryPrisList)
if score < bestScore {
bestScore = score
}
if !isIrregularOrSearchOnly(kForm.Tags) {
hasRegular = true
}
if kForm.Common {
hasCommonForm = true
}
}
}
commonFlag := isCommon || hasCommonForm
kanjiWordsMap[r] = append(kanjiWordsMap[r], kanjiWordItem{
priScore: bestScore,
isCommon: commonFlag,
isRegular: hasRegular,
entryIndex: curEntryIndex,
})
}
curEntryIndex++
}
if _, err := dec.Token(); err != nil {
return err
}
default:
var discard any
if err := dec.Decode(&discard); err != nil {
return err
}
}
}
// 6. Optional: Stream JMnedict names
if bo.namesReader != nil {
nameIndex, err := streamNames(bo.namesReader, curEntryIndex, entryBufWriter, entryWriter, interner, &entryOffsets, &postings, ¤tEntryOffset)
if err != nil {
return fmt.Errorf("stream names: %w", err)
}
curEntryIndex = nameIndex
}
if err := entryBufWriter.Flush(); err != nil {
return err
}
// 7. Sort postings
sort.Slice(postings, func(i, j int) bool {
if postings[i].key != postings[j].key {
return postings[i].key < postings[j].key
}
if postings[i].priScore != postings[j].priScore {
return postings[i].priScore < postings[j].priScore
}
if postings[i].isCommon != postings[j].isCommon {
return postings[i].isCommon
}
if postings[i].isRegular != postings[j].isRegular {
return postings[i].isRegular
}
return postings[i].entryIndex < postings[j].entryIndex
})
// 8. Build key index, key data, and posting array
var keyIndexBuf bytes.Buffer
var keyDataBuf bytes.Buffer
var postingDataBuf bytes.Buffer
var rec [dict.KeyIndexRecordSize]byte
numUniqueKeys := uint32(0)
totalPostings := uint32(0)
i := 0
for i < len(postings) {
k := postings[i].key
start := i
for i < len(postings) && postings[i].key == k {
i++
}
group := postings[start:i]
keyOffset := uint32(keyDataBuf.Len())
keyBytes := []byte(k)
keyDataBuf.Write(keyBytes)
postOffset := totalPostings
pCount := uint16(len(group))
totalPostings += uint32(pCount)
for _, p := range group {
var postBytes [4]byte
binary.LittleEndian.PutUint32(postBytes[:], p.entryIndex)
postingDataBuf.Write(postBytes[:])
}
binary.LittleEndian.PutUint32(rec[0:4], keyOffset)
binary.LittleEndian.PutUint16(rec[4:6], uint16(len(keyBytes)))
binary.LittleEndian.PutUint16(rec[6:8], pCount)
binary.LittleEndian.PutUint32(rec[8:12], postOffset)
keyIndexBuf.Write(rec[:])
numUniqueKeys++
}
// 9. Build interned strings section
var internedBuf bytes.Buffer
var scratch [10]byte
for _, s := range interner.list {
n := binary.PutUvarint(scratch[:], uint64(len(s)))
internedBuf.Write(scratch[:n])
internedBuf.WriteString(s)
}
// 10. Build tag map section
var tagMapBuf bytes.Buffer
for code, label := range tagsMap {
n := binary.PutUvarint(scratch[:], uint64(len(code)))
tagMapBuf.Write(scratch[:n])
tagMapBuf.WriteString(code)
n = binary.PutUvarint(scratch[:], uint64(len(label)))
tagMapBuf.Write(scratch[:n])
tagMapBuf.WriteString(label)
}
// 11. Build entry index section
var entryIndexBuf bytes.Buffer
for _, off := range entryOffsets {
var b [4]byte
binary.LittleEndian.PutUint32(b[:], off)
entryIndexBuf.Write(b[:])
}
// 12. Build kanji index and kanji data sections
var kanjiIndexBuf bytes.Buffer
var kanjiDataBuf bytes.Buffer
kanjiByteWriter := dict.NewByteWriter(1024)
var kRec [dict.KanjiIndexRecordSize]byte
for _, k := range kanjiList {
r, _ := utf8.DecodeRuneInString(k.Literal)
kOffset := uint32(kanjiDataBuf.Len())
kanjiByteWriter.Reset()
dict.EncodeKanji(kanjiByteWriter, &k)
kanjiDataBuf.Write(kanjiByteWriter.Buf)
binary.LittleEndian.PutUint32(kRec[0:4], uint32(r))
binary.LittleEndian.PutUint32(kRec[4:8], kOffset)
kanjiIndexBuf.Write(kRec[:])
}
// 13. Build furigana sections if furigana reader provided
var furiganaIndexBuf bytes.Buffer
var furiganaKeyBuf bytes.Buffer
var furiganaDataBuf bytes.Buffer
numFurigana := uint32(0)
if bo.furiganaReader != nil {
furiList, err := parseFurigana(bo.furiganaReader, validPairs)
if err != nil {
return fmt.Errorf("parse furigana: %w", err)
}
sort.Slice(furiList, func(i, j int) bool {
return furiList[i].key < furiList[j].key
})
furiByteWriter := dict.NewByteWriter(1024)
var fRec [dict.FuriganaIndexRecordSize]byte
for _, item := range furiList {
keyOff := uint32(furiganaKeyBuf.Len())
keyBytes := []byte(item.key)
furiganaKeyBuf.Write(keyBytes)
dataOff := uint32(furiganaDataBuf.Len())
furiByteWriter.Reset()
dict.EncodeRubies(furiByteWriter, item.rubies)
furiganaDataBuf.Write(furiByteWriter.Buf)
binary.LittleEndian.PutUint32(fRec[0:4], keyOff)
binary.LittleEndian.PutUint16(fRec[4:6], uint16(len(keyBytes)))
binary.LittleEndian.PutUint16(fRec[6:8], 0)
binary.LittleEndian.PutUint32(fRec[8:12], dataOff)
furiganaIndexBuf.Write(fRec[:])
numFurigana++
}
}
// 13b. Build KRADFILE sections
var kradIndexBuf bytes.Buffer
var kradDataBuf bytes.Buffer
numKrad := uint32(0)
if len(kradMap) > 0 {
sortedKrad := make([]rune, 0, len(kradMap))
for k := range kradMap {
sortedKrad = append(sortedKrad, k)
}
sort.Slice(sortedKrad, func(i, j int) bool {
return sortedKrad[i] < sortedKrad[j]
})
var rec [dict.KradIndexRecordSize]byte
for _, k := range sortedKrad {
comps := kradMap[k]
off := uint32(kradDataBuf.Len())
count := uint16(len(comps))
for _, c := range comps {
var b [4]byte
binary.LittleEndian.PutUint32(b[:], uint32(c))
kradDataBuf.Write(b[:])
}
binary.LittleEndian.PutUint32(rec[0:4], uint32(k))
binary.LittleEndian.PutUint32(rec[4:8], off)
binary.LittleEndian.PutUint16(rec[8:10], count)
binary.LittleEndian.PutUint16(rec[10:12], 0)
kradIndexBuf.Write(rec[:])
numKrad++
}
}
// 13c. Build RADKFILE sections
var radkIndexBuf bytes.Buffer
var radkDataBuf bytes.Buffer
numRadk := uint32(0)
if len(radkMap) > 0 {
sortedRadk := make([]rune, 0, len(radkMap))
for r := range radkMap {
sortedRadk = append(sortedRadk, r)
}
sort.Slice(sortedRadk, func(i, j int) bool {
return sortedRadk[i] < sortedRadk[j]
})
var rec [dict.RadkIndexRecordSize]byte
for _, rad := range sortedRadk {
kList := radkMap[rad]
off := uint32(radkDataBuf.Len())
count := uint32(len(kList))
for _, k := range kList {
var b [4]byte
binary.LittleEndian.PutUint32(b[:], uint32(k))
radkDataBuf.Write(b[:])
}
binary.LittleEndian.PutUint32(rec[0:4], uint32(rad))
binary.LittleEndian.PutUint32(rec[4:8], off)
binary.LittleEndian.PutUint32(rec[8:12], count)
radkIndexBuf.Write(rec[:])
numRadk++
}
}
// 13d. Build KanjiWords sections
var kanjiWordsIndexBuf bytes.Buffer
var kanjiWordsDataBuf bytes.Buffer
numKanjiWords := uint32(0)
sortedKanjiWords := make([]rune, 0, len(kanjiWordsMap))
for r := range kanjiWordsMap {
sortedKanjiWords = append(sortedKanjiWords, r)
}
sort.Slice(sortedKanjiWords, func(i, j int) bool {
return sortedKanjiWords[i] < sortedKanjiWords[j]
})
var kwRec [dict.KanjiWordsIndexRecordSize]byte
for _, r := range sortedKanjiWords {
items := kanjiWordsMap[r]
sort.Slice(items, func(i, j int) bool {
if items[i].priScore != items[j].priScore {
return items[i].priScore < items[j].priScore
}
if items[i].isCommon != items[j].isCommon {
return items[i].isCommon
}
if items[i].isRegular != items[j].isRegular {
return items[i].isRegular
}
return items[i].entryIndex < items[j].entryIndex
})
if len(items) > 64 {
items = items[:64]
}
off := uint32(kanjiWordsDataBuf.Len())
count := uint16(len(items))
for _, item := range items {
var b [4]byte
binary.LittleEndian.PutUint32(b[:], item.entryIndex)
kanjiWordsDataBuf.Write(b[:])
}
binary.LittleEndian.PutUint32(kwRec[0:4], uint32(r))
binary.LittleEndian.PutUint32(kwRec[4:8], off)
binary.LittleEndian.PutUint16(kwRec[8:10], count)
binary.LittleEndian.PutUint16(kwRec[10:12], 0)
kanjiWordsIndexBuf.Write(kwRec[:])
numKanjiWords++
}
// 14. Assemble final dictionary file
tmpOut := out + ".tmp"
outFile, err := os.OpenFile(tmpOut, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0644)
if err != nil {
return fmt.Errorf("create out file: %w", err)
}
defer func() {
outFile.Close()
os.Remove(tmpOut)
}()
bufOut := bufio.NewWriterSize(outFile, 1<<20)
var zeroHeader [dict.HeaderSize]byte
if _, err := bufOut.Write(zeroHeader[:]); err != nil {
return err
}
hdr := dict.Header{
Version: dict.CurrentVersion,
}
copy(hdr.Magic[:], dict.MagicString)
curOffset := uint32(dict.HeaderSize)
// Section 1: InternedStrings
hdr.InternedStrings.Offset = curOffset
hdr.InternedStrings.Count = uint32(len(interner.list))
hdr.InternedStrings.Size = uint32(internedBuf.Len())
bufOut.Write(internedBuf.Bytes())
curOffset += hdr.InternedStrings.Size
// Section 2: TagMap
hdr.TagMap.Offset = curOffset
hdr.TagMap.Count = uint32(len(tagsMap))
hdr.TagMap.Size = uint32(tagMapBuf.Len())
bufOut.Write(tagMapBuf.Bytes())
curOffset += hdr.TagMap.Size
// Section 3: KeyIndex
hdr.KeyIndex.Offset = curOffset
hdr.KeyIndex.Count = numUniqueKeys
hdr.KeyIndex.Size = uint32(keyIndexBuf.Len())
bufOut.Write(keyIndexBuf.Bytes())
curOffset += hdr.KeyIndex.Size
// Section 4: KeyData
hdr.KeyData.Offset = curOffset
hdr.KeyData.Size = uint32(keyDataBuf.Len())
bufOut.Write(keyDataBuf.Bytes())
curOffset += hdr.KeyData.Size
// Section 5: PostingData
hdr.PostingData.Offset = curOffset
hdr.PostingData.Count = totalPostings
hdr.PostingData.Size = uint32(postingDataBuf.Len())
bufOut.Write(postingDataBuf.Bytes())
curOffset += hdr.PostingData.Size
// Section 6: EntryIndex
hdr.EntryIndex.Offset = curOffset
hdr.EntryIndex.Count = uint32(len(entryOffsets))
hdr.EntryIndex.Size = uint32(entryIndexBuf.Len())
bufOut.Write(entryIndexBuf.Bytes())
curOffset += hdr.EntryIndex.Size
// Section 7: EntryData
if err := bufOut.Flush(); err != nil {
return err
}
hdr.EntryData.Offset = curOffset
hdr.EntryData.Size = currentEntryOffset
curOffset += currentEntryOffset
if _, err := entryTempFile.Seek(0, io.SeekStart); err != nil {
return err
}
if _, err := io.Copy(outFile, entryTempFile); err != nil {
return fmt.Errorf("copy entries to out: %w", err)
}
bufOut.Reset(outFile)
// Section 8: KanjiIndex
hdr.KanjiIndex.Offset = curOffset
hdr.KanjiIndex.Count = uint32(len(kanjiList))
hdr.KanjiIndex.Size = uint32(kanjiIndexBuf.Len())
bufOut.Write(kanjiIndexBuf.Bytes())
curOffset += hdr.KanjiIndex.Size
// Section 9: KanjiData
hdr.KanjiData.Offset = curOffset
hdr.KanjiData.Size = uint32(kanjiDataBuf.Len())
bufOut.Write(kanjiDataBuf.Bytes())
curOffset += hdr.KanjiData.Size
// Section 10: FuriganaIndex
hdr.FuriganaIndex.Offset = curOffset
hdr.FuriganaIndex.Count = numFurigana
hdr.FuriganaIndex.Size = uint32(furiganaIndexBuf.Len())
bufOut.Write(furiganaIndexBuf.Bytes())
curOffset += hdr.FuriganaIndex.Size
// Section 11: FuriganaKeyData
hdr.FuriganaKeyData.Offset = curOffset
hdr.FuriganaKeyData.Size = uint32(furiganaKeyBuf.Len())
bufOut.Write(furiganaKeyBuf.Bytes())
curOffset += hdr.FuriganaKeyData.Size
// Section 12: FuriganaData
hdr.FuriganaData.Offset = curOffset
hdr.FuriganaData.Size = uint32(furiganaDataBuf.Len())
bufOut.Write(furiganaDataBuf.Bytes())
curOffset += hdr.FuriganaData.Size
// Section 13: KradIndex
hdr.KradIndex.Offset = curOffset
hdr.KradIndex.Count = numKrad
hdr.KradIndex.Size = uint32(kradIndexBuf.Len())
bufOut.Write(kradIndexBuf.Bytes())
curOffset += hdr.KradIndex.Size
// Section 14: KradData
hdr.KradData.Offset = curOffset
hdr.KradData.Count = 0
hdr.KradData.Size = uint32(kradDataBuf.Len())
bufOut.Write(kradDataBuf.Bytes())
curOffset += hdr.KradData.Size
// Section 15: RadkIndex
hdr.RadkIndex.Offset = curOffset
hdr.RadkIndex.Count = numRadk
hdr.RadkIndex.Size = uint32(radkIndexBuf.Len())
bufOut.Write(radkIndexBuf.Bytes())
curOffset += hdr.RadkIndex.Size
// Section 16: RadkData
hdr.RadkData.Offset = curOffset
hdr.RadkData.Count = 0
hdr.RadkData.Size = uint32(radkDataBuf.Len())
bufOut.Write(radkDataBuf.Bytes())
curOffset += hdr.RadkData.Size
// Section 17: KanjiWordsIndex
hdr.KanjiWordsIndex.Offset = curOffset
hdr.KanjiWordsIndex.Count = numKanjiWords
hdr.KanjiWordsIndex.Size = uint32(kanjiWordsIndexBuf.Len())
bufOut.Write(kanjiWordsIndexBuf.Bytes())
curOffset += hdr.KanjiWordsIndex.Size
// Section 18: KanjiWordsData
hdr.KanjiWordsData.Offset = curOffset
hdr.KanjiWordsData.Count = 0
hdr.KanjiWordsData.Size = uint32(kanjiWordsDataBuf.Len())
bufOut.Write(kanjiWordsDataBuf.Bytes())
curOffset += hdr.KanjiWordsData.Size
if err := bufOut.Flush(); err != nil {
return err
}
if _, err := outFile.Seek(0, io.SeekStart); err != nil {
return err
}
headerBytes := hdr.Encode()
if _, err := outFile.Write(headerBytes); err != nil {
return err
}
if err := outFile.Sync(); err != nil {
return err
}
if err := outFile.Close(); err != nil {
return err
}
return os.Rename(tmpOut, out)
}
func parseJLPTDecks(readers map[string]io.Reader) (map[string]string, error) {
jlptMap := make(map[string]string)
// Process levels in order from hardest to easiest: N1, N2, N3, N4, N5
// so easier levels naturally overwrite if present
levels := []string{"N1", "N2", "N3", "N4", "N5"}
for _, lvl := range levels {
r, ok := readers[lvl]
if !ok {
r = readers[strings.ToLower(lvl)]
}
if r == nil {
continue
}
cr := csv.NewReader(r)
cr.FieldsPerRecord = -1
rows, err := cr.ReadAll()
if err != nil {
return nil, fmt.Errorf("read jlpt %s csv: %w", lvl, err)
}
for i, row := range rows {
if i == 0 || len(row) < 2 {
continue
}
exprRaw := strings.TrimSpace(row[0])
readingRaw := strings.TrimSpace(row[1])
subExprs := strings.Split(exprRaw, ";")
subReadings := strings.Split(readingRaw, ";")
for _, e := range subExprs {
e = strings.TrimSpace(e)
if e == "" {
continue
}
for _, rd := range subReadings {
rd = strings.TrimSpace(rd)
if rd == "" {
continue
}
key := e + "\x00" + jp.ToHiragana(rd)
oldLvl := jlptMap[key]
if oldLvl == "" || jlptRank(lvl) > jlptRank(oldLvl) {
jlptMap[key] = lvl
}
}
}
}
}
return jlptMap, nil
}
type rawNameTranslation struct {
Type []string `json:"type"`
Translation []struct {
Lang string `json:"lang"`
Text string `json:"text"`
} `json:"translation"`
}
type rawNameWord struct {
ID string `json:"id"`
Kanji []rawKanjiForm `json:"kanji"`
Kana []rawKanaForm `json:"kana"`
Translation []rawNameTranslation `json:"translation"`
}
func streamNames(
r io.Reader,
startEntryIndex uint32,
bufWriter *bufio.Writer,
entryWriter *dict.ByteWriter,
interner *stringInterner,
entryOffsets *[]uint32,
postings *[]postingItem,
currentOffset *uint32,
) (uint32, error) {
br := bufio.NewReader(r)
if head, err := br.Peek(3); err == nil && len(head) == 3 && head[0] == 0xef && head[1] == 0xbb && head[2] == 0xbf {
br.Discard(3)
}
dec := json.NewDecoder(br)
tok, err := dec.Token()
if err != nil {
return startEntryIndex, err
}
if delim, ok := tok.(json.Delim); !ok || delim != '{' {
return startEntryIndex, errors.New("expected { at root of jmnedict")
}
curIdx := startEntryIndex
for dec.More() {
tok, err := dec.Token()
if err != nil {
return curIdx, err
}
key, ok := tok.(string)
if !ok {
continue
}
if key == "words" {
arrTok, err := dec.Token()
if err != nil {
return curIdx, err
}
if delim, ok := arrTok.(json.Delim); !ok || delim != '[' {
return curIdx, errors.New("expected [ for words in jmnedict")
}
for dec.More() {
var raw rawNameWord
if err := dec.Decode(&raw); err != nil {
return curIdx, err
}
idNum, _ := strconv.ParseUint(raw.ID, 10, 32)
e := dict.Entry{
ID: uint32(idNum) + 100000000,
}
if len(raw.Kanji) > 0 {
e.Kanji = make([]dict.Form, len(raw.Kanji))
for i, k := range raw.Kanji {
e.Kanji[i].Text = k.Text
}
}
if len(raw.Kana) > 0 {
e.Kana = make([]dict.Form, len(raw.Kana))
for i, r := range raw.Kana {
e.Kana[i].Text = r.Text
}
}
if len(raw.Translation) > 0 {
e.Senses = make([]dict.Sense, len(raw.Translation))
for i, t := range raw.Translation {
if len(t.Type) > 0 {
e.Senses[i].Misc = make([]string, len(t.Type))
for m, mt := range t.Type {
interner.Intern(mt)
e.Senses[i].Misc[m] = mt
}
}
if len(t.Translation) > 0 {
e.Senses[i].Gloss = make([]string, len(t.Translation))
for g, tr := range t.Translation {
e.Senses[i].Gloss[g] = tr.Text
}
}
}
}
entryWriter.Reset()
dict.EncodeEntry(entryWriter, &e, interner.m)
n, err := bufWriter.Write(entryWriter.Buf)
if err != nil {
return curIdx, err
}
*entryOffsets = append(*entryOffsets, *currentOffset)
*currentOffset += uint32(n)
// Keys for this name, ranked after JMdict entries (priScore=60000)
nameKeys := make(map[string]bool)
for _, k := range raw.Kanji {
nameKeys[k.Text] = true
}
for _, r := range raw.Kana {
norm := jp.ToHiragana(r.Text)
nameKeys[norm] = true
}
for k := range nameKeys {
*postings = append(*postings, postingItem{
key: k,
priScore: 60000, // strictly ranked after all JMdict entries
isCommon: false,
isRegular: false,
entryIndex: curIdx,
})
}
curIdx++
}
if _, err := dec.Token(); err != nil {
return curIdx, err
}
} else {
var discard any
dec.Decode(&discard)
}
}
return curIdx, nil
}
func calcPriScore(formPris, entryPris []string) uint16 {
if len(formPris) == 0 && len(entryPris) == 0 {
return 1000
}
pris := append(append([]string(nil), formPris...), entryPris...)
var bestNF int = -1
for _, p := range pris {
if strings.HasPrefix(p, "nf") && len(p) == 4 {
if n, err := strconv.Atoi(p[2:]); err == nil {
if bestNF == -1 || n < bestNF {
bestNF = n
}
}
}
}
var base int
if bestNF != -1 {
base = bestNF * 10
} else {
hasStrong := false
hasWeak := false
for _, p := range pris {
switch p {
case "news1", "ichi1", "spec1", "gai1":
hasStrong = true
case "news2", "ichi2", "spec2", "gai2":
hasWeak = true
}
}
if hasStrong {
base = 250
} else if hasWeak {
base = 450
} else {
base = 600
}
}
seen := make(map[string]bool)
for _, p := range pris {
if seen[p] {
continue
}
seen[p] = true
switch p {
case "news1", "ichi1", "spec1", "gai1":
base -= 5
case "news2", "ichi2", "spec2", "gai2":
base -= 2
}
}
if base < 0 {
base = 0
}
if base > 1000 {
base = 1000
}
return uint16(base)
}
func parseJMdictXML(r io.Reader) (map[entFormKey][]string, map[uint32][]string, error) {
br := bufio.NewReader(r)
magic, err := br.Peek(2)
var reader io.Reader = br
if err == nil && len(magic) == 2 && magic[0] == 0x1f && magic[1] == 0x8b {
gz, err := gzip.NewReader(br)
if err != nil {
return nil, nil, err
}
defer gz.Close()
reader = gz
}
formPriorities := make(map[entFormKey][]string)
entryPriorities := make(map[uint32][]string)
scanner := bufio.NewScanner(reader)
buf := make([]byte, 64*1024)
scanner.Buffer(buf, 1024*1024)
var curEntSeq uint32
var curForm string
var curFormPris []string
var entryPris []string
flushForm := func() {
if curEntSeq > 0 && curForm != "" && len(curFormPris) > 0 {
k := entFormKey{id: curEntSeq, form: curForm}
formPriorities[k] = append([]string(nil), curFormPris...)
}
curForm = ""
curFormPris = nil
}
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if strings.HasPrefix(line, "<ent_seq>") && strings.HasSuffix(line, "</ent_seq>") {
flushForm()
if curEntSeq > 0 && len(entryPris) > 0 {
entryPriorities[curEntSeq] = append([]string(nil), entryPris...)
}
val := line[9 : len(line)-10]
if id, err := strconv.ParseUint(val, 10, 32); err == nil {
curEntSeq = uint32(id)
}
entryPris = nil
} else if strings.HasPrefix(line, "<keb>") && strings.HasSuffix(line, "</keb>") {
flushForm()
curForm = line[5 : len(line)-6]
} else if strings.HasPrefix(line, "<reb>") && strings.HasSuffix(line, "</reb>") {
flushForm()
curForm = line[5 : len(line)-6]
} else if strings.HasPrefix(line, "<ke_pri>") && strings.HasSuffix(line, "</ke_pri>") {
p := line[8 : len(line)-9]
curFormPris = append(curFormPris, p)
entryPris = append(entryPris, p)
} else if strings.HasPrefix(line, "<re_pri>") && strings.HasSuffix(line, "</re_pri>") {
p := line[8 : len(line)-9]
curFormPris = append(curFormPris, p)
entryPris = append(entryPris, p)
} else if strings.HasPrefix(line, "</entry>") {
flushForm()
if curEntSeq > 0 && len(entryPris) > 0 {
entryPriorities[curEntSeq] = append([]string(nil), entryPris...)
}
curEntSeq = 0
entryPris = nil
}
}
return formPriorities, entryPriorities, scanner.Err()
}
type furiParsed struct {
key string
rubies []dict.Ruby
}
func parseFurigana(r io.Reader, validPairs map[string]bool) ([]furiParsed, error) {
br := bufio.NewReader(r)
if head, err := br.Peek(3); err == nil && len(head) == 3 && head[0] == 0xef && head[1] == 0xbb && head[2] == 0xbf {
br.Discard(3)
}
dec := json.NewDecoder(br)
tok, err := dec.Token()
if err != nil {
return nil, err
}
if delim, ok := tok.(json.Delim); !ok || delim != '[' {
return nil, errors.New("expected [ at root of furigana json")
}
type rawRuby struct {
Ruby string `json:"ruby"`
Rt string `json:"rt"`
}
type rawFuriEntry struct {
Text string `json:"text"`
Reading string `json:"reading"`
Furigana []rawRuby `json:"furigana"`
}
var results []furiParsed
for dec.More() {
var raw rawFuriEntry
if err := dec.Decode(&raw); err != nil {
return nil, err
}
normReading := jp.ToHiragana(raw.Reading)
key := raw.Text + "\x00" + normReading
if validPairs != nil && !validPairs[key] {
continue
}
rubies := make([]dict.Ruby, len(raw.Furigana))
for i, fr := range raw.Furigana {
rubies[i] = dict.Ruby{
Text: fr.Ruby,
Ruby: fr.Rt,
}
}
results = append(results, furiParsed{
key: key,
rubies: rubies,
})
}
return results, nil
}
type rawKradfile struct {
Version string `json:"version"`
Kanji map[string][]string `json:"kanji"`
}
type rawRadkRadical struct {
StrokeCount int `json:"strokeCount"`
Kanji []string `json:"kanji"`
}
type rawRadkfile struct {
Version string `json:"version"`
Radicals map[string]rawRadkRadical `json:"radicals"`
}
func parseKradfile(r io.Reader) (map[rune][]rune, error) {
var raw rawKradfile
if err := json.NewDecoder(r).Decode(&raw); err != nil {
return nil, err
}
out := make(map[rune][]rune, len(raw.Kanji))
for kStr, cStrs := range raw.Kanji {
kRune, _ := utf8.DecodeRuneInString(kStr)
if kRune == utf8.RuneError {
continue
}
comps := make([]rune, 0, len(cStrs))
for _, cStr := range cStrs {
cRune, _ := utf8.DecodeRuneInString(cStr)
if cRune != utf8.RuneError {
comps = append(comps, dict.DisplayComponent(cRune))
}
}
out[kRune] = comps
}
return out, nil
}
func parseRadkfile(r io.Reader) (map[rune][]rune, error) {
var raw rawRadkfile
if err := json.NewDecoder(r).Decode(&raw); err != nil {
return nil, err
}
out := make(map[rune][]rune, len(raw.Radicals)*2)
for rStr, radData := range raw.Radicals {
rRune, _ := utf8.DecodeRuneInString(rStr)
if rRune == utf8.RuneError {
continue
}
kList := make([]rune, 0, len(radData.Kanji))
for _, kStr := range radData.Kanji {
kRune, _ := utf8.DecodeRuneInString(kStr)
if kRune != utf8.RuneError {
kList = append(kList, kRune)
}
}
out[rRune] = kList
disp := dict.DisplayComponent(rRune)
if disp != rRune {
out[disp] = append([]rune(nil), kList...)
}
}
return out, nil
}
func parseAccents(r io.Reader) (map[string][]int, error) {
accents := make(map[string][]int)
scanner := bufio.NewScanner(r)
buf := make([]byte, 64*1024)
scanner.Buffer(buf, 1024*1024)
reParen := regexp.MustCompile(`\([^)]*\)`)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
parts := strings.Split(line, "\t")
if len(parts) < 3 {
continue
}
word := parts[0]
reading := parts[1]
downstepsRaw := parts[2]
var steps []int
for _, item := range strings.Split(downstepsRaw, ",") {
cleaned := strings.TrimSpace(reParen.ReplaceAllString(item, ""))
if cleaned != "" {
if n, err := strconv.Atoi(cleaned); err == nil {
found := false
for _, s := range steps {
if s == n {
found = true
break
}
}
if !found {
steps = append(steps, n)
}
}
}
}
if len(steps) == 0 {
continue
}
key := pitchKey(word, reading)
accents[key] = mergeIntSlices(accents[key], steps)
if reading == "" {
accents[pitchKey(word, word)] = mergeIntSlices(accents[pitchKey(word, word)], steps)
}
wordNorm := jp.ToHiragana(word)
readingNorm := jp.ToHiragana(reading)
if wordNorm != word || readingNorm != reading {
accents[pitchKey(wordNorm, readingNorm)] = mergeIntSlices(accents[pitchKey(wordNorm, readingNorm)], steps)
}
}
return accents, scanner.Err()
}
func pitchKey(word, reading string) string {
return word + "\x00" + reading
}
func mergeIntSlices(a, b []int) []int {
res := append([]int(nil), a...)
for _, v := range b {
found := false
for _, cur := range res {
if cur == v {
found = true
break
}
}
if !found {
res = append(res, v)
}
}
return res
}
func attachPitch(e *dict.Entry, accentsMap map[string][]int) {
type readingPitch struct {
reading string
downsteps []int
}
var pitches []readingPitch
addPitch := func(reading string, steps []int) {
for i := range pitches {
if pitches[i].reading == reading {
pitches[i].downsteps = mergeIntSlices(pitches[i].downsteps, steps)
return
}
}
pitches = append(pitches, readingPitch{reading: reading, downsteps: steps})
}
if len(e.Kanji) > 0 {
for _, r := range e.Kana {
reading := r.Text
readingNorm := jp.ToHiragana(reading)
for _, k := range e.Kanji {
word := k.Text
if steps, ok := accentsMap[pitchKey(word, reading)]; ok {
addPitch(reading, steps)
} else if steps, ok := accentsMap[pitchKey(word, readingNorm)]; ok {
addPitch(reading, steps)
}
}
}
} else {
for _, r := range e.Kana {
reading := r.Text
readingNorm := jp.ToHiragana(reading)
if steps, ok := accentsMap[pitchKey(reading, reading)]; ok {
addPitch(reading, steps)
} else if steps, ok := accentsMap[pitchKey(readingNorm, readingNorm)]; ok {
addPitch(reading, steps)
}
}
}
if len(pitches) > 0 {
e.Pitch = make([]dict.Pitch, len(pitches))
for i, p := range pitches {
e.Pitch[i] = dict.Pitch{
Reading: p.reading,
Downsteps: p.downsteps,
}
}
}
}
type rawKanjidicChar struct {
Literal string `json:"literal"`
Misc struct {
Grade *int `json:"grade"`
StrokeCounts []int `json:"strokeCounts"`
Frequency *int `json:"frequency"`
JLPTLevel *int `json:"jlptLevel"`
} `json:"misc"`
ReadingMeaning struct {
Groups []struct {
Readings []struct {
Type string `json:"type"`
Value string `json:"value"`
} `json:"readings"`
Meanings []struct {
Lang string `json:"lang"`
Value string `json:"value"`
} `json:"meanings"`
} `json:"groups"`
Nanori []string `json:"nanori"`
} `json:"readingMeaning"`
}
func parseKanjidic(r io.Reader) ([]dict.Kanji, error) {
dec := json.NewDecoder(r)
tok, err := dec.Token()
if err != nil {
return nil, err
}
if delim, ok := tok.(json.Delim); !ok || delim != '{' {
return nil, errors.New("expected { at root of kanjidic")
}
var kanjiList []dict.Kanji
for dec.More() {
tok, err := dec.Token()
if err != nil {
return nil, err
}
key, ok := tok.(string)
if !ok {
continue
}
if key == "characters" {
arrTok, err := dec.Token()
if err != nil {
return nil, err
}
if delim, ok := arrTok.(json.Delim); !ok || delim != '[' {
return nil, errors.New("expected [ for characters")
}
for dec.More() {
var c rawKanjidicChar
if err := dec.Decode(&c); err != nil {
return nil, err
}
k := dict.Kanji{
Literal: c.Literal,
}
if len(c.Misc.StrokeCounts) > 0 {
k.Strokes = c.Misc.StrokeCounts[0]
}
if c.Misc.Grade != nil {
k.Grade = *c.Misc.Grade
}
if c.Misc.Frequency != nil {
k.Freq = *c.Misc.Frequency
}
if c.Misc.JLPTLevel != nil {
k.JLPT = *c.Misc.JLPTLevel
}
for _, g := range c.ReadingMeaning.Groups {
for _, rd := range g.Readings {
switch rd.Type {
case "ja_on":
k.On = append(k.On, rd.Value)
case "ja_kun":
k.Kun = append(k.Kun, rd.Value)
}
}
for _, m := range g.Meanings {
if m.Lang == "en" {
k.Meanings = append(k.Meanings, m.Value)
}
}
}
k.Nanori = c.ReadingMeaning.Nanori
kanjiList = append(kanjiList, k)
}
dec.Token()
} else {
var discard any
dec.Decode(&discard)
}
}
return kanjiList, nil
}