package build

// Attribution:
// JLPT word lists are based on data from Jonathan Waller's JLPT resources
// (http://www.tanos.co.uk/jlpt/), licensed under Creative Commons Attribution (CC BY),
// curated in github.com/jamsinclair/open-anki-jlpt-decks.

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/binary"
	"encoding/csv"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"slices"
	"sort"
	"strconv"
	"strings"
	"unicode/utf8"

	"chirami/internal/dict"
	"chirami/internal/jp"
)

type BuildOption func(*buildOptions)

type buildOptions struct {
	prioritiesReader io.Reader
	furiganaReader   io.Reader
	jlptReaders      map[string]io.Reader
	namesReader      io.Reader
	kradfileReader   io.Reader
	radkfileReader   io.Reader
}

// WithKradfile supplies an io.Reader of kradfile JSON to embed kanji component decompositions.
func WithKradfile(r io.Reader) BuildOption {
	return func(o *buildOptions) {
		o.kradfileReader = r
	}
}

// WithRadkfile supplies an io.Reader of radkfile JSON to embed radical to kanji mappings.
func WithRadkfile(r io.Reader) BuildOption {
	return func(o *buildOptions) {
		o.radkfileReader = r
	}
}

// WithPriorities supplies an io.Reader of EDRDG JMdict XML (plain or gzip-compressed)
// to extract frequency scores for ranking.
func WithPriorities(r io.Reader) BuildOption {
	return func(o *buildOptions) {
		o.prioritiesReader = r
	}
}

// WithFurigana supplies an io.Reader of JmdictFurigana JSON to embed per-kanji furigana.
func WithFurigana(r io.Reader) BuildOption {
	return func(o *buildOptions) {
		o.furiganaReader = r
	}
}

// WithJLPT supplies readers for JLPT deck CSVs mapped by level ("N5".."N1").
func WithJLPT(levels map[string]io.Reader) BuildOption {
	return func(o *buildOptions) {
		o.jlptReaders = levels
	}
}

// WithNames supplies an io.Reader of jmnedict JSON to append proper name entries.
func WithNames(r io.Reader) BuildOption {
	return func(o *buildOptions) {
		o.namesReader = r
	}
}

type stringInterner struct {
	m    map[string]uint32
	list []string
}

func newStringInterner() *stringInterner {
	return &stringInterner{
		m:    make(map[string]uint32),
		list: make([]string, 0, 1024),
	}
}

func (si *stringInterner) Intern(s string) uint32 {
	if id, ok := si.m[s]; ok {
		return id
	}
	id := uint32(len(si.list))
	si.m[s] = id
	si.list = append(si.list, s)
	return id
}

type postingItem struct {
	key        string
	priScore   uint16 // lower is better / higher frequency
	isCommon   bool   // true before false
	isRegular  bool   // true before false (not search-only/irregular)
	entryIndex uint32 // JMdict order
}

type rawKanjiForm struct {
	Common bool     `json:"common"`
	Text   string   `json:"text"`
	Tags   []string `json:"tags"`
}

type rawKanaForm struct {
	Common         bool     `json:"common"`
	Text           string   `json:"text"`
	Tags           []string `json:"tags"`
	AppliesToKanji []string `json:"appliesToKanji"`
}

type rawGloss struct {
	Lang string `json:"lang"`
	Text string `json:"text"`
}

type rawSense struct {
	PartOfSpeech   []string   `json:"partOfSpeech"`
	AppliesToKanji []string   `json:"appliesToKanji"`
	AppliesToKana  []string   `json:"appliesToKana"`
	Field          []string   `json:"field"`
	Dialect        []string   `json:"dialect"`
	Misc           []string   `json:"misc"`
	Info           []string   `json:"info"`
	Gloss          []rawGloss `json:"gloss"`
}

type rawWord struct {
	ID    string         `json:"id"`
	Kanji []rawKanjiForm `json:"kanji"`
	Kana  []rawKanaForm  `json:"kana"`
	Sense []rawSense     `json:"sense"`
}

func (w *rawWord) Common() bool {
	for _, k := range w.Kanji {
		if k.Common {
			return true
		}
	}
	for _, r := range w.Kana {
		if r.Common {
			return true
		}
	}
	return false
}

func isIrregularOrSearchOnly(tags []string) bool {
	for _, t := range tags {
		switch t {
		case "sK", "sk", "ik", "iK", "rK", "rk", "io":
			return true
		}
	}
	return false
}

type entFormKey struct {
	id   uint32
	form string
}

func jlptRank(lvl string) int {
	switch strings.ToUpper(lvl) {
	case "N5":
		return 5
	case "N4":
		return 4
	case "N3":
		return 3
	case "N2":
		return 2
	case "N1":
		return 1
	default:
		return 0
	}
}

// Build compiles JMdict, KANJIDIC2, and kanjium pitch accent data into a single
// memory-mapped dictionary file at out, optionally attaching EDRDG priority tags, furigana,
// JLPT levels, and JMnedict names.
func Build(jmdictReader, kanjidicReader, accentsReader io.Reader, out string, opts ...BuildOption) error {
	var bo buildOptions
	for _, opt := range opts {
		opt(&bo)
	}

	if err := os.MkdirAll(filepath.Dir(out), 0755); err != nil {
		return err
	}

	// 1. Parse pitch accents
	accentsMap, err := parseAccents(accentsReader)
	if err != nil {
		return fmt.Errorf("parse accents: %w", err)
	}

	// 2. Parse kanjidic
	kanjiList, err := parseKanjidic(kanjidicReader)
	if err != nil {
		return fmt.Errorf("parse kanjidic: %w", err)
	}

	sort.Slice(kanjiList, func(i, j int) bool {
		rI, _ := utf8.DecodeRuneInString(kanjiList[i].Literal)
		rJ, _ := utf8.DecodeRuneInString(kanjiList[j].Literal)
		return rI < rJ
	})

	type kanjiMeta struct {
		freq    int
		strokes int
	}
	kanjiMetaMap := make(map[rune]kanjiMeta, len(kanjiList))
	for _, k := range kanjiList {
		r, _ := utf8.DecodeRuneInString(k.Literal)
		kanjiMetaMap[r] = kanjiMeta{freq: k.Freq, strokes: k.Strokes}
	}

	// 3. Parse EDRDG priority tags if provided
	var formPris map[entFormKey][]string
	var entryPris map[uint32][]string
	if bo.prioritiesReader != nil {
		fp, ep, err := parseJMdictXML(bo.prioritiesReader)
		if err != nil {
			return fmt.Errorf("parse priorities xml: %w", err)
		}
		formPris = fp
		entryPris = ep
	}

	// 4. Parse JLPT CSVs if provided
	var jlptMap map[string]string // key: expr + "\x00" + readingNorm -> level
	if len(bo.jlptReaders) > 0 {
		jlptMap, err = parseJLPTDecks(bo.jlptReaders)
		if err != nil {
			return fmt.Errorf("parse jlpt decks: %w", err)
		}
	}

	// 4b. Parse KRADFILE / RADKFILE if provided
	var kradMap map[rune][]rune
	if bo.kradfileReader != nil {
		km, err := parseKradfile(bo.kradfileReader)
		if err != nil {
			return fmt.Errorf("parse kradfile: %w", err)
		}
		kradMap = km
	}

	var radkMap map[rune][]rune
	if bo.radkfileReader != nil {
		rm, err := parseRadkfile(bo.radkfileReader)
		if err != nil {
			return fmt.Errorf("parse radkfile: %w", err)
		}
		radkMap = rm
	} else if kradMap != nil {
		radkMap = make(map[rune][]rune)
		for k, comps := range kradMap {
			for _, c := range comps {
				radkMap[c] = append(radkMap[c], k)
			}
		}
	}

	// Ensure both display form and placeholder form exist in radkMap
	for r, kList := range radkMap {
		disp := dict.DisplayComponent(r)
		if disp != r && len(radkMap[disp]) == 0 {
			radkMap[disp] = append([]rune(nil), kList...)
		}
		if orig, ok := dict.ToPlaceholderForm[r]; ok && orig != r && len(radkMap[orig]) == 0 {
			radkMap[orig] = append([]rune(nil), kList...)
		}
	}

	for rad, kList := range radkMap {
		sort.Slice(kList, func(i, j int) bool {
			rI, rJ := kList[i], kList[j]
			mI, mJ := kanjiMetaMap[rI], kanjiMetaMap[rJ]
			if mI.freq > 0 && mJ.freq > 0 {
				if mI.freq != mJ.freq {
					return mI.freq < mJ.freq
				}
				if mI.strokes != mJ.strokes {
					return mI.strokes < mJ.strokes
				}
				return rI < rJ
			}
			if mI.freq > 0 && mJ.freq == 0 {
				return true
			}
			if mI.freq == 0 && mJ.freq > 0 {
				return false
			}
			if mI.strokes != mJ.strokes {
				return mI.strokes < mJ.strokes
			}
			return rI < rJ
		})
		radkMap[rad] = kList
	}

	// 5. Stream JMdict words into temporary entries file
	entryTempFile, err := os.CreateTemp("", "chirami-dict-entries-*.bin")
	if err != nil {
		return fmt.Errorf("create temp entries file: %w", err)
	}
	defer func() {
		entryTempFile.Close()
		os.Remove(entryTempFile.Name())
	}()

	interner := newStringInterner()
	interner.Intern("*")

	type kanjiWordItem struct {
		priScore   uint16
		isCommon   bool
		isRegular  bool
		entryIndex uint32
	}
	kanjiWordsMap := make(map[rune][]kanjiWordItem)

	entryOffsets := make([]uint32, 0, 300000)
	postings := make([]postingItem, 0, 700000)
	tagsMap := make(map[string]string)
	validPairs := make(map[string]bool)

	entryBufWriter := bufio.NewWriterSize(entryTempFile, 1<<20)
	entryWriter := dict.NewByteWriter(4096)
	var currentEntryOffset uint32

	dec := json.NewDecoder(jmdictReader)
	tok, err := dec.Token()
	if err != nil {
		return fmt.Errorf("invalid jmdict json: %w", err)
	}
	if delim, ok := tok.(json.Delim); !ok || delim != '{' {
		return errors.New("expected { at root of jmdict json")
	}

	curEntryIndex := uint32(0)

	for dec.More() {
		tok, err := dec.Token()
		if err != nil {
			return err
		}
		key, ok := tok.(string)
		if !ok {
			continue
		}

		switch key {
		case "tags":
			if err := dec.Decode(&tagsMap); err != nil {
				return fmt.Errorf("decode tags: %w", err)
			}
		case "words":
			arrTok, err := dec.Token()
			if err != nil {
				return err
			}
			if delim, ok := arrTok.(json.Delim); !ok || delim != '[' {
				return errors.New("expected [ for words")
			}

			for dec.More() {
				var raw rawWord
				if err := dec.Decode(&raw); err != nil {
					return fmt.Errorf("decode word %d: %w", curEntryIndex, err)
				}

				e := dict.Entry{}
				if idNum, err := strconv.ParseUint(raw.ID, 10, 32); err == nil {
					e.ID = uint32(idNum)
				}

				isCommon := raw.Common()

				if len(raw.Kanji) > 0 {
					e.Kanji = make([]dict.Form, len(raw.Kanji))
					for i, k := range raw.Kanji {
						e.Kanji[i].Text = k.Text
						e.Kanji[i].Common = k.Common
						if len(k.Tags) > 0 {
							e.Kanji[i].Tags = make([]string, len(k.Tags))
							for t, tag := range k.Tags {
								interner.Intern(tag)
								e.Kanji[i].Tags[t] = tag
							}
						}
					}
				}

				if len(raw.Kana) > 0 {
					e.Kana = make([]dict.Form, len(raw.Kana))
					for i, r := range raw.Kana {
						e.Kana[i].Text = r.Text
						e.Kana[i].Common = r.Common
						if len(r.Tags) > 0 {
							e.Kana[i].Tags = make([]string, len(r.Tags))
							for t, tag := range r.Tags {
								interner.Intern(tag)
								e.Kana[i].Tags[t] = tag
							}
						}
					}
				}

				if len(raw.Sense) > 0 {
					e.Senses = make([]dict.Sense, len(raw.Sense))
					for i, s := range raw.Sense {
						if len(s.PartOfSpeech) > 0 {
							e.Senses[i].POS = make([]string, len(s.PartOfSpeech))
							for p, pos := range s.PartOfSpeech {
								interner.Intern(pos)
								e.Senses[i].POS[p] = pos
							}
						}
						if len(s.Misc) > 0 {
							e.Senses[i].Misc = make([]string, len(s.Misc))
							for m, misc := range s.Misc {
								interner.Intern(misc)
								e.Senses[i].Misc[m] = misc
							}
						}
						if len(s.Field) > 0 {
							e.Senses[i].Field = make([]string, len(s.Field))
							for f, fld := range s.Field {
								interner.Intern(fld)
								e.Senses[i].Field[f] = fld
							}
						}
						if len(s.Dialect) > 0 {
							e.Senses[i].Dialect = make([]string, len(s.Dialect))
							for d, dia := range s.Dialect {
								interner.Intern(dia)
								e.Senses[i].Dialect[d] = dia
							}
						}
						e.Senses[i].Info = s.Info
						if len(s.Gloss) > 0 {
							e.Senses[i].Gloss = make([]string, 0, len(s.Gloss))
							for _, g := range s.Gloss {
								if g.Text != "" {
									e.Senses[i].Gloss = append(e.Senses[i].Gloss, g.Text)
								}
							}
						}
						if len(s.AppliesToKanji) > 0 {
							e.Senses[i].AppliesToKanji = make([]string, len(s.AppliesToKanji))
							for a, atk := range s.AppliesToKanji {
								interner.Intern(atk)
								e.Senses[i].AppliesToKanji[a] = atk
							}
						}
						if len(s.AppliesToKana) > 0 {
							e.Senses[i].AppliesToKana = make([]string, len(s.AppliesToKana))
							for a, atk := range s.AppliesToKana {
								interner.Intern(atk)
								e.Senses[i].AppliesToKana[a] = atk
							}
						}
					}
				}

				attachPitch(&e, accentsMap)

				// Match JLPT level if map provided
				if len(jlptMap) > 0 {
					var bestLevel string
					for _, k := range raw.Kanji {
						for _, r := range raw.Kana {
							key := k.Text + "\x00" + jp.ToHiragana(r.Text)
							if lvl, ok := jlptMap[key]; ok {
								if jlptRank(lvl) > jlptRank(bestLevel) {
									bestLevel = lvl
								}
							}
						}
					}
					// Also check kana-only keys
					for _, r := range raw.Kana {
						norm := jp.ToHiragana(r.Text)
						key := norm + "\x00" + norm
						if lvl, ok := jlptMap[key]; ok {
							if jlptRank(lvl) > jlptRank(bestLevel) {
								bestLevel = lvl
							}
						}
					}
					e.JLPT = bestLevel
				}

				// Record valid (word, reading) pairs for furigana filtering
				if bo.furiganaReader != nil {
					for _, k := range raw.Kanji {
						for _, r := range raw.Kana {
							validPairs[k.Text+"\x00"+jp.ToHiragana(r.Text)] = true
						}
					}
				}

				entryWriter.Reset()
				dict.EncodeEntry(entryWriter, &e, interner.m)
				n, err := entryBufWriter.Write(entryWriter.Buf)
				if err != nil {
					return fmt.Errorf("write entry %d: %w", curEntryIndex, err)
				}
				entryOffsets = append(entryOffsets, currentEntryOffset)
				currentEntryOffset += uint32(n)

				// Calculate priorities & post keys
				entryPrisList := entryPris[e.ID]
				inKana := dict.WrittenInKana(e)

				type keyMeta struct {
					priScore  uint16
					isCommon  bool
					isRegular bool
				}
				entryKeys := make(map[string]keyMeta)

				addKey := func(k string, formText string, isKanaKey, formCommon, irregular bool) {
					if k == "" {
						return
					}
					var fPris []string
					if formPris != nil {
						fPris = formPris[entFormKey{id: e.ID, form: formText}]
					}
					ePris := entryPrisList
					if isCommon && !formCommon {
						ePris = nil
					}
					score := calcPriScore(fPris, ePris)
					if isKanaKey && inKana {
						if score > 30 {
							score -= 30
						} else {
							score = 0
						}
					}

					reg := !irregular
					if existing, ok := entryKeys[k]; !ok {
						entryKeys[k] = keyMeta{priScore: score, isCommon: formCommon, isRegular: reg}
					} else {
						newScore := existing.priScore
						if score < newScore {
							newScore = score
						}
						entryKeys[k] = keyMeta{
							priScore:  newScore,
							isCommon:  existing.isCommon || formCommon,
							isRegular: existing.isRegular || reg,
						}
					}
				}

				for _, k := range raw.Kanji {
					addKey(k.Text, k.Text, false, k.Common, isIrregularOrSearchOnly(k.Tags))
				}
				for _, r := range raw.Kana {
					norm := jp.ToHiragana(r.Text)
					addKey(norm, r.Text, true, r.Common, isIrregularOrSearchOnly(r.Tags))
				}

				for k, meta := range entryKeys {
					postings = append(postings, postingItem{
						key:        k,
						priScore:   meta.priScore,
						isCommon:   meta.isCommon,
						isRegular:  meta.isRegular,
						entryIndex: curEntryIndex,
					})
				}

				// Index entries by kanji for WordsWith
				seenKanjiInEntry := make(map[rune]bool)
				for _, kForm := range raw.Kanji {
					for _, r := range kForm.Text {
						if jp.IsKanji(r) && !seenKanjiInEntry[r] {
							seenKanjiInEntry[r] = true
						}
					}
				}

				for r := range seenKanjiInEntry {
					allSearchOnly := true
					for _, kForm := range raw.Kanji {
						if strings.ContainsRune(kForm.Text, r) {
							if !slices.Contains(kForm.Tags, "sK") {
								allSearchOnly = false
								break
							}
						}
					}
					if allSearchOnly {
						continue
					}

					bestScore := uint16(1000)
					hasRegular := false
					hasCommonForm := false
					for _, kForm := range raw.Kanji {
						if strings.ContainsRune(kForm.Text, r) && !slices.Contains(kForm.Tags, "sK") {
							var fPris []string
							if formPris != nil {
								fPris = formPris[entFormKey{id: e.ID, form: kForm.Text}]
							}
							score := calcPriScore(fPris, entryPrisList)
							if score < bestScore {
								bestScore = score
							}
							if !isIrregularOrSearchOnly(kForm.Tags) {
								hasRegular = true
							}
							if kForm.Common {
								hasCommonForm = true
							}
						}
					}
					commonFlag := isCommon || hasCommonForm
					kanjiWordsMap[r] = append(kanjiWordsMap[r], kanjiWordItem{
						priScore:   bestScore,
						isCommon:   commonFlag,
						isRegular:  hasRegular,
						entryIndex: curEntryIndex,
					})
				}

				curEntryIndex++
			}

			if _, err := dec.Token(); err != nil {
				return err
			}
		default:
			var discard any
			if err := dec.Decode(&discard); err != nil {
				return err
			}
		}
	}

	// 6. Optional: Stream JMnedict names
	if bo.namesReader != nil {
		nameIndex, err := streamNames(bo.namesReader, curEntryIndex, entryBufWriter, entryWriter, interner, &entryOffsets, &postings, &currentEntryOffset)
		if err != nil {
			return fmt.Errorf("stream names: %w", err)
		}
		curEntryIndex = nameIndex
	}

	if err := entryBufWriter.Flush(); err != nil {
		return err
	}

	// 7. Sort postings
	sort.Slice(postings, func(i, j int) bool {
		if postings[i].key != postings[j].key {
			return postings[i].key < postings[j].key
		}
		if postings[i].priScore != postings[j].priScore {
			return postings[i].priScore < postings[j].priScore
		}
		if postings[i].isCommon != postings[j].isCommon {
			return postings[i].isCommon
		}
		if postings[i].isRegular != postings[j].isRegular {
			return postings[i].isRegular
		}
		return postings[i].entryIndex < postings[j].entryIndex
	})

	// 8. Build key index, key data, and posting array
	var keyIndexBuf bytes.Buffer
	var keyDataBuf bytes.Buffer
	var postingDataBuf bytes.Buffer
	var rec [dict.KeyIndexRecordSize]byte

	numUniqueKeys := uint32(0)
	totalPostings := uint32(0)

	i := 0
	for i < len(postings) {
		k := postings[i].key
		start := i
		for i < len(postings) && postings[i].key == k {
			i++
		}
		group := postings[start:i]

		keyOffset := uint32(keyDataBuf.Len())
		keyBytes := []byte(k)
		keyDataBuf.Write(keyBytes)

		postOffset := totalPostings
		pCount := uint16(len(group))
		totalPostings += uint32(pCount)

		for _, p := range group {
			var postBytes [4]byte
			binary.LittleEndian.PutUint32(postBytes[:], p.entryIndex)
			postingDataBuf.Write(postBytes[:])
		}

		binary.LittleEndian.PutUint32(rec[0:4], keyOffset)
		binary.LittleEndian.PutUint16(rec[4:6], uint16(len(keyBytes)))
		binary.LittleEndian.PutUint16(rec[6:8], pCount)
		binary.LittleEndian.PutUint32(rec[8:12], postOffset)
		keyIndexBuf.Write(rec[:])

		numUniqueKeys++
	}

	// 9. Build interned strings section
	var internedBuf bytes.Buffer
	var scratch [10]byte
	for _, s := range interner.list {
		n := binary.PutUvarint(scratch[:], uint64(len(s)))
		internedBuf.Write(scratch[:n])
		internedBuf.WriteString(s)
	}

	// 10. Build tag map section
	var tagMapBuf bytes.Buffer
	for code, label := range tagsMap {
		n := binary.PutUvarint(scratch[:], uint64(len(code)))
		tagMapBuf.Write(scratch[:n])
		tagMapBuf.WriteString(code)

		n = binary.PutUvarint(scratch[:], uint64(len(label)))
		tagMapBuf.Write(scratch[:n])
		tagMapBuf.WriteString(label)
	}

	// 11. Build entry index section
	var entryIndexBuf bytes.Buffer
	for _, off := range entryOffsets {
		var b [4]byte
		binary.LittleEndian.PutUint32(b[:], off)
		entryIndexBuf.Write(b[:])
	}

	// 12. Build kanji index and kanji data sections
	var kanjiIndexBuf bytes.Buffer
	var kanjiDataBuf bytes.Buffer
	kanjiByteWriter := dict.NewByteWriter(1024)
	var kRec [dict.KanjiIndexRecordSize]byte

	for _, k := range kanjiList {
		r, _ := utf8.DecodeRuneInString(k.Literal)
		kOffset := uint32(kanjiDataBuf.Len())

		kanjiByteWriter.Reset()
		dict.EncodeKanji(kanjiByteWriter, &k)
		kanjiDataBuf.Write(kanjiByteWriter.Buf)

		binary.LittleEndian.PutUint32(kRec[0:4], uint32(r))
		binary.LittleEndian.PutUint32(kRec[4:8], kOffset)
		kanjiIndexBuf.Write(kRec[:])
	}

	// 13. Build furigana sections if furigana reader provided
	var furiganaIndexBuf bytes.Buffer
	var furiganaKeyBuf bytes.Buffer
	var furiganaDataBuf bytes.Buffer
	numFurigana := uint32(0)

	if bo.furiganaReader != nil {
		furiList, err := parseFurigana(bo.furiganaReader, validPairs)
		if err != nil {
			return fmt.Errorf("parse furigana: %w", err)
		}
		sort.Slice(furiList, func(i, j int) bool {
			return furiList[i].key < furiList[j].key
		})

		furiByteWriter := dict.NewByteWriter(1024)
		var fRec [dict.FuriganaIndexRecordSize]byte

		for _, item := range furiList {
			keyOff := uint32(furiganaKeyBuf.Len())
			keyBytes := []byte(item.key)
			furiganaKeyBuf.Write(keyBytes)

			dataOff := uint32(furiganaDataBuf.Len())
			furiByteWriter.Reset()
			dict.EncodeRubies(furiByteWriter, item.rubies)
			furiganaDataBuf.Write(furiByteWriter.Buf)

			binary.LittleEndian.PutUint32(fRec[0:4], keyOff)
			binary.LittleEndian.PutUint16(fRec[4:6], uint16(len(keyBytes)))
			binary.LittleEndian.PutUint16(fRec[6:8], 0)
			binary.LittleEndian.PutUint32(fRec[8:12], dataOff)
			furiganaIndexBuf.Write(fRec[:])
			numFurigana++
		}
	}

	// 13b. Build KRADFILE sections
	var kradIndexBuf bytes.Buffer
	var kradDataBuf bytes.Buffer
	numKrad := uint32(0)
	if len(kradMap) > 0 {
		sortedKrad := make([]rune, 0, len(kradMap))
		for k := range kradMap {
			sortedKrad = append(sortedKrad, k)
		}
		sort.Slice(sortedKrad, func(i, j int) bool {
			return sortedKrad[i] < sortedKrad[j]
		})
		var rec [dict.KradIndexRecordSize]byte
		for _, k := range sortedKrad {
			comps := kradMap[k]
			off := uint32(kradDataBuf.Len())
			count := uint16(len(comps))
			for _, c := range comps {
				var b [4]byte
				binary.LittleEndian.PutUint32(b[:], uint32(c))
				kradDataBuf.Write(b[:])
			}
			binary.LittleEndian.PutUint32(rec[0:4], uint32(k))
			binary.LittleEndian.PutUint32(rec[4:8], off)
			binary.LittleEndian.PutUint16(rec[8:10], count)
			binary.LittleEndian.PutUint16(rec[10:12], 0)
			kradIndexBuf.Write(rec[:])
			numKrad++
		}
	}

	// 13c. Build RADKFILE sections
	var radkIndexBuf bytes.Buffer
	var radkDataBuf bytes.Buffer
	numRadk := uint32(0)
	if len(radkMap) > 0 {
		sortedRadk := make([]rune, 0, len(radkMap))
		for r := range radkMap {
			sortedRadk = append(sortedRadk, r)
		}
		sort.Slice(sortedRadk, func(i, j int) bool {
			return sortedRadk[i] < sortedRadk[j]
		})
		var rec [dict.RadkIndexRecordSize]byte
		for _, rad := range sortedRadk {
			kList := radkMap[rad]
			off := uint32(radkDataBuf.Len())
			count := uint32(len(kList))
			for _, k := range kList {
				var b [4]byte
				binary.LittleEndian.PutUint32(b[:], uint32(k))
				radkDataBuf.Write(b[:])
			}
			binary.LittleEndian.PutUint32(rec[0:4], uint32(rad))
			binary.LittleEndian.PutUint32(rec[4:8], off)
			binary.LittleEndian.PutUint32(rec[8:12], count)
			radkIndexBuf.Write(rec[:])
			numRadk++
		}
	}

	// 13d. Build KanjiWords sections
	var kanjiWordsIndexBuf bytes.Buffer
	var kanjiWordsDataBuf bytes.Buffer
	numKanjiWords := uint32(0)

	sortedKanjiWords := make([]rune, 0, len(kanjiWordsMap))
	for r := range kanjiWordsMap {
		sortedKanjiWords = append(sortedKanjiWords, r)
	}
	sort.Slice(sortedKanjiWords, func(i, j int) bool {
		return sortedKanjiWords[i] < sortedKanjiWords[j]
	})

	var kwRec [dict.KanjiWordsIndexRecordSize]byte
	for _, r := range sortedKanjiWords {
		items := kanjiWordsMap[r]
		sort.Slice(items, func(i, j int) bool {
			if items[i].priScore != items[j].priScore {
				return items[i].priScore < items[j].priScore
			}
			if items[i].isCommon != items[j].isCommon {
				return items[i].isCommon
			}
			if items[i].isRegular != items[j].isRegular {
				return items[i].isRegular
			}
			return items[i].entryIndex < items[j].entryIndex
		})
		if len(items) > 64 {
			items = items[:64]
		}

		off := uint32(kanjiWordsDataBuf.Len())
		count := uint16(len(items))
		for _, item := range items {
			var b [4]byte
			binary.LittleEndian.PutUint32(b[:], item.entryIndex)
			kanjiWordsDataBuf.Write(b[:])
		}

		binary.LittleEndian.PutUint32(kwRec[0:4], uint32(r))
		binary.LittleEndian.PutUint32(kwRec[4:8], off)
		binary.LittleEndian.PutUint16(kwRec[8:10], count)
		binary.LittleEndian.PutUint16(kwRec[10:12], 0)
		kanjiWordsIndexBuf.Write(kwRec[:])
		numKanjiWords++
	}

	// 14. Assemble final dictionary file
	tmpOut := out + ".tmp"
	outFile, err := os.OpenFile(tmpOut, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0644)
	if err != nil {
		return fmt.Errorf("create out file: %w", err)
	}
	defer func() {
		outFile.Close()
		os.Remove(tmpOut)
	}()

	bufOut := bufio.NewWriterSize(outFile, 1<<20)

	var zeroHeader [dict.HeaderSize]byte
	if _, err := bufOut.Write(zeroHeader[:]); err != nil {
		return err
	}

	hdr := dict.Header{
		Version: dict.CurrentVersion,
	}
	copy(hdr.Magic[:], dict.MagicString)

	curOffset := uint32(dict.HeaderSize)

	// Section 1: InternedStrings
	hdr.InternedStrings.Offset = curOffset
	hdr.InternedStrings.Count = uint32(len(interner.list))
	hdr.InternedStrings.Size = uint32(internedBuf.Len())
	bufOut.Write(internedBuf.Bytes())
	curOffset += hdr.InternedStrings.Size

	// Section 2: TagMap
	hdr.TagMap.Offset = curOffset
	hdr.TagMap.Count = uint32(len(tagsMap))
	hdr.TagMap.Size = uint32(tagMapBuf.Len())
	bufOut.Write(tagMapBuf.Bytes())
	curOffset += hdr.TagMap.Size

	// Section 3: KeyIndex
	hdr.KeyIndex.Offset = curOffset
	hdr.KeyIndex.Count = numUniqueKeys
	hdr.KeyIndex.Size = uint32(keyIndexBuf.Len())
	bufOut.Write(keyIndexBuf.Bytes())
	curOffset += hdr.KeyIndex.Size

	// Section 4: KeyData
	hdr.KeyData.Offset = curOffset
	hdr.KeyData.Size = uint32(keyDataBuf.Len())
	bufOut.Write(keyDataBuf.Bytes())
	curOffset += hdr.KeyData.Size

	// Section 5: PostingData
	hdr.PostingData.Offset = curOffset
	hdr.PostingData.Count = totalPostings
	hdr.PostingData.Size = uint32(postingDataBuf.Len())
	bufOut.Write(postingDataBuf.Bytes())
	curOffset += hdr.PostingData.Size

	// Section 6: EntryIndex
	hdr.EntryIndex.Offset = curOffset
	hdr.EntryIndex.Count = uint32(len(entryOffsets))
	hdr.EntryIndex.Size = uint32(entryIndexBuf.Len())
	bufOut.Write(entryIndexBuf.Bytes())
	curOffset += hdr.EntryIndex.Size

	// Section 7: EntryData
	if err := bufOut.Flush(); err != nil {
		return err
	}
	hdr.EntryData.Offset = curOffset
	hdr.EntryData.Size = currentEntryOffset
	curOffset += currentEntryOffset

	if _, err := entryTempFile.Seek(0, io.SeekStart); err != nil {
		return err
	}
	if _, err := io.Copy(outFile, entryTempFile); err != nil {
		return fmt.Errorf("copy entries to out: %w", err)
	}

	bufOut.Reset(outFile)

	// Section 8: KanjiIndex
	hdr.KanjiIndex.Offset = curOffset
	hdr.KanjiIndex.Count = uint32(len(kanjiList))
	hdr.KanjiIndex.Size = uint32(kanjiIndexBuf.Len())
	bufOut.Write(kanjiIndexBuf.Bytes())
	curOffset += hdr.KanjiIndex.Size

	// Section 9: KanjiData
	hdr.KanjiData.Offset = curOffset
	hdr.KanjiData.Size = uint32(kanjiDataBuf.Len())
	bufOut.Write(kanjiDataBuf.Bytes())
	curOffset += hdr.KanjiData.Size

	// Section 10: FuriganaIndex
	hdr.FuriganaIndex.Offset = curOffset
	hdr.FuriganaIndex.Count = numFurigana
	hdr.FuriganaIndex.Size = uint32(furiganaIndexBuf.Len())
	bufOut.Write(furiganaIndexBuf.Bytes())
	curOffset += hdr.FuriganaIndex.Size

	// Section 11: FuriganaKeyData
	hdr.FuriganaKeyData.Offset = curOffset
	hdr.FuriganaKeyData.Size = uint32(furiganaKeyBuf.Len())
	bufOut.Write(furiganaKeyBuf.Bytes())
	curOffset += hdr.FuriganaKeyData.Size

	// Section 12: FuriganaData
	hdr.FuriganaData.Offset = curOffset
	hdr.FuriganaData.Size = uint32(furiganaDataBuf.Len())
	bufOut.Write(furiganaDataBuf.Bytes())
	curOffset += hdr.FuriganaData.Size

	// Section 13: KradIndex
	hdr.KradIndex.Offset = curOffset
	hdr.KradIndex.Count = numKrad
	hdr.KradIndex.Size = uint32(kradIndexBuf.Len())
	bufOut.Write(kradIndexBuf.Bytes())
	curOffset += hdr.KradIndex.Size

	// Section 14: KradData
	hdr.KradData.Offset = curOffset
	hdr.KradData.Count = 0
	hdr.KradData.Size = uint32(kradDataBuf.Len())
	bufOut.Write(kradDataBuf.Bytes())
	curOffset += hdr.KradData.Size

	// Section 15: RadkIndex
	hdr.RadkIndex.Offset = curOffset
	hdr.RadkIndex.Count = numRadk
	hdr.RadkIndex.Size = uint32(radkIndexBuf.Len())
	bufOut.Write(radkIndexBuf.Bytes())
	curOffset += hdr.RadkIndex.Size

	// Section 16: RadkData
	hdr.RadkData.Offset = curOffset
	hdr.RadkData.Count = 0
	hdr.RadkData.Size = uint32(radkDataBuf.Len())
	bufOut.Write(radkDataBuf.Bytes())
	curOffset += hdr.RadkData.Size

	// Section 17: KanjiWordsIndex
	hdr.KanjiWordsIndex.Offset = curOffset
	hdr.KanjiWordsIndex.Count = numKanjiWords
	hdr.KanjiWordsIndex.Size = uint32(kanjiWordsIndexBuf.Len())
	bufOut.Write(kanjiWordsIndexBuf.Bytes())
	curOffset += hdr.KanjiWordsIndex.Size

	// Section 18: KanjiWordsData
	hdr.KanjiWordsData.Offset = curOffset
	hdr.KanjiWordsData.Count = 0
	hdr.KanjiWordsData.Size = uint32(kanjiWordsDataBuf.Len())
	bufOut.Write(kanjiWordsDataBuf.Bytes())
	curOffset += hdr.KanjiWordsData.Size

	if err := bufOut.Flush(); err != nil {
		return err
	}

	if _, err := outFile.Seek(0, io.SeekStart); err != nil {
		return err
	}
	headerBytes := hdr.Encode()
	if _, err := outFile.Write(headerBytes); err != nil {
		return err
	}

	if err := outFile.Sync(); err != nil {
		return err
	}
	if err := outFile.Close(); err != nil {
		return err
	}

	return os.Rename(tmpOut, out)
}

func parseJLPTDecks(readers map[string]io.Reader) (map[string]string, error) {
	jlptMap := make(map[string]string)
	// Process levels in order from hardest to easiest: N1, N2, N3, N4, N5
	// so easier levels naturally overwrite if present
	levels := []string{"N1", "N2", "N3", "N4", "N5"}
	for _, lvl := range levels {
		r, ok := readers[lvl]
		if !ok {
			r = readers[strings.ToLower(lvl)]
		}
		if r == nil {
			continue
		}
		cr := csv.NewReader(r)
		cr.FieldsPerRecord = -1
		rows, err := cr.ReadAll()
		if err != nil {
			return nil, fmt.Errorf("read jlpt %s csv: %w", lvl, err)
		}
		for i, row := range rows {
			if i == 0 || len(row) < 2 {
				continue
			}
			exprRaw := strings.TrimSpace(row[0])
			readingRaw := strings.TrimSpace(row[1])

			subExprs := strings.Split(exprRaw, ";")
			subReadings := strings.Split(readingRaw, ";")

			for _, e := range subExprs {
				e = strings.TrimSpace(e)
				if e == "" {
					continue
				}
				for _, rd := range subReadings {
					rd = strings.TrimSpace(rd)
					if rd == "" {
						continue
					}
					key := e + "\x00" + jp.ToHiragana(rd)
					oldLvl := jlptMap[key]
					if oldLvl == "" || jlptRank(lvl) > jlptRank(oldLvl) {
						jlptMap[key] = lvl
					}
				}
			}
		}
	}
	return jlptMap, nil
}

type rawNameTranslation struct {
	Type        []string `json:"type"`
	Translation []struct {
		Lang string `json:"lang"`
		Text string `json:"text"`
	} `json:"translation"`
}

type rawNameWord struct {
	ID          string               `json:"id"`
	Kanji       []rawKanjiForm       `json:"kanji"`
	Kana        []rawKanaForm        `json:"kana"`
	Translation []rawNameTranslation `json:"translation"`
}

func streamNames(
	r io.Reader,
	startEntryIndex uint32,
	bufWriter *bufio.Writer,
	entryWriter *dict.ByteWriter,
	interner *stringInterner,
	entryOffsets *[]uint32,
	postings *[]postingItem,
	currentOffset *uint32,
) (uint32, error) {
	br := bufio.NewReader(r)
	if head, err := br.Peek(3); err == nil && len(head) == 3 && head[0] == 0xef && head[1] == 0xbb && head[2] == 0xbf {
		br.Discard(3)
	}

	dec := json.NewDecoder(br)
	tok, err := dec.Token()
	if err != nil {
		return startEntryIndex, err
	}
	if delim, ok := tok.(json.Delim); !ok || delim != '{' {
		return startEntryIndex, errors.New("expected { at root of jmnedict")
	}

	curIdx := startEntryIndex

	for dec.More() {
		tok, err := dec.Token()
		if err != nil {
			return curIdx, err
		}
		key, ok := tok.(string)
		if !ok {
			continue
		}
		if key == "words" {
			arrTok, err := dec.Token()
			if err != nil {
				return curIdx, err
			}
			if delim, ok := arrTok.(json.Delim); !ok || delim != '[' {
				return curIdx, errors.New("expected [ for words in jmnedict")
			}

			for dec.More() {
				var raw rawNameWord
				if err := dec.Decode(&raw); err != nil {
					return curIdx, err
				}

				idNum, _ := strconv.ParseUint(raw.ID, 10, 32)
				e := dict.Entry{
					ID: uint32(idNum) + 100000000,
				}

				if len(raw.Kanji) > 0 {
					e.Kanji = make([]dict.Form, len(raw.Kanji))
					for i, k := range raw.Kanji {
						e.Kanji[i].Text = k.Text
					}
				}

				if len(raw.Kana) > 0 {
					e.Kana = make([]dict.Form, len(raw.Kana))
					for i, r := range raw.Kana {
						e.Kana[i].Text = r.Text
					}
				}

				if len(raw.Translation) > 0 {
					e.Senses = make([]dict.Sense, len(raw.Translation))
					for i, t := range raw.Translation {
						if len(t.Type) > 0 {
							e.Senses[i].Misc = make([]string, len(t.Type))
							for m, mt := range t.Type {
								interner.Intern(mt)
								e.Senses[i].Misc[m] = mt
							}
						}
						if len(t.Translation) > 0 {
							e.Senses[i].Gloss = make([]string, len(t.Translation))
							for g, tr := range t.Translation {
								e.Senses[i].Gloss[g] = tr.Text
							}
						}
					}
				}

				entryWriter.Reset()
				dict.EncodeEntry(entryWriter, &e, interner.m)
				n, err := bufWriter.Write(entryWriter.Buf)
				if err != nil {
					return curIdx, err
				}
				*entryOffsets = append(*entryOffsets, *currentOffset)
				*currentOffset += uint32(n)

				// Keys for this name, ranked after JMdict entries (priScore=60000)
				nameKeys := make(map[string]bool)
				for _, k := range raw.Kanji {
					nameKeys[k.Text] = true
				}
				for _, r := range raw.Kana {
					norm := jp.ToHiragana(r.Text)
					nameKeys[norm] = true
				}

				for k := range nameKeys {
					*postings = append(*postings, postingItem{
						key:        k,
						priScore:   60000, // strictly ranked after all JMdict entries
						isCommon:   false,
						isRegular:  false,
						entryIndex: curIdx,
					})
				}

				curIdx++
			}

			if _, err := dec.Token(); err != nil {
				return curIdx, err
			}
		} else {
			var discard any
			dec.Decode(&discard)
		}
	}

	return curIdx, nil
}

func calcPriScore(formPris, entryPris []string) uint16 {
	if len(formPris) == 0 && len(entryPris) == 0 {
		return 1000
	}
	pris := append(append([]string(nil), formPris...), entryPris...)

	var bestNF int = -1
	for _, p := range pris {
		if strings.HasPrefix(p, "nf") && len(p) == 4 {
			if n, err := strconv.Atoi(p[2:]); err == nil {
				if bestNF == -1 || n < bestNF {
					bestNF = n
				}
			}
		}
	}

	var base int
	if bestNF != -1 {
		base = bestNF * 10
	} else {
		hasStrong := false
		hasWeak := false
		for _, p := range pris {
			switch p {
			case "news1", "ichi1", "spec1", "gai1":
				hasStrong = true
			case "news2", "ichi2", "spec2", "gai2":
				hasWeak = true
			}
		}
		if hasStrong {
			base = 250
		} else if hasWeak {
			base = 450
		} else {
			base = 600
		}
	}

	seen := make(map[string]bool)
	for _, p := range pris {
		if seen[p] {
			continue
		}
		seen[p] = true
		switch p {
		case "news1", "ichi1", "spec1", "gai1":
			base -= 5
		case "news2", "ichi2", "spec2", "gai2":
			base -= 2
		}
	}

	if base < 0 {
		base = 0
	}
	if base > 1000 {
		base = 1000
	}
	return uint16(base)
}

func parseJMdictXML(r io.Reader) (map[entFormKey][]string, map[uint32][]string, error) {
	br := bufio.NewReader(r)
	magic, err := br.Peek(2)
	var reader io.Reader = br
	if err == nil && len(magic) == 2 && magic[0] == 0x1f && magic[1] == 0x8b {
		gz, err := gzip.NewReader(br)
		if err != nil {
			return nil, nil, err
		}
		defer gz.Close()
		reader = gz
	}

	formPriorities := make(map[entFormKey][]string)
	entryPriorities := make(map[uint32][]string)

	scanner := bufio.NewScanner(reader)
	buf := make([]byte, 64*1024)
	scanner.Buffer(buf, 1024*1024)

	var curEntSeq uint32
	var curForm string
	var curFormPris []string
	var entryPris []string

	flushForm := func() {
		if curEntSeq > 0 && curForm != "" && len(curFormPris) > 0 {
			k := entFormKey{id: curEntSeq, form: curForm}
			formPriorities[k] = append([]string(nil), curFormPris...)
		}
		curForm = ""
		curFormPris = nil
	}

	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if strings.HasPrefix(line, "<ent_seq>") && strings.HasSuffix(line, "</ent_seq>") {
			flushForm()
			if curEntSeq > 0 && len(entryPris) > 0 {
				entryPriorities[curEntSeq] = append([]string(nil), entryPris...)
			}
			val := line[9 : len(line)-10]
			if id, err := strconv.ParseUint(val, 10, 32); err == nil {
				curEntSeq = uint32(id)
			}
			entryPris = nil
		} else if strings.HasPrefix(line, "<keb>") && strings.HasSuffix(line, "</keb>") {
			flushForm()
			curForm = line[5 : len(line)-6]
		} else if strings.HasPrefix(line, "<reb>") && strings.HasSuffix(line, "</reb>") {
			flushForm()
			curForm = line[5 : len(line)-6]
		} else if strings.HasPrefix(line, "<ke_pri>") && strings.HasSuffix(line, "</ke_pri>") {
			p := line[8 : len(line)-9]
			curFormPris = append(curFormPris, p)
			entryPris = append(entryPris, p)
		} else if strings.HasPrefix(line, "<re_pri>") && strings.HasSuffix(line, "</re_pri>") {
			p := line[8 : len(line)-9]
			curFormPris = append(curFormPris, p)
			entryPris = append(entryPris, p)
		} else if strings.HasPrefix(line, "</entry>") {
			flushForm()
			if curEntSeq > 0 && len(entryPris) > 0 {
				entryPriorities[curEntSeq] = append([]string(nil), entryPris...)
			}
			curEntSeq = 0
			entryPris = nil
		}
	}
	return formPriorities, entryPriorities, scanner.Err()
}

type furiParsed struct {
	key    string
	rubies []dict.Ruby
}

func parseFurigana(r io.Reader, validPairs map[string]bool) ([]furiParsed, error) {
	br := bufio.NewReader(r)
	if head, err := br.Peek(3); err == nil && len(head) == 3 && head[0] == 0xef && head[1] == 0xbb && head[2] == 0xbf {
		br.Discard(3)
	}
	dec := json.NewDecoder(br)
	tok, err := dec.Token()
	if err != nil {
		return nil, err
	}
	if delim, ok := tok.(json.Delim); !ok || delim != '[' {
		return nil, errors.New("expected [ at root of furigana json")
	}

	type rawRuby struct {
		Ruby string `json:"ruby"`
		Rt   string `json:"rt"`
	}
	type rawFuriEntry struct {
		Text     string    `json:"text"`
		Reading  string    `json:"reading"`
		Furigana []rawRuby `json:"furigana"`
	}

	var results []furiParsed

	for dec.More() {
		var raw rawFuriEntry
		if err := dec.Decode(&raw); err != nil {
			return nil, err
		}
		normReading := jp.ToHiragana(raw.Reading)
		key := raw.Text + "\x00" + normReading
		if validPairs != nil && !validPairs[key] {
			continue
		}

		rubies := make([]dict.Ruby, len(raw.Furigana))
		for i, fr := range raw.Furigana {
			rubies[i] = dict.Ruby{
				Text: fr.Ruby,
				Ruby: fr.Rt,
			}
		}
		results = append(results, furiParsed{
			key:    key,
			rubies: rubies,
		})
	}
	return results, nil
}

type rawKradfile struct {
	Version string              `json:"version"`
	Kanji   map[string][]string `json:"kanji"`
}

type rawRadkRadical struct {
	StrokeCount int      `json:"strokeCount"`
	Kanji       []string `json:"kanji"`
}

type rawRadkfile struct {
	Version  string                    `json:"version"`
	Radicals map[string]rawRadkRadical `json:"radicals"`
}

func parseKradfile(r io.Reader) (map[rune][]rune, error) {
	var raw rawKradfile
	if err := json.NewDecoder(r).Decode(&raw); err != nil {
		return nil, err
	}
	out := make(map[rune][]rune, len(raw.Kanji))
	for kStr, cStrs := range raw.Kanji {
		kRune, _ := utf8.DecodeRuneInString(kStr)
		if kRune == utf8.RuneError {
			continue
		}
		comps := make([]rune, 0, len(cStrs))
		for _, cStr := range cStrs {
			cRune, _ := utf8.DecodeRuneInString(cStr)
			if cRune != utf8.RuneError {
				comps = append(comps, dict.DisplayComponent(cRune))
			}
		}
		out[kRune] = comps
	}
	return out, nil
}

func parseRadkfile(r io.Reader) (map[rune][]rune, error) {
	var raw rawRadkfile
	if err := json.NewDecoder(r).Decode(&raw); err != nil {
		return nil, err
	}
	out := make(map[rune][]rune, len(raw.Radicals)*2)
	for rStr, radData := range raw.Radicals {
		rRune, _ := utf8.DecodeRuneInString(rStr)
		if rRune == utf8.RuneError {
			continue
		}
		kList := make([]rune, 0, len(radData.Kanji))
		for _, kStr := range radData.Kanji {
			kRune, _ := utf8.DecodeRuneInString(kStr)
			if kRune != utf8.RuneError {
				kList = append(kList, kRune)
			}
		}
		out[rRune] = kList
		disp := dict.DisplayComponent(rRune)
		if disp != rRune {
			out[disp] = append([]rune(nil), kList...)
		}
	}
	return out, nil
}
func parseAccents(r io.Reader) (map[string][]int, error) {
	accents := make(map[string][]int)
	scanner := bufio.NewScanner(r)
	buf := make([]byte, 64*1024)
	scanner.Buffer(buf, 1024*1024)

	reParen := regexp.MustCompile(`\([^)]*\)`)

	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if line == "" {
			continue
		}
		parts := strings.Split(line, "\t")
		if len(parts) < 3 {
			continue
		}
		word := parts[0]
		reading := parts[1]
		downstepsRaw := parts[2]

		var steps []int
		for _, item := range strings.Split(downstepsRaw, ",") {
			cleaned := strings.TrimSpace(reParen.ReplaceAllString(item, ""))
			if cleaned != "" {
				if n, err := strconv.Atoi(cleaned); err == nil {
					found := false
					for _, s := range steps {
						if s == n {
							found = true
							break
						}
					}
					if !found {
						steps = append(steps, n)
					}
				}
			}
		}

		if len(steps) == 0 {
			continue
		}

		key := pitchKey(word, reading)
		accents[key] = mergeIntSlices(accents[key], steps)

		if reading == "" {
			accents[pitchKey(word, word)] = mergeIntSlices(accents[pitchKey(word, word)], steps)
		}

		wordNorm := jp.ToHiragana(word)
		readingNorm := jp.ToHiragana(reading)
		if wordNorm != word || readingNorm != reading {
			accents[pitchKey(wordNorm, readingNorm)] = mergeIntSlices(accents[pitchKey(wordNorm, readingNorm)], steps)
		}
	}
	return accents, scanner.Err()
}

func pitchKey(word, reading string) string {
	return word + "\x00" + reading
}

func mergeIntSlices(a, b []int) []int {
	res := append([]int(nil), a...)
	for _, v := range b {
		found := false
		for _, cur := range res {
			if cur == v {
				found = true
				break
			}
		}
		if !found {
			res = append(res, v)
		}
	}
	return res
}

func attachPitch(e *dict.Entry, accentsMap map[string][]int) {
	type readingPitch struct {
		reading   string
		downsteps []int
	}
	var pitches []readingPitch

	addPitch := func(reading string, steps []int) {
		for i := range pitches {
			if pitches[i].reading == reading {
				pitches[i].downsteps = mergeIntSlices(pitches[i].downsteps, steps)
				return
			}
		}
		pitches = append(pitches, readingPitch{reading: reading, downsteps: steps})
	}

	if len(e.Kanji) > 0 {
		for _, r := range e.Kana {
			reading := r.Text
			readingNorm := jp.ToHiragana(reading)
			for _, k := range e.Kanji {
				word := k.Text
				if steps, ok := accentsMap[pitchKey(word, reading)]; ok {
					addPitch(reading, steps)
				} else if steps, ok := accentsMap[pitchKey(word, readingNorm)]; ok {
					addPitch(reading, steps)
				}
			}
		}
	} else {
		for _, r := range e.Kana {
			reading := r.Text
			readingNorm := jp.ToHiragana(reading)
			if steps, ok := accentsMap[pitchKey(reading, reading)]; ok {
				addPitch(reading, steps)
			} else if steps, ok := accentsMap[pitchKey(readingNorm, readingNorm)]; ok {
				addPitch(reading, steps)
			}
		}
	}

	if len(pitches) > 0 {
		e.Pitch = make([]dict.Pitch, len(pitches))
		for i, p := range pitches {
			e.Pitch[i] = dict.Pitch{
				Reading:   p.reading,
				Downsteps: p.downsteps,
			}
		}
	}
}

type rawKanjidicChar struct {
	Literal string `json:"literal"`
	Misc    struct {
		Grade        *int  `json:"grade"`
		StrokeCounts []int `json:"strokeCounts"`
		Frequency    *int  `json:"frequency"`
		JLPTLevel    *int  `json:"jlptLevel"`
	} `json:"misc"`
	ReadingMeaning struct {
		Groups []struct {
			Readings []struct {
				Type  string `json:"type"`
				Value string `json:"value"`
			} `json:"readings"`
			Meanings []struct {
				Lang  string `json:"lang"`
				Value string `json:"value"`
			} `json:"meanings"`
		} `json:"groups"`
		Nanori []string `json:"nanori"`
	} `json:"readingMeaning"`
}

func parseKanjidic(r io.Reader) ([]dict.Kanji, error) {
	dec := json.NewDecoder(r)
	tok, err := dec.Token()
	if err != nil {
		return nil, err
	}
	if delim, ok := tok.(json.Delim); !ok || delim != '{' {
		return nil, errors.New("expected { at root of kanjidic")
	}

	var kanjiList []dict.Kanji

	for dec.More() {
		tok, err := dec.Token()
		if err != nil {
			return nil, err
		}
		key, ok := tok.(string)
		if !ok {
			continue
		}
		if key == "characters" {
			arrTok, err := dec.Token()
			if err != nil {
				return nil, err
			}
			if delim, ok := arrTok.(json.Delim); !ok || delim != '[' {
				return nil, errors.New("expected [ for characters")
			}
			for dec.More() {
				var c rawKanjidicChar
				if err := dec.Decode(&c); err != nil {
					return nil, err
				}
				k := dict.Kanji{
					Literal: c.Literal,
				}
				if len(c.Misc.StrokeCounts) > 0 {
					k.Strokes = c.Misc.StrokeCounts[0]
				}
				if c.Misc.Grade != nil {
					k.Grade = *c.Misc.Grade
				}
				if c.Misc.Frequency != nil {
					k.Freq = *c.Misc.Frequency
				}
				if c.Misc.JLPTLevel != nil {
					k.JLPT = *c.Misc.JLPTLevel
				}

				for _, g := range c.ReadingMeaning.Groups {
					for _, rd := range g.Readings {
						switch rd.Type {
						case "ja_on":
							k.On = append(k.On, rd.Value)
						case "ja_kun":
							k.Kun = append(k.Kun, rd.Value)
						}
					}
					for _, m := range g.Meanings {
						if m.Lang == "en" {
							k.Meanings = append(k.Meanings, m.Value)
						}
					}
				}
				k.Nanori = c.ReadingMeaning.Nanori
				kanjiList = append(kanjiList, k)
			}
			dec.Token()
		} else {
			var discard any
			dec.Decode(&discard)
		}
	}
	return kanjiList, nil
}