merge compound particles, rank kana lookups, and wake chromium ax
Signed-off-by: dawn <90008@klbr.net>
1 unmodified line234567891029 unmodified lines40414243444546474849505112 unmodified lines64656667686970717273747549 unmodified lines12512612712812913013113213378 unmodified lines21221321421520720821621721821922022122222322422521622622722822923022023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327420 unmodified lines2952962972982993003013023033043053063072663083093102693113123133143153163173181 unmodified line1 unmodified line// and finds the dictionary entries for each word.package analyze
import ( "slices" "strings" "unicode" "unicode/utf8"
29 unmodified lines Key string // dictionary form of the head morpheme KeyReading string // its hiragana reading, "" when unknown Content bool // a word worth looking up first (not a particle or symbol) Symbol bool Parts []Word // constituent particles when merged from a compound particle
pos string // head morpheme's major POS (t.POS[0])
// katakana reading and pronunciation not yet turned into Romaji; romaji // is made per run rather than per morpheme so a trailing っ doubles the // next morpheme's consonant (なかっ+た -> nakatta) kana, pron string12 unmodified lines var cur *Word var prev *morph.Token for i := range toks { t := &toks[i] if strings.TrimSpace(t.Surface) == "" { cur, prev = nil, nil continue } switch { case cur == nil || !attaches(t, prev, cur): words = append(words, Word{Start: t.Start}) cur = &words[len(words)-1]49 unmodified lines}
// head sets the word's dictionary key from its first morpheme.func (a *Analyzer) head(w *Word, t *morph.Token) { w.pos = t.POS[0] w.Symbol = t.POS[0] == "記号" || (!t.Known && !jp.HasJapanese(t.Surface)) w.Content = !w.Symbol && t.POS[0] != "助詞" && t.POS[0] != "助動詞" if w.Symbol { return78 unmodified lines } return strings.TrimSuffix(reading, sTail) + jp.ToHiragana(bTail)}
// mergeCompounds joins runs of nouns that the dictionary knows as one word// (東京+大学 -> 東京大学), so the card shows the compound's entry.// mergeCompounds joins runs of nouns or particles that the dictionary knows// as one word (東京+大学 -> 東京大学, と+は -> とは), so the card shows the// compound's entry.func (a *Analyzer) mergeCompounds(text string, words []Word) []Word { if a.Dict == nil || len(words) < 2 { return words } out := words[:0:0] for i := 0; i < len(words); i++ { w := words[i] if isNounish(w) { switch { case isNounish(w): if n, es := a.Dict.LongestPrefix(text[w.Start:], 16); len(es) > 0 { end := w.Start + n j := i + 1 for j < len(words) && words[j].End <= end && isNounish(words[j]) { for j < len(words) && words[j].Start == words[j-1].End && words[j].End <= end && isNounish(words[j]) { j++ } if j-i > 1 && words[j-1].End == end { w = a.join(words[i:j]) i = j - 1 } } case w.pos == "助詞": j := i + 1 for j < len(words) && j-i < 3 && words[j].Start == words[j-1].End && words[j].pos == "助詞" { j++ } for k := j; k > i+1; k-- { if a.isCompoundParticle(text[w.Start:words[k-1].End]) { w = a.join(words[i:k]) w.Parts = slices.Clone(words[i:k]) i = k - 1 break } } } out = append(out, w) } return out}
func (a *Analyzer) isCompoundParticle(s string) bool { for _, e := range a.Dict.Lookup(s) { if !dict.WrittenInKana(e) || len(e.Senses) == 0 { continue } pos := e.Senses[0].POS if slices.Contains(pos, "prt") || slices.Contains(pos, "exp") || slices.Contains(pos, "conj") { return true } } return false}
// isNounish reports whether every rune is kanji or katakana, the shape of// noun compounds; good enough without carrying POS along.func isNounish(w Word) bool { if w.Symbol || !w.Content {20 unmodified lines
// join merges consecutive words into one looked up as a whole.func (a *Analyzer) join(words []Word) Word { m := words[0] m.Segs = slices.Clone(m.Segs) sep := " " if m.pos == "助詞" { sep = "" } for _, x := range words[1:] { m.End = x.End m.Surface += x.Surface m.Reading += x.Reading m.Romaji += " " + x.Romaji m.Romaji += sep + x.Romaji m.Segs = appendSegs(m.Segs, x.Segs) } m.Key, m.KeyReading, m.Content = m.Surface, m.Reading, true m.Key, m.KeyReading = m.Surface, m.Reading if m.pos != "助詞" { m.Content = true } if rs, ok := a.Dict.Furigana(m.Surface, m.Reading); ok { m.Segs = segs(rs) } return m1 unmodified line123456789101110111213141516171819202122232417252627282129303132253334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711861 unmodified linespackage analyze
import ( "slices" "strings" "unicode/utf8"
"chirami/internal/dict" "chirami/internal/jp")
// Entries finds dictionary entries for w, best first: the exact surface,// then the dictionary form of its head, then the longest known prefix.// Entries finds dictionary entries for w, best first: the exact surface (or// dictionary form first for inflected verbs/adjectives), then the dictionary// form of its head, then the longest known prefix.func (a *Analyzer) Entries(w Word) []dict.Entry { if a.Dict == nil || w.Symbol { return nil } if w.Key != "" && w.Key != w.Surface && (w.pos == "動詞" || w.pos == "形容詞") { if es := a.Dict.LookupReading(w.Key, w.KeyReading); len(es) > 0 { return rankByPOS(es, w) } } if es := a.Dict.LookupReading(w.Surface, w.Reading); len(es) > 0 { return es return rankByPOS(es, w) } if w.Key != "" && w.Key != w.Surface { if es := a.Dict.LookupReading(w.Key, w.KeyReading); len(es) > 0 { return es return rankByPOS(es, w) } } if _, es := a.Dict.LongestPrefix(w.Surface, 12); len(es) > 0 { return es return rankByPOS(es, w) } return nil}
// rankByPOS prefers entries whose JMdict part of speech matches the// tokenizer's POS for kana words (e.g. かも as particle vs 鴨 as noun).func rankByPOS(es []dict.Entry, w Word) []dict.Entry { if w.pos == "" || len(es) <= 1 || !jp.IsKanaOnly(w.Surface) { return es } slices.SortStableFunc(es, func(a, b dict.Entry) int { return posRank(b, w) - posRank(a, w) }) return es}
func posRank(e dict.Entry, w Word) int { if !dict.WrittenInKana(e) { return 0 } common := hasCommonKana(e, w.Reading) if !common && w.Content { return 0 } s := 0 if common { s += 2 } if matchesPOS(e, w.pos) { s += 4 } return s}
func hasCommonKana(e dict.Entry, reading string) bool { for _, r := range e.Kana { if r.Common && jp.ToHiragana(r.Text) == reading { return true } } return false}
func matchesPOS(e dict.Entry, pos string) bool { for _, s := range e.Senses { for _, p := range s.POS { switch pos { case "助詞": if p == "prt" || p == "exp" || p == "conj" { return true } case "助動詞": if p == "cop" || p == "aux" || p == "aux-v" || p == "aux-adj" { return true } case "接続詞": if p == "conj" || p == "prt" || p == "exp" { return true } case "副詞": if p == "adv" || p == "adv-to" { return true } case "動詞": if strings.HasPrefix(p, "v") { return true } case "形容詞": if p == "adj-i" || p == "adj-ix" { return true } case "名詞": if p == "n" || p == "pn" || p == "adj-na" || p == "adj-no" || p == "num" || p == "ctr" { return true } } } } return false}
// Kanji returns KANJIDIC entries for the distinct kanji in s, at most max.func (a *Analyzer) Kanji(s string, max int) []dict.Kanji { if a.Dict == nil { return nil61 unmodified lines1 unmodified line2345678910125 unmodified linesExpand all1361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122138 unmodified lines1 unmodified line
import ( "os" "path/filepath" "slices" "strings" "testing"
"chirami/internal/dict"125 unmodified linesExpand all t.Fatal("not a phrase the dictionary knows") }}
func TestCompoundParticlesAndLookup(t *testing.T) { d := testDict{entries: map[string][]dict.Entry{ "とは": {{ Kana: []dict.Form{{Text: "とは", Common: true}}, Senses: []dict.Sense{{POS: []string{"exp", "prt"}, Gloss: []string{"indicates word or phrase being defined"}}}, }}, "と": { {Kanji: []dict.Form{{Text: "十", Common: true}}, Kana: []dict.Form{{Text: "じゅう", Common: true}, {Text: "と"}}, Senses: []dict.Sense{{POS: []string{"num"}, Gloss: []string{"ten"}}}}, {Kana: []dict.Form{{Text: "と", Common: true}}, Senses: []dict.Sense{{POS: []string{"prt"}, Gloss: []string{"if; when"}}}}, }, "は": { {Kanji: []dict.Form{{Text: "葉", Common: true}}, Kana: []dict.Form{{Text: "は", Common: true}}, Senses: []dict.Sense{{POS: []string{"n"}, Gloss: []string{"leaf"}}}}, {Kana: []dict.Form{{Text: "は", Common: true}}, Senses: []dict.Sense{{POS: []string{"prt"}, Gloss: []string{"indicates sentence topic"}}}}, }, "した": { {Kanji: []dict.Form{{Text: "舌", Common: true}}, Kana: []dict.Form{{Text: "した", Common: true}}, Senses: []dict.Sense{{POS: []string{"n"}, Gloss: []string{"tongue"}}}}, }, "する": { {Kanji: []dict.Form{{Text: "為る", Common: true}}, Kana: []dict.Form{{Text: "する", Common: true}}, Senses: []dict.Sense{{POS: []string{"vs-i"}, Misc: []string{"uk"}, Gloss: []string{"to do"}}}}, }, }} a := &Analyzer{Morph: tokenizer, Dict: d} ws := a.Words("ミックスリストとは、YouTube が作成した") var surfaces, romaji []string for _, w := range ws { surfaces = append(surfaces, w.Surface) romaji = append(romaji, w.Romaji) } if got, want := strings.Join(surfaces, "|"), "ミックス|リスト|とは|、|YouTube|が|作成|した"; got != want { t.Fatalf("words = %q, want %q", got, want) } if got, want := strings.Join(romaji, "|"), "mikkusu|risuto|towa|,|YouTube|ga|sakusei|shita"; got != want { t.Fatalf("romaji = %q, want %q", got, want) } towa := ws[2] if len(towa.Parts) != 2 || towa.Parts[0].Surface != "と" || towa.Parts[1].Surface != "は" { t.Fatalf("とは parts = %+v, want [と は]", towa.Parts) } if es := a.Entries(towa.Parts[1]); len(es) == 0 || es[0].Senses[0].Gloss[0] != "indicates sentence topic" { t.Fatalf("は entries = %+v, want topic particle first", es) } shita := ws[len(ws)-1] if es := a.Entries(shita); len(es) == 0 || es[0].Senses[0].Gloss[0] != "to do" { t.Fatalf("した entries = %+v, want する (to do) rather than 舌 (tongue)", es) }}
type testDict struct { entries map[string][]dict.Entry}
func (d testDict) Lookup(w string) []dict.Entry { return slices.Clone(d.entries[w]) }func (d testDict) LookupReading(w, r string) []dict.Entry { es := d.Lookup(w) slices.SortStableFunc(es, func(a, b dict.Entry) int { var ak, bk int if dict.WrittenInKana(a) { ak = 1 } if dict.WrittenInKana(b) { bk = 1 } return bk - ak }) return es}func (testDict) LongestPrefix(s string, n int) (int, []dict.Entry) { return 0, nil }func (testDict) Kanji(r rune) (dict.Kanji, bool) { return dict.Kanji{}, false }func (testDict) Furigana(w, r string) ([]dict.Ruby, bool) { return nil, false }
type phraseDict struct{ known string }
func (d phraseDict) Lookup(w string) []dict.Entry { if w == d.known {8 unmodified lines427 unmodified linesExpand all428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478427 unmodified linesExpand all if time.Since(start) > time.Second { t.Fatal("show mode waited for its timeout on text with no Japanese") }}
type particleDict struct{}
func (particleDict) Lookup(w string) []dict.Entry { switch w { case "とは": return []dict.Entry{{Kana: []dict.Form{{Text: "とは", Common: true}}, Senses: []dict.Sense{{POS: []string{"exp", "prt"}, Gloss: []string{"indicates word or phrase being defined"}}}}} case "と": return []dict.Entry{{Kana: []dict.Form{{Text: "と", Common: true}}, Senses: []dict.Sense{{POS: []string{"prt"}, Gloss: []string{"if; when"}}}}} case "は": return []dict.Entry{{Kana: []dict.Form{{Text: "は", Common: true}}, Senses: []dict.Sense{{POS: []string{"prt"}, Gloss: []string{"indicates sentence topic"}}}}} } return nil}func (d particleDict) LookupReading(w, r string) []dict.Entry { return d.Lookup(w) }func (particleDict) LongestPrefix(s string, n int) (int, []dict.Entry) { return 0, nil }func (particleDict) Kanji(r rune) (dict.Kanji, bool) { return dict.Kanji{}, false }func (particleDict) Furigana(w, r string) ([]dict.Ruby, bool) { return nil, false }
func TestCompoundParticlePartsAndLatinRomaji(t *testing.T) { fonts, err := card.LoadFonts() if err != nil { t.Skip(err) } a := New(DefaultConfig, Deps{ Analyzer: &analyze.Analyzer{Morph: tokenizer(t), Dict: particleDict{}}, Renderer: &card.Renderer{Fonts: fonts, Theme: card.Night}, }) v := a.newView("ミックスリストとは、YouTube が作成した") v.expanded = true v.focus = 2 // とは a.lookup(v) wv := a.wordModel(v) if wv.Words[3].Romaji != "" { t.Fatalf("punctuation romaji = %q, want empty", wv.Words[3].Romaji) } if wv.Words[4].Romaji != "YouTube" || wv.Words[4].Lookup { t.Fatalf("YouTube word = %+v, want Romaji=YouTube Lookup=false", wv.Words[4]) } parts := wv.Words[2].Parts if len(parts) != 2 || parts[0].Gloss != "if" || parts[1].Gloss != "topic" { t.Fatalf("とは parts = %+v, want [if, topic]", parts) } if len(wv.Entries) == 0 || wv.Entries[0].Head != "とは" || wv.Entries[0].Same { t.Fatalf("とは entry = %+v, want Head=とは Same=false", wv.Entries) }}2 unmodified lines34567891011142 unmodified linesExpand all15415515615715715815916016116232 unmodified lines19519619719819819920019920020120220320420520620720820921021121221321421521621721821922022122222322422522622747 unmodified lines27527627727827928028128228328428528628728828929029129229329425729529629729829930030130230330430530630730830931031131231331419 unmodified lines334335336337290338339340341342133 unmodified linesExpand all2 unmodified linesimport ( "fmt" "slices" "strings" "unicode"
"chirami/internal/analyze" "chirami/internal/card" "chirami/internal/dict"142 unmodified linesExpand all wv := &card.WordView{Focus: v.focus, Expanded: v.expanded, Status: v.status, Phrase: v.phrase != nil, WholePhrase: v.focus == wholePhrase} for i, w := range v.words { cw := card.Word{Segs: segs(w.Segs), Romaji: w.Romaji, Lookup: !w.Symbol} if w.Symbol || a.cfg.NoRomaji { if a.cfg.NoRomaji || (w.Symbol && !hasAlnum(w.Romaji)) { cw.Romaji = "" // punctuation needs no romaji under it } if v.expanded && i == v.focus { cw.Parts = a.parts(w, v.entries)32 unmodified lines } return a.dict.Kanji(r)}
// parts splits an inflected word for display: the stem glossed with the// word's meaning, each inflection with a short name and Yomitan's// description on hover.func hasAlnum(s string) bool { return strings.ContainsFunc(s, func(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) })}
// parts splits a compound particle or inflected word for display: each// particle or stem glossed with its meaning, each inflection with a short// name and Yomitan's description on hover.func (a *App) parts(w analyze.Word, entries []dict.Entry) []card.Part { if len(w.Parts) > 0 { var out []card.Part for _, p := range w.Parts { cp := card.Part{Segs: segs(p.Segs)} if !a.cfg.NoRomaji { cp.Romaji = p.Romaji } if es := a.an.Entries(p); len(es) > 0 { cp.Gloss = shortGloss(&es[0]) cp.Tip = a.partTip(p, &es[0]) } out = append(out, cp) } return out } pieces, ok := a.an.Inflection(w) if !ok { return nil }47 unmodified lines } return strings.Join(keep, "\n")}
func (a *App) partTip(w analyze.Word, e *dict.Entry) string { ce := a.entry(e, w) head := w.Surface if ce.Tip != "" { head += " · " + ce.Tip } var lines []string for i, m := range ce.Meanings { lines = append(lines, fmt.Sprintf("%d. %s", i+1, m)) } if len(lines) == 0 { return head } return head + "\n" + strings.Join(lines, "\n")}
// shortGloss is the first meaning cut to a label: "to eat" -> "eat".// shortGloss is the first meaning cut to a label: "to eat" -> "eat",// "indicates sentence topic" -> "topic".func shortGloss(e *dict.Entry) string { if len(e.Senses) == 0 || len(e.Senses[0].Gloss) == 0 { return "" } g := strings.TrimPrefix(e.Senses[0].Gloss[0], "to ") for _, pfx := range []string{"indicates sentence ", "indicates the ", "indicates a ", "indicates "} { if after, ok := strings.CutPrefix(g, pfx); ok { g = after break } } for _, sfx := range []string{" of a sentence", " of action"} { g = strings.TrimSuffix(g, sfx) } if i := strings.IndexAny(g, "(;,"); i > 0 { g = g[:i] } return strings.TrimSpace(g)19 unmodified lines if head == "" { ce.Head, ce.Reading = reading, "" } ce.DictForm = w.Key != "" && w.Key != w.Surface && (ce.Head == w.Key || ce.Reading == w.Key) ce.Same = ce.Head == w.Surface ce.Same = ce.Head == w.Surface && len(w.Parts) == 0 if !a.cfg.NoRomaji { ce.Romaji = a.an.Romaji(cmp(head, reading), reading) } for _, p := range e.Pitch {133 unmodified linesExpand all571 unmodified linesExpand all572573574575576577578579580581582430 unmodified linesExpand all571 unmodified linesExpand all } i = j } y += rows.ruby + rows.text + float64(glossLines)*rows.gloss if glossLines > 0 { y += 4 } } return y}
430 unmodified linesExpand all1234567891011121314295 unmodified linesExpand all31031131231331231431531631731831932032132231932332432532632732832933033133233333433530 unmodified lines366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428372 unmodified linesExpand allpackage dict
import ( "bytes" "cmp" "encoding/binary" "errors" "fmt" "os" "slices" "unicode/utf8"
"chirami/internal/jp" "golang.org/x/sys/unix"295 unmodified linesExpand all return entries}
// LookupReading returns entries matching word, with entries where kana==reading// placed first, followed by the rest of Lookup(word).// placed first, followed by the rest of Lookup(word). When word is kana-only,// entries ordinarily written in kana (particles, kana words, and "uk" entries)// come before kanji words that merely share the reading.func (d *Dict) LookupReading(word, reading string) []Entry { entries := d.Lookup(word) if len(entries) <= 1 { return entries }
readingNorm := jp.ToHiragana(reading) wordNorm := jp.ToHiragana(word) if jp.IsKanaOnly(word) { slices.SortStableFunc(entries, func(a, b Entry) int { return cmp.Compare(kanaRank(b, word, wordNorm), kanaRank(a, word, wordNorm)) }) return entries }
readingNorm := jp.ToHiragana(reading) var matched []Entry var rest []Entry
for _, e := range entries {30 unmodified lines
return append(matched, rest...)}
var ( nonDisplayKanji = []string{"sK", "iK", "rK", "oK"} nonDisplayKana = []string{"sk", "ik", "rk", "ok"})
func hasAnyTag(tags, set []string) bool { for _, t := range tags { if slices.Contains(set, t) { return true } } return false}
// WrittenInKana reports whether e is ordinarily written in kana: it has no// displayable kanji form, or its primary sense is marked "uk" (usually kana).func WrittenInKana(e Entry) bool { if len(e.Senses) > 0 && slices.Contains(e.Senses[0].Misc, "uk") { return true } for _, k := range e.Kanji { if !hasAnyTag(k.Tags, nonDisplayKanji) { return false } } return true}
func kanaRank(e Entry, word, wordNorm string) int { best := 0 inKana := WrittenInKana(e) for _, r := range e.Kana { if r.Text != word && jp.ToHiragana(r.Text) != wordNorm { continue } s := 1 if !hasAnyTag(r.Tags, nonDisplayKana) { s += 16 } if r.Common || (word != wordNorm && r.Text == word && inKana) { s += 8 if inKana { s += 4 } } if r.Text == word { s += 2 } if s > best { best = s } } return best}
// LongestPrefix finds the longest prefix of s (up to maxRunes) that has// dictionary entries. It returns the byte length n of the prefix and the entries.func (d *Dict) LongestPrefix(s string, maxRunes int) (int, []Entry) { if len(s) == 0 || maxRunes <= 0 {372 unmodified linesExpand all282 unmodified linesExpand all283284285286287288289290291292293294295296297298299419 unmodified linesExpand all282 unmodified linesExpand all } if konnichiEntries[0].Kana[0].Text != "こんにち" { t.Errorf("first entry reading = %q, want 'こんにち'", konnichiEntries[0].Kana[0].Text) }
// Kana-only lookup ranks the particle は ahead of kanji homophone 葉 (nf03). haEntries := d.LookupReading("は", "は") if len(haEntries) < 2 { t.Fatalf("expected at least 2 entries for 'は', got %d", len(haEntries)) } if haEntries[0].ID != 2028920 { t.Errorf("first entry for 'は' = ID %d (%v), want 2028920 (particle は)", haEntries[0].ID, haEntries[0].Kanji) }}
func TestLongestPrefix(t *testing.T) { d := buildTestDict(t, false)419 unmodified linesExpand all426 unmodified linesExpand all4274284294304314314324334347 unmodified lines44244344444544744844944644744844982 unmodified lines5325335345355405365375385395405415425435445485455465475485495505515525565575535545555565575585595605615621 unmodified line5645655665675675685695705715725735735745745755765775785795805815825795835845855865835875885895905915925935905945955965975981200 unmodified linesExpand all426 unmodified linesExpand all } } }
hasUK := false if len(raw.Sense) > 0 { e.Senses = make([]dict.Sense, len(raw.Sense)) for i, s := range raw.Sense { if len(s.PartOfSpeech) > 0 {7 unmodified lines e.Senses[i].Misc = make([]string, len(s.Misc)) for m, misc := range s.Misc { interner.Intern(misc) e.Senses[i].Misc[m] = misc if misc == "uk" { hasUK = true } } } if len(s.Field) > 0 { e.Senses[i].Field = make([]string, len(s.Field))82 unmodified lines currentEntryOffset += uint32(n)
// Calculate priorities & post keys entryPrisList := entryPris[e.ID] hasNoKanji := len(raw.Kanji) == 0 inKana := dict.WrittenInKana(e)
type keyMeta struct { priScore uint16 isCommon bool isRegular bool } entryKeys := make(map[string]keyMeta)
addKey := func(k string, formText string, isKanaKey bool, irregular bool) { addKey := func(k string, formText string, isKanaKey, formCommon, irregular bool) { if k == "" { return } var fPris []string if formPris != nil { fPris = formPris[entFormKey{id: e.ID, form: formText}] } score := calcPriScore(fPris, entryPrisList) if isKanaKey && (hasUK || hasNoKanji) { ePris := entryPrisList if isCommon && !formCommon { ePris = nil } score := calcPriScore(fPris, ePris) if isKanaKey && inKana { if score > 30 { score -= 30 } else { score = 01 unmodified line }
reg := !irregular if existing, ok := entryKeys[k]; !ok { entryKeys[k] = keyMeta{priScore: score, isRegular: reg} entryKeys[k] = keyMeta{priScore: score, isCommon: formCommon, isRegular: reg} } else { newScore := existing.priScore if score < newScore { newScore = score } newReg := existing.isRegular || reg entryKeys[k] = keyMeta{priScore: newScore, isRegular: newReg} entryKeys[k] = keyMeta{ priScore: newScore, isCommon: existing.isCommon || formCommon, isRegular: existing.isRegular || reg, } } }
for _, k := range raw.Kanji { addKey(k.Text, k.Text, false, isIrregularOrSearchOnly(k.Tags)) addKey(k.Text, k.Text, false, k.Common, isIrregularOrSearchOnly(k.Tags)) } for _, r := range raw.Kana { norm := jp.ToHiragana(r.Text) addKey(norm, r.Text, true, isIrregularOrSearchOnly(r.Tags)) addKey(norm, r.Text, true, r.Common, isIrregularOrSearchOnly(r.Tags)) }
for k, meta := range entryKeys { postings = append(postings, postingItem{ key: k, priScore: meta.priScore, isCommon: isCommon, isCommon: meta.isCommon, isRegular: meta.isRegular, entryIndex: curEntryIndex, }) }1200 unmodified linesExpand all155 unmodified linesExpand all156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206565 unmodified linesExpand all155 unmodified linesExpand all } ] }, { "id": "1546550", "kanji": [ { "common": true, "text": "葉", "tags": [] } ], "kana": [ { "common": true, "text": "は", "tags": [], "appliesToKanji": [ "*" ] } ], "sense": [ { "partOfSpeech": [ "n" ], "appliesToKanji": [ "*" ], "appliesToKana": [ "*" ], "field": [], "dialect": [], "misc": [], "info": [], "gloss": [ { "lang": "eng", "text": "leaf" } ] } ] }, { "id": "2028920", "kanji": [], "kana": [ {565 unmodified linesExpand all68 unmodified lines69707172737475767778798081828384858687888990919293949568 unmodified lines<re_pri>news1</re_pri><re_pri>nf02</re_pri></r_ele></entry><entry><ent_seq>1546550</ent_seq><k_ele><keb>葉</keb><ke_pri>ichi1</ke_pri><ke_pri>news1</ke_pri><ke_pri>nf03</ke_pri></k_ele><r_ele><reb>は</reb><re_pri>ichi1</re_pri><re_pri>news1</re_pri><re_pri>nf03</re_pri></r_ele></entry><entry><ent_seq>2028920</ent_seq><r_ele><reb>は</reb><re_pri>ichi1</re_pri></r_ele></entry></JMdict>