package build
// Attribution:
// JLPT word lists are based on data from Jonathan Waller's JLPT resources
// (http://www.tanos.co.uk/jlpt/), licensed under Creative Commons Attribution (CC BY),
// curated in github.com/jamsinclair/open-anki-jlpt-decks.
import (
"archive/zip"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
)
const (
accentsURL = "https://raw.githubusercontent.com/mifunetoshiro/kanjium/master/data/source_files/raw/accents.txt"
jmdictSimplified = "https://api.github.com/repos/scriptin/jmdict-simplified/releases/latest"
jmdictXMLURL = "http://ftp.edrdg.org/pub/Nihongo/JMdict_e.gz"
furiganaAPIURL = "https://api.github.com/repos/Doublevil/JmdictFurigana/releases/latest"
jlptBaseURL = "https://raw.githubusercontent.com/jamsinclair/open-anki-jlpt-decks/master/src/"
// Fallback URLs
fallbackJMdictURL = "https://github.com/scriptin/jmdict-simplified/releases/download/3.6.2%2B20261005200550/jmdict-eng-3.6.2%2B20261005200550.json.zip"
fallbackKanjidicURL = "https://github.com/scriptin/jmdict-simplified/releases/download/3.6.2%2B20261005200550/kanjidic2-en-3.6.2%2B20261005200550.json.zip"
fallbackJmnedictURL = "https://github.com/scriptin/jmdict-simplified/releases/download/3.6.2%2B20261005200550/jmnedict-all-3.6.2%2B20261005200550.json.zip"
fallbackFuriganaURL = "https://github.com/Doublevil/JmdictFurigana/releases/download/2.3.1%2B2026-09-25/JmdictFurigana.json.zip"
fallbackKradURL = "https://github.com/scriptin/jmdict-simplified/releases/download/3.6.2%2B20261005200550/kradfile-3.6.2%2B20261005200550.json.zip"
fallbackRadkURL = "https://github.com/scriptin/jmdict-simplified/releases/download/3.6.2%2B20261005200550/radkfile-3.6.2%2B20261005200550.json.zip"
)
type FetchOptions struct {
Names bool
}
type FetchOption func(*FetchOptions)
func FetchWithNames() FetchOption {
return func(o *FetchOptions) {
o.Names = true
}
}
type githubRelease struct {
Assets []struct {
Name string `json:"name"`
BrowserDownloadURL string `json:"browser_download_url"`
Size int64 `json:"size"`
} `json:"assets"`
}
// Fetch downloads the required dictionary sources into a temporary directory under dir,
// builds dict.bin atomically in dir, and cleans up all temporary download files.
func Fetch(ctx context.Context, dir string, opts ...FetchOption) error {
var fo FetchOptions
for _, opt := range opts {
opt(&fo)
}
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
// Download into a temporary subdirectory on the same filesystem
tmpDir, err := os.MkdirTemp(dir, "dict-download-*")
if err != nil {
return fmt.Errorf("create temp download dir: %w", err)
}
defer os.RemoveAll(tmpDir)
jmdictURL, kanjidicURL, jmnedictURL, kradURL, radkURL, err := resolveReleaseAssets(ctx)
if err != nil {
jmdictURL = fallbackJMdictURL
kanjidicURL = fallbackKanjidicURL
jmnedictURL = fallbackJmnedictURL
kradURL = fallbackKradURL
radkURL = fallbackRadkURL
}
if kradURL == "" {
kradURL = fallbackKradURL
}
if radkURL == "" {
radkURL = fallbackRadkURL
}
jmdictZipPath := filepath.Join(tmpDir, "jmdict-eng.json.zip")
kanjidicZipPath := filepath.Join(tmpDir, "kanjidic2-en.json.zip")
accentsPath := filepath.Join(tmpDir, "accents.txt")
jmdictXMLPath := filepath.Join(tmpDir, "JMdict_e.gz")
furiganaZipPath := filepath.Join(tmpDir, "JmdictFurigana.json.zip")
jmnedictZipPath := filepath.Join(tmpDir, "jmnedict-all.json.zip")
kradZipPath := filepath.Join(tmpDir, "kradfile.json.zip")
radkZipPath := filepath.Join(tmpDir, "radkfile.json.zip")
// 1. Download JMdict zip
if err := downloadZip(ctx, jmdictURL, jmdictZipPath); err != nil {
return fmt.Errorf("download jmdict: %w", err)
}
// 2. Download Kanjidic zip
if err := downloadZip(ctx, kanjidicURL, kanjidicZipPath); err != nil {
return fmt.Errorf("download kanjidic: %w", err)
}
// 2b. Download Kradfile zip
if err := downloadZip(ctx, kradURL, kradZipPath); err != nil {
return fmt.Errorf("download kradfile: %w", err)
}
// 2c. Download Radkfile zip
if err := downloadZip(ctx, radkURL, radkZipPath); err != nil {
return fmt.Errorf("download radkfile: %w", err)
}
// 3. Download Accents txt
if err := downloadFile(ctx, accentsURL, accentsPath, false); err != nil {
return fmt.Errorf("download accents: %w", err)
}
// 4. Download JMdict_e.gz (best-effort)
_ = downloadFile(ctx, jmdictXMLURL, jmdictXMLPath, false)
// 5. Download JmdictFurigana.json.zip (best-effort)
furiURL, err := resolveFuriganaAsset(ctx)
if err != nil {
furiURL = fallbackFuriganaURL
}
_ = downloadZip(ctx, furiURL, furiganaZipPath)
// 6. Download JLPT decks n1.csv..n5.csv (best-effort)
jlptLevels := []string{"n1", "n2", "n3", "n4", "n5"}
for _, lvl := range jlptLevels {
dst := filepath.Join(tmpDir, lvl+".csv")
_ = downloadFile(ctx, jlptBaseURL+lvl+".csv", dst, false)
}
// 7. Download JMnedict names if requested
if fo.Names {
if jmnedictURL == "" {
jmnedictURL = fallbackJmnedictURL
}
_ = downloadZip(ctx, jmnedictURL, jmnedictZipPath)
}
// Open JMdict reader from zip
jmz, err := zip.OpenReader(jmdictZipPath)
if err != nil {
return fmt.Errorf("open jmdict zip: %w", err)
}
defer jmz.Close()
var jmdictEntry *zip.File
for _, f := range jmz.File {
if strings.HasSuffix(f.Name, ".json") {
jmdictEntry = f
break
}
}
if jmdictEntry == nil {
return errors.New("no .json file found inside jmdict zip")
}
jmdictReader, err := jmdictEntry.Open()
if err != nil {
return fmt.Errorf("open jmdict json in zip: %w", err)
}
defer jmdictReader.Close()
// Open Kanjidic reader from zip
kz, err := zip.OpenReader(kanjidicZipPath)
if err != nil {
return fmt.Errorf("open kanjidic zip: %w", err)
}
defer kz.Close()
var kanjidicEntry *zip.File
for _, f := range kz.File {
if strings.HasSuffix(f.Name, ".json") {
kanjidicEntry = f
break
}
}
if kanjidicEntry == nil {
return errors.New("no .json file found inside kanjidic zip")
}
kanjidicReader, err := kanjidicEntry.Open()
if err != nil {
return fmt.Errorf("open kanjidic json in zip: %w", err)
}
defer kanjidicReader.Close()
// Open accents file
accentsFile, err := os.Open(accentsPath)
if err != nil {
return fmt.Errorf("open accents.txt: %w", err)
}
defer accentsFile.Close()
var buildOpts []BuildOption
// Open Kradfile zip if available
if kz, err := zip.OpenReader(kradZipPath); err == nil {
defer kz.Close()
for _, f := range kz.File {
if strings.HasSuffix(f.Name, ".json") {
rc, err := f.Open()
if err == nil {
defer rc.Close()
buildOpts = append(buildOpts, WithKradfile(rc))
}
break
}
}
}
// Open Radkfile zip if available
if rz, err := zip.OpenReader(radkZipPath); err == nil {
defer rz.Close()
for _, f := range rz.File {
if strings.HasSuffix(f.Name, ".json") {
rc, err := f.Open()
if err == nil {
defer rc.Close()
buildOpts = append(buildOpts, WithRadkfile(rc))
}
break
}
}
}
// Open priorities XML if available
if fi, err := os.Stat(jmdictXMLPath); err == nil && fi.Size() > 0 {
xmlFile, err := os.Open(jmdictXMLPath)
if err == nil {
defer xmlFile.Close()
buildOpts = append(buildOpts, WithPriorities(xmlFile))
}
}
// Open furigana zip if available
if fz, err := zip.OpenReader(furiganaZipPath); err == nil {
defer fz.Close()
for _, f := range fz.File {
if strings.HasSuffix(f.Name, ".json") {
rc, err := f.Open()
if err == nil {
defer rc.Close()
buildOpts = append(buildOpts, WithFurigana(rc))
}
break
}
}
}
// Open JLPT decks if available
jlptMap := make(map[string]io.Reader)
for _, lvl := range jlptLevels {
path := filepath.Join(tmpDir, lvl+".csv")
if fi, err := os.Stat(path); err == nil && fi.Size() > 0 {
f, err := os.Open(path)
if err == nil {
defer f.Close()
jlptMap[strings.ToUpper(lvl)] = f
}
}
}
if len(jlptMap) > 0 {
buildOpts = append(buildOpts, WithJLPT(jlptMap))
}
// Open JMnedict if names requested
if fo.Names {
if nz, err := zip.OpenReader(jmnedictZipPath); err == nil {
defer nz.Close()
for _, f := range nz.File {
if strings.HasSuffix(f.Name, ".json") {
rc, err := f.Open()
if err == nil {
defer rc.Close()
buildOpts = append(buildOpts, WithNames(rc))
}
break
}
}
}
}
// Build dict into temporary file in tmpDir, then atomically rename to dir/dict.bin
tmpDictBin := filepath.Join(tmpDir, "dict.bin")
if err := Build(jmdictReader, kanjidicReader, accentsFile, tmpDictBin, buildOpts...); err != nil {
return fmt.Errorf("build dict.bin: %w", err)
}
finalDictBin := filepath.Join(dir, "dict.bin")
if err := os.Rename(tmpDictBin, finalDictBin); err != nil {
return fmt.Errorf("atomically rename dict.bin: %w", err)
}
return nil
}
func resolveReleaseAssets(ctx context.Context) (jmdictURL, kanjidicURL, jmnedictURL, kradURL, radkURL string, err error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, jmdictSimplified, nil)
if err != nil {
return "", "", "", "", "", err
}
req.Header.Set("User-Agent", "chirami")
req.Header.Set("Accept", "application/vnd.github+json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", "", "", "", "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", "", "", "", "", fmt.Errorf("github api returned status %d", resp.StatusCode)
}
var rel githubRelease
if err := json.NewDecoder(resp.Body).Decode(&rel); err != nil {
return "", "", "", "", "", err
}
for _, a := range rel.Assets {
name := a.Name
if strings.HasPrefix(name, "jmdict-eng-") && strings.HasSuffix(name, ".json.zip") &&
!strings.Contains(name, "common") && !strings.Contains(name, "examples") {
jmdictURL = a.BrowserDownloadURL
}
if strings.HasPrefix(name, "kanjidic2-en-") && strings.HasSuffix(name, ".json.zip") {
kanjidicURL = a.BrowserDownloadURL
}
if strings.HasPrefix(name, "jmnedict-all-") && strings.HasSuffix(name, ".json.zip") {
jmnedictURL = a.BrowserDownloadURL
}
if strings.HasPrefix(name, "kradfile-") && strings.HasSuffix(name, ".json.zip") {
kradURL = a.BrowserDownloadURL
}
if strings.HasPrefix(name, "radkfile-") && strings.HasSuffix(name, ".json.zip") {
radkURL = a.BrowserDownloadURL
}
}
if jmdictURL == "" || kanjidicURL == "" {
return "", "", "", "", "", fmt.Errorf("failed to find matching assets in release (jmdict=%q, kanjidic=%q)", jmdictURL, kanjidicURL)
}
return jmdictURL, kanjidicURL, jmnedictURL, kradURL, radkURL, nil
}
func resolveFuriganaAsset(ctx context.Context) (string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, furiganaAPIURL, nil)
if err != nil {
return "", err
}
req.Header.Set("User-Agent", "chirami")
req.Header.Set("Accept", "application/vnd.github+json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("github api returned status %d", resp.StatusCode)
}
var rel githubRelease
if err := json.NewDecoder(resp.Body).Decode(&rel); err != nil {
return "", err
}
for _, a := range rel.Assets {
if a.Name == "JmdictFurigana.json.zip" {
return a.BrowserDownloadURL, nil
}
}
for _, a := range rel.Assets {
if a.Name == "JmdictFurigana.json" {
return a.BrowserDownloadURL, nil
}
}
return "", errors.New("no JmdictFurigana asset found")
}
func downloadZip(ctx context.Context, url, dst string) error {
if z, err := zip.OpenReader(dst); err == nil {
z.Close()
return nil
}
return downloadFile(ctx, url, dst, true)
}
func downloadFile(ctx context.Context, url, dst string, mustBeZip bool) error {
if fi, err := os.Stat(dst); err == nil && fi.Size() > 0 {
if !mustBeZip {
return nil
}
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
req.Header.Set("User-Agent", "chirami")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("GET %s: %s", url, resp.Status)
}
tmpDst := dst + ".download"
out, err := os.Create(tmpDst)
if err != nil {
return err
}
defer func() {
out.Close()
os.Remove(tmpDst)
}()
if _, err := io.Copy(out, resp.Body); err != nil {
return err
}
if err := out.Close(); err != nil {
return err
}
return os.Rename(tmpDst, dst)
}