try fixing merge ipa and speed up conversion

yomidevs · Jul 13, 2024 · 7181e8f · 7181e8f
1 parent b9093db
commit 7181e8f
Show file tree

Hide file tree

Showing 3 changed files with 64 additions and 15 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -9,6 +9,7 @@ jobs:
     outputs:
       tag: ${{ steps.tag.outputs.tag }}
       languages: ${{ steps.load-languages.outputs.languages }}
+      isos: ${{ steps.load-languages.outputs.isos }}
       calver: ${{ steps.tag.outputs.calver }}
     steps:
       - name: Checkout repository
@@ -47,6 +48,8 @@ jobs:
         run: |
           languages=$(jq -r '.[].language' languages.json | jq -R -s -c 'split("\n") | map(select(. != ""))')
           echo "languages=$languages" >> $GITHUB_OUTPUT
+          isos=$(jq -r '.[].iso' languages.json | jq -R -s -c 'split("\n") | map(select(. != ""))')
+          echo "isos=$isos" >> $GITHUB_OUTPUT
       
       - name: Generate markdown table
         id: generate_markdown
@@ -84,7 +87,7 @@ jobs:
     strategy:
       max-parallel: 1
       matrix:
-        target_language: ${{fromJson(needs.prepare.outputs.languages)}}
+        edition_language: ${{fromJson(needs.prepare.outputs.languages)}}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -111,10 +114,7 @@ jobs:
         run: sudo apt-get install -y gzip
 
       - name: Run auto.sh script
-        run: ./auto.sh ? "${{ matrix.target_language }}"
-
-      - name: Run merge-ipa
-        run: node merge-ipa.js
+        run: ./auto.sh "${{ matrix.edition_language }}" ? ?
 
       - name: Generate list of .zip files
         id: generate_file_list
@@ -125,7 +125,52 @@ jobs:
           cat zip_files.txt >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
 
-      - name: Release
+      - name: Upload dictionary files
+        uses: softprops/action-gh-release@v2
+        with:
+          name: ${{ needs.prepare.outputs.tag }}
+          tag_name: ${{ needs.prepare.outputs.tag }}
+          prerelease: true
+          generate_release_notes: true
+          files: ${{ steps.generate_file_list.outputs.zip_files }}
+
+  merge-ipa:
+    needs: ["prepare", "convert"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20'
+
+      - name: Download IPA dicts
+        run: |
+          for source_iso in ${{needs.prepare.outputs.isos}}; do
+            for target_iso in ${{needs.prepare.outputs.isos}}; do
+              url="https://github.com/themoeway/kaikki-to-yomitan/releases/download/${{needs.prepare.outputs.tag}}/kty-${source_iso}-${target_iso}-ipa.zip"
+              wget -nv "$url"
+            done
+          done
+
+      - name: Run merge-ipa
+        run: node merge-ipa.js
+
+      - name: Delete downloaded IPA files
+        run: rm *.zip
+
+      - name: Generate list of .zip files
+        id: generate_file_list
+        run: |
+          find data/language -type f -name '*.zip' > zip_files.txt
+          cat zip_files.txt
+          echo "zip_files<<EOF" >> $GITHUB_OUTPUT
+          cat zip_files.txt >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+      
+      - name: Upload dictionary files
         uses: softprops/action-gh-release@v2
         with:
           name: ${{ needs.prepare.outputs.tag }}

diff --git a/auto.sh b/auto.sh
@@ -88,8 +88,8 @@ max_memory_mb=${MAX_MEMORY_MB:-8192}
 
 
 # Check for the source_language and target_language arguments
-if [ -z "$1" ] || [ -z "$2" ]; then
-  echo "Usage: $0 <source_language> <target_language> [flags]"
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+  echo "Usage: $0 <edition> <source_language> <target_language> [flags]"
   exit 1
 fi
 
@@ -103,7 +103,7 @@ glossary_only=false
 
 flags=('d' 't' 'y' 'F' 'k' 'g')
 for flag in "${flags[@]}"; do
-  case "$3" in 
+  case "$4" in 
     *"$flag"*) 
       case "$flag" in
         'd') redownload=true ;;
@@ -139,8 +139,9 @@ npm i
 # Step 2: Run create-folder.js
 node 1-create-folders.js
 
-requested_source="$1"
-requested_target="$2"
+requested_edition="$1"
+requested_source="$2"
+requested_target="$3"
 
 declare -a languages="($(
   jq -r '.[] | @json | @sh' languages.json
@@ -157,6 +158,10 @@ for edition_lang in "${languages[@]}"; do
     continue
   fi
 
+  if [ "$edition_name" != "$requested_edition" ] && [ "$requested_edition" != "?" ]; then
+    continue
+  fi
+
   downloaded_edition_extract=false
 
   #Iterate over every language
@@ -207,7 +212,7 @@ for edition_lang in "${languages[@]}"; do
       if [ ! -f "$filepath" ] || [ "$redownload" = true ]; then
         url="https://kaikki.org/dictionary/$download_language/$filename"
         echo "Downloading $filename from $url"
-        wget "$url" -O "$filepath"
+        wget -nv "$url" -O "$filepath" 
       else
         echo "Kaikki dict already exists. Skipping download."
       fi
@@ -218,7 +223,7 @@ for edition_lang in "${languages[@]}"; do
       if [ ! -f "$edition_extract_path" ] || [ "$redownload" = true ] && [ "$downloaded_edition_extract" = false ]; then
         url="https://kaikki.org/dictionary/downloads/$edition_iso/$edition_extract.gz"
         echo "Downloading $edition_extract from $url"
-        wget "$url" -O "$edition_extract_path".gz
+        wget -nv "$url" -O "$edition_extract_path".gz 
         echo "Extracting $edition_extract"
         gunzip -f "$edition_extract_path".gz
         downloaded_edition_extract=true

diff --git a/merge-ipa.js b/merge-ipa.js
@@ -16,9 +16,8 @@ async function main(){
             let localIpa = [];
             let localTags = [];
 
-            const file = `data/language/${sourceIso}/${targetIso}/kty-${sourceIso}-${targetIso}-ipa.zip`;
+            const file = `kty-${sourceIso}-${targetIso}-ipa.zip`;
             if (existsSync(file)) {
-                console.log("found", file);
                 const zip = new StreamZip.async({ file });
                 const files = Object.keys(await zip.entries());
                 for (const file of files) {