Finish conversion to JS objects

classifier/WhosOnFirstClassifier.js

@@ -13,19 +13,19 @@ const normalize = require('../tokenization/normalizer')({ lowercase: true, remov
 // note: these should be defined from most granular to least granular
 const placetypes = {
   'locality': {
-    files: ['name:*_x_preferred.txt'],
+    files: ['name:eng_x_preferred.txt', 'name:fra_x_preferred.txt'],
     classifications: [AreaClassification, LocalityClassification]
   },
   'region': {
-    files: ['abrv:*_x_preferred.txt', 'name:*_x_preferred.txt'],
+    files: ['abrv:eng_x_preferred.txt', 'name:eng_x_preferred.txt'],
     classifications: [AreaClassification, RegionClassification]
   },
   // 'dependency': {
   //   files: ['wof:shortcode.txt', 'name:eng_x_preferred.txt'],
   //   classifications: [AreaClassification, DependencyClassification]
   // },
   'country': {
-    files: ['name:*_x_preferred.txt', 'wof:country.txt', 'wof:country_alpha3.txt'],
+    files: ['name:eng_x_preferred.txt', 'wof:country.txt', 'wof:country_alpha3.txt'],
     classifications: [AreaClassification, CountryClassification]
   }
 }

resources/libpostal/libpostal.js

@@ -11,11 +11,9 @@ function load (index, langs, filename, options) {
 
   // Load libpostal/dictionaries
   langs.forEach(lang => {
-    let dict = libpostaldict[lang][key]
-    if (dict === undefined) { return }
-    dict.split('\n').forEach(row => {
-      row.split('|').forEach(add.bind(null, lang))
-    }, this)
+    let d = libpostaldict[lang][key]
+    if (d === undefined) { return }
+    d.forEach(entry => add(lang, entry))
   }, this)
 
   // Load pelias/dictionaries/libostal

resources/libpostal/resources-download.sh

@@ -17,8 +17,11 @@ for d in dictionaries/*/; do
   dname=$(basename "$d")
   files=$(for f in "$d"*.txt; do
     fname=$(basename "$f" .txt)
-    # Read the file and convert its content into a JSON string:
-    content=$(jq -R -s . "$f")
+    # Read the file and convert it to the format the library expects:
+    content=$(jq -c -R -s 'split("\n")
+                           | map(select(length > 0))
+                           | map(split("|"))
+                           | flatten' "$f")
     echo "\"$fname\": $content"
   done | paste -sd, -)
   echo "{\"$dname\": { $files }}"