Merge remote-tracking branch 'origin/master'

lib/extract.js

@@ -2,14 +2,14 @@
  * Generates sources from DB
  */
 
-const { createDir, writeJSON } = require( './utils' )
+const { createDir, writeJSON, akharToUnicode } = require( './utils' )
 
 const knex = require( 'knex' )( require( '../knexfile' ) )
 
 const OUTPUT_DIR = './seeds'
 
 
-const getTable = async tableName => knex( tableName ).select().map( ( { name } ) => name )
+const getTable = async tableName => knex( tableName ).select().orderBy( 'id' )
 
 const main = async () => {
   // Pull out lists from two-columned tables
@@ -25,44 +25,44 @@ const main = async () => {
     [ writers, `${OUTPUT_DIR}/writers.json` ],
     [ raags, `${OUTPUT_DIR}/raags.json` ],
     [ lineTypes, `${OUTPUT_DIR}/line_types.json` ],
-  ].map( ( [ data, path ] ) => writeJSON( path, data ) ) )
+  ]
+  // Extract name only
+    .map( ( [ data, path ] ) => [ data.map( ( { name } ) => name ), path ] )
+    // Write it to disk
+    .map( ( [ data, path ] ) => writeJSON( path, data ) ) )
 
   const banisDir = `${OUTPUT_DIR}/banis`
   createDir( banisDir )
 
   // Generate the files for each of the banis, with ranges
-  banis.forEach( async name => {
-    const id = banis.indexOf( name ) + 1
+  banis.forEach( async ( { name, id: baniId } ) => {
     const path = `${banisDir}/${name}.json`
 
     const lines = ( await knex( 'bani_lines' )
       .min( 'line_id as start_line' )
       .max( 'line_id as end_line' )
-      .where( 'bani_id', id )
+      .where( 'bani_id', baniId )
       .groupBy( 'line_group' ) )
 
     await writeJSON( path, lines )
     console.log( `Bani ${name} written to ${path}` )
   } )
 
-  console.log( `Loaded sources: ${sources}` )
-  sources.forEach( async source => {
-    const sourceId = sources.indexOf( source ) + 1
-    const sourcesDir = `${OUTPUT_DIR}/sources/${source}`
-    const shabadsDir = `${OUTPUT_DIR}/shabads/${source}`
+  sources.forEach( async ( { name: source, id: sourceId } ) => {
+    const sourcesDir = `${OUTPUT_DIR}/sources/${sourceId}`
+    const shabadsDir = `${OUTPUT_DIR}/shabads/${sourceId}`
 
       // Create source folder if it does not already exist
     ;[ sourcesDir, shabadsDir ].forEach( createDir )
 
     // Create shabad files, by writer
-    writers.forEach( async ( name, index ) => {
-      const id = index + 1
-      const path = `${shabadsDir}/${name}.json`
+    writers.forEach( async ( { name, id: writerId } ) => {
+      const path = `${shabadsDir}/${writerId}.json`
 
       const shabads = await knex( 'shabads' )
         .select()
         .where( 'source_id', sourceId )
-        .andWhere( 'writer_id', id )
+        .andWhere( 'writer_id', writerId )
         .orderBy( 'id' )
 
       if ( shabads.length ) {
@@ -104,11 +104,17 @@ const main = async () => {
         ] )
         .join( 'shabads', 'shabads.id', 'lines.shabad_id' )
         .whereBetween( 'ang', [ angBatch, angBatch + 99 ] )
-        .andWhere( 'source_id', sources.indexOf( source ) + 1 )
+        .andWhere( 'source_id', sourceId )
         .orderBy( 'lines.id' )
 
-      // Transform each line into groups of angs
       const angs = lines
+      // Convert ascii to unicode
+        .map( line => ( {
+          ...line,
+          gurmukhi: akharToUnicode( line.gurmukhi ),
+          pronunciation: akharToUnicode( line.pronunciation || '' ),
+        } ) )
+        // Transform each line into groups of angs
         .reduce( ( angs, line ) => ( {
           ...angs,
           [ line.ang ]: [ ...( angs[ line.ang ] || [] ),

lib/mappings.json

@@ -0,0 +1,85 @@
+{
+  "0": "੦",
+  "1": "੧",
+  "2": "੨",
+  "3": "੩",
+  "4": "੪",
+  "5": "੫",
+  "6": "੬",
+  "7": "੭",
+  "8": "੮",
+  "9": "੯",
+  "]": "॥",
+  "[": "।",
+  ">": "",
+  "@": "੍ਹ",
+  "\\": "ਞ",
+  "&": "ਫ਼",
+  "†": "੍ਟ",
+  "`": "ੱ",
+  "´": "੍ਯ",
+  "^": "ਖ਼",
+  "ˆ": "ਂ",
+  "®": "੍ਰ",
+  "<": "ੴ",
+  "|": "ਙ",
+  "~": "੍ਨ",
+  "A": "ਅ",
+  "a": "ੳ",
+  "B": "ਭ",
+  "b": "ਬ",
+  "C": "ਛ",
+  "c": "ਚ",
+  "ç": "੍ਚ",
+  "D": "ਧ",
+  "d": "ਦ",
+  "E": "ਓ",
+  "e": "ੲ",
+  "F": "ਢ",
+  "f": "ਡ",
+  "G": "ਘ",
+  "g": "ਗ",
+  "H": "੍ਹ",
+  "h": "ਹ",
+  "I": "ੀ",
+  "i": "ਿ",
+  "Í": "੍ਵ",
+  "J": "ਝ",
+  "j": "ਜ",
+  "K": "ਖ",
+  "k": "ਕ",
+  "L": "ਲ਼",
+  "l": "ਲ",
+  "M": "ੰ",
+  "m": "ਮ",
+  "N": "ਂ",
+  "n": "ਨ",
+  "O": "ੌ",
+  "o": "ੋ",
+  "œ": "੍ਤ",
+  "P": "ਫ",
+  "p": "ਪ",
+  "Q": "ਥ",
+  "q": "ਤ",
+  "R": "੍ਰ",
+  "r": "ਰ",
+  "S": "ਸ਼",
+  "s": "ਸ",
+  "T": "ਠ",
+  "t": "ਟ",
+  "U": "ੂ",
+  "u": "ੁ",
+  "Ú": "ਃ",
+  "ü": "ੁ",
+  "V": "ੜ",
+  "v": "ਵ",
+  "W": "ਾਂ",
+  "w": "ਾ",
+  "X": "ਯ",
+  "x": "ਣ",
+  "Y": "ੈ",
+  "y": "ੇ",
+  "Z": "ਗ਼",
+  "z": "ਜ਼",
+  "µ": "ੰ"
+}

lib/utils.js

@@ -6,6 +6,7 @@ const { promisify } = require( 'util' )
 const { writeFile, existsSync, mkdirSync } = require( 'fs' )
 const { basename } = require( 'path' )
 
+const unicodeMappings = require( './mappings' )
 
 // Wraps writeFile in a promise
 const writeFileAsync = promisify( writeFile )
@@ -41,39 +42,41 @@ const createDir = path => {
  * Generates a string of the first letters in a line.
  * Removes common strings, then maps each individual word to a letter.
  * First letters are returned if the first letters string > 1.
- * @param source The source of the line, from `sources.json`
+ * @param sourceId The source of the line, from `sources.json`
  * @param gurmukhi The line
  */
-const generateFirstLetters = ( source, { gurmukhi } ) => [
-  [ /] ([0-9]*) ]/, ']$1]' ],
-  [ /] rhwau ]/, '] ]' ],
-  [ /] rhwau dUjw ]/, '] ]' ],
-  [ /] suDu/, ']' ],
-  [ /] jumlw/, ']' ],
-  [ /] bweIs caupdy qQw pMcpdy/, ']' ],
-  [ /] Ckw 1/, ']' ],
-  [ /] Cky 2/, ']' ],
-  [ /] Cky 3/, ']' ],
-  [ /] joVu/, ']' ],
-  [ /^Awsw ] iqpdw ] iekqukw ]$/, '' ],
-  [ /^kbIru ] mwrU ]$/, '' ],
-  [ /^muK Bwg$/, '' ],
-  [ /.*m \d.*|.*mhlw \d.*|.*hlI bwc.*|.*kbIr jI.*|.*bwc ].*/, '' ],
-  [ /.*CMd ].*/, match => ( source === 'D' ? '' : match ) ],
+const generateFirstLetters = ( sourceId, { gurmukhi } ) => [
+  [ /] ([੦੧੨੩੪੫੬੭੮੯]*) ॥/, '॥$1॥' ],
+  [ /॥ ਰਹਾੳੁ ॥/, '॥ ॥' ],
+  [ /॥ ਰਹਾੳੁ ਦੂਜਾ ॥/, '॥ ॥' ],
+  [ /॥ ਸੁਧੁ/, '॥' ],
+  [ /॥ ਜੁਮਲਾ/, '॥' ],
+  [ /॥ ਬਾੲੀਸ ਚੳੁਪਦੇ ਤਥਾ ਪੰਚਪਦੇ/, '॥' ],
+  [ /॥ ਛਕਾ ੧/, '॥' ],
+  [ /॥ ਛਕਾ ੨/, '॥' ],
+  [ /॥ ਛਕਾ ੩/, '॥' ],
+  [ /॥ ਜੋੜੁ/, '॥' ],
+  [ /^ਅਾਸਾ ॥ ਤਿਪਦਾ ॥ ੲਿਕਤੁਕਾ ॥$/, '' ],
+  [ /^ਕਬੀਰੁ ॥ ਮਾਰੂ ॥$/, '' ],
+  [ /^ਮੁਖ ਭਾਗ$/, '' ],
+  [ /.*ਮ [੦੧੨੩੪੫੬੭੮੯].*|.*ਮਹਲਾ [੦੧੨੩੪੫੬੭੮੯].*|.*ਹਲੀ ਬਾਚ.*|.*ਕਬੀਰ ਜੀ.*|.*ਬਾਚ ॥.*/, '' ],
+  // eslint-disable-next-line
+  [ /.*ਛੰਦ ॥.*/, match => ( sourceId == 2 ? '' : match ) ],
 ]
   .reduce( ( result, [ exp, sub ] ) => result.replace( exp, sub ), gurmukhi )
   .split( ' ' )
   .map( ( [ firstLetter, secondLetter ] ) => {
     const replacements = {
-      i: secondLetter || '',
-      E: 'a',
-      S: 's',
-      z: 'j',
-      Z: 'g',
-      L: 'l',
-      '^': 'K',
-      '&': 'P',
-      ']': '',
+      'ਿ': secondLetter || '',
+      ਓ: 'ੳ',
+      ਸ਼: 'ਸ',
+      ਜ਼: 'ਜ',
+      ਗ਼: 'ਗ',
+      ਲ਼: 'ਲ',
+      ਖ਼: 'ਖ',
+      ਫ਼: 'ਫ',
+      '॥': '',
+      '।': '',
     }
 
     // Return the replacement if there was one, otherwise the first letter
@@ -82,4 +85,15 @@ const generateFirstLetters = ( source, { gurmukhi } ) => [
   .join( '' )
   .replace( /^.$/, '' )
 
-module.exports = { writeJSON, stripExtension, createDir, generateFirstLetters }
+/**
+ * Converts ASCII text used in the GurmukhiAkhar font to Unicode.
+ * @param text The ascii text to convert
+ */
+const akharToUnicode = text => text
+  .replace( /i./g, match => match.split( '' ).reverse().join( '' ) )
+  .split( '' )
+  .map( c => ( unicodeMappings[ c ] !== undefined ? unicodeMappings[ c ] : c ) )
+  .join( '' )
+  .replace( /ਿ੍./g, match => match.slice( 1, 3 ).concat( 'ਿ' ) )
+
+module.exports = { writeJSON, stripExtension, createDir, generateFirstLetters, akharToUnicode }

seeds/line_types.json

@@ -1,6 +1,6 @@
 [
-  "Manglacharan",
-  "head",
-  "tuk",
-  "rahau"
+  "ਮੰਗਲ",
+  "ਸਿਰਲੇਖ",
+  "ਤੁਕ",
+  "ਰਹਾਉ"
 ]

seeds/raags.json

@@ -1,57 +1,57 @@
 [
-  "1",
-  "2",
-  "3",
-  "4",
-  "5",
-  "6",
-  "7",
-  "8",
-  "9",
-  "10",
-  "11",
-  "12",
-  "13",
-  "14",
-  "15",
-  "16",
-  "17",
-  "18",
-  "19",
-  "20",
-  "21",
-  "22",
-  "23",
-  "24",
-  "25",
-  "26",
-  "27",
-  "28",
-  "29",
-  "30",
-  "31",
-  "32",
-  "33",
-  "34",
-  "35",
-  "36",
-  "37",
-  "38",
-  "39",
-  "40",
-  "41",
-  "42",
-  "43",
-  "44",
-  "45",
-  "46",
-  "47",
-  "48",
-  "49",
-  "50",
-  "51",
-  "52",
-  "53",
-  "54",
-  "55"
+  "ਜਪੁ ਜੀ ਸਾਹਿਬ",
+  "ਸੋ ਦਰ",
+  "ਸੋ ਪੁਰਖੁ",
+  "ਸੋਹਿਲਾ ਸਾਹਿਬ",
+  "ਸਿਰੀ ਰਾਗ",
+  "ਰਾਗੁ ਮਾਝ",
+  "ਰਾਗੁ ਗਉੜੀ",
+  "ਰਾਗੁ ਆਸਾ",
+  "ਰਾਗੁ ਗੂਜਰੀ",
+  "ਰਾਗੁ ਦੇਵਗੰਧਾਰੀ",
+  "ਰਾਗੁ ਬਿਹਾਗੜਾ",
+  "ਰਾਗੁ ਵਡਹੰਸ",
+  "ਰਾਗੁ ਸੋਰਠ",
+  "ਰਾਗੁ ਧਨਾਸਰੀ",
+  "ਰਾਗੁ ਜੈਤਸਰੀ",
+  "ਰਾਗੁ ਟੋਡੀ",
+  "ਰਾਗੁ ਬੈਰਾੜੀ",
+  "ਰਾਗੁ ਤਿਲੰਗ",
+  "ਰਾਗੁ ਸੂਹੀ",
+  "ਰਾਗੁ ਬਿਲਾਵਲ",
+  "ਰਾਗੁ ਗੌਂਡ",
+  "ਰਾਗੁ ਰਾਮਕਲੀ",
+  "ਰਾਗੁ ਨਟ ਨਰਾਇਣ",
+  "ਰਾਗੁ ਮਾਲੀ ਗਉੜਾ",
+  "ਰਾਗੁ ਮਾਰੂ",
+  "ਰਾਗੁ ਤੁਖਾਰੀ",
+  "ਰਾਗੁ ਕੇਦਾਰਾ",
+  "ਰਾਗੁ ਭੈਰਉ",
+  "ਰਾਗੁ ਬਸੰਤੁ",
+  "ਰਾਗੁ ਸਾਰੰਗ",
+  "ਰਾਗੁ ਮਲਾਰ",
+  "ਰਾਗੁ ਕਾਨੜਾ",
+  "ਰਾਗੁ ਕਲਿਆਣ",
+  "ਰਾਗੁ ਪਰਭਾਤੀ",
+  "ਰਾਗੁ ਜੈਜਾਵੰਤੀ",
+  "ਸਲੋਕ ਸਹਸਕ੍ਰਿਤੀ",
+  "ਗਾਥਾ",
+  "ਫ਼ੁਨਹੇ",
+  "ਚਉਬੋਲੇ",
+  "ਸਲੋਕ ਭਗਤ ਕਬੀਰ ਜੀਉ ਕੇ",
+  "ਸਲੋਕ ਸੇਖ ਫਰੀਦ ਕੇ",
+  "ਸਵਯੇ ਸ੍ਰੀ ਮੁਖਬਾਕ੍ਯ ਮਹਲਾ ੫",
+  "ਸਵਈਏ ਮਹਲੇ ਪਹਿਲੇ ਕੇ",
+  "ਸਵਈਏ ਮਹਲੇ ਦੂਜੇ ਕੇ",
+  "ਸਵਈਏ ਮਹਲੇ ਤੀਜੇ ਕੇ",
+  "ਸਵਈਏ ਮਹਲੇ ਚਉਥੇ ਕੇ",
+  "ਸਵਈਏ ਮਹਲੇ ਪੰਜਵੇ ਕੇ",
+  "ਸਲੋਕ ਵਾਰਾਂ ਤੇ ਵਧੀਕ ",
+  "ਸਲੋਕ ਮਹਲਾ ੯",
+  "ਮੁੰਦਾਵਣੀ",
+  "ਰਾਗਮਾਲਾ",
+  "ਸ੍ਰੀ ਦਸਮ ਗ੍ਰੰਥ",
+  "ਰਚਨਾਵਾਂ ਭਾਈ ਗੁਰਦਾਸ ਜੀ",
+  "ਕਵਿਤਾਵਾਂ ਭਾਈ ਨੰਦ ਲਾਲ ਜੀ",
+  "ਅੰਮ੍ਰਿਤ ਕੀਰਤਨ"
 ]