Refactor lib and use keyList (#11)

* Refactor lib and use keyList for unused detection instead of downloading all locales
* fix unit tests

packages/nx-phrase/.gitignore

@@ -1,2 +1,4 @@
 /.phraseapp.yml
 src/executors/build/api
+**/.env.*
+**/.env

packages/nx-phrase/src/executors/build/schema.d.ts

@@ -1,4 +1,4 @@
-import { NonSensitiveArgs } from "../lib/types"
+import { NonSensitiveArgs } from "../../lib/types"
 
 export type Operation = "push" | "pull" | "find-unused"
 

packages/nx-phrase/src/executors/find-unused/__snapshots__/executor.spec.ts.snap

@@ -34,6 +34,12 @@ Array [
 ]
 `;
 
+exports[`find unused filters keys: keys that exist in phrase more than once 1`] = `
+Array [
+  "a.b.c.d",
+]
+`;
+
 exports[`find unused transforms keys: keys downloaded from phrase that are NOT used in the source (anymore) 1`] = `
 Array [
   "a.c.d.e",

packages/nx-phrase/src/executors/find-unused/executor.spec.ts

@@ -4,8 +4,8 @@ import { ExecutorContext } from "@nrwl/devkit"
 import nock from "nock"
 
 import executor from "./executor"
-import { NonSensitiveArgs } from "../lib/types"
-import { NPM_SCOPE } from "../../utils"
+import { NonSensitiveArgs } from "../../lib/types"
+import { NPM_SCOPE } from "../../lib/utils"
 import { readFileSync } from "fs"
 
 const options: Partial<NonSensitiveArgs> = {
@@ -16,15 +16,11 @@ const TEST_ASSETS_DIR = resolve(__dirname, "../../../test")
 
 function nockForProject(projectId = "project_id") {
     nock("https://api.phrase.com/v2")
-        .get(`/projects/${projectId}/locales`)
+        .get(`/projects/${projectId}/keys`)
         .matchHeader("Authorization", /token .*/)
-        .query({ per_page: 20 })
-        .replyWithFile(200, `${TEST_ASSETS_DIR}/localesListResponse.json`)
-        .get(/\/projects\/[^/]+\/locales\/[^/]+\/download/)
-        .matchHeader("Authorization", /token .*/)
-        .query({ file_format: "react_simple_json" })
-        .thrice()
-        .replyWithFile(200, `${TEST_ASSETS_DIR}/localeDownloadResponse.json`)
+        .query({ page: "1", per_page: "20" })
+        .once()
+        .replyWithFile(200, `${TEST_ASSETS_DIR}/keyListResponse.json`)
 }
 
 describe("find unused", () => {
@@ -100,6 +96,14 @@ describe("find unused", () => {
                 ).toString()
             )
         ).toMatchSnapshot("keys from phrase that are NOT valid according to the filter function")
+
+        expect(
+            JSON.parse(
+                readFileSync(
+                    resolve(TEST_ASSETS_DIR, ".nx-phrase/unused/test_app.phrase-duplicate-keys.json")
+                ).toString()
+            )
+        ).toMatchSnapshot("keys that exist in phrase more than once")
     })
 
     it("filters before applying transformers", async () => {

packages/nx-phrase/src/executors/find-unused/executor.ts

@@ -1,12 +1,12 @@
 import { ExecutorContext } from "@nrwl/devkit"
 import { existsSync, writeFileSync } from "fs"
 import { resolve } from "path"
-import { prepareOutput } from "../../utils"
+import { prepareOutput } from "../../lib/utils"
 
-import { getConfig, InternalPhraseConfig } from "../lib/config"
-import { PullHelper } from "../lib/pull"
-import { extractTranslations } from "../lib/push"
-import { NonSensitiveArgs } from "../lib/types"
+import { getConfig, InternalPhraseConfig } from "../../lib/config"
+import { extractTranslations } from "../../lib/push"
+import { NonSensitiveArgs } from "../../lib/types"
+import { PhraseClient } from "../../lib/phrase"
 
 const requiredConfigProperties = ["projectId", "output"]
 
@@ -43,21 +43,30 @@ async function getKeysFromSource(config: InternalPhraseConfig, outputFilePath: s
     return compiledTranslationKeys
 }
 
-async function getKeysFromPhrase(config: InternalPhraseConfig) {
-    const pull = new PullHelper({ ...config, fileFormat: "react_simple_json" })
+async function getKeysFromPhrase(config: InternalPhraseConfig): Promise<string[]> {
+    const phrase = new PhraseClient(config.phraseClientConfig)
+    const keysInPhrase = await phrase.keysListAll({ projectId: config.projectId, branch: config.branch })
+    const keysInPhraseList = keysInPhrase.map(({ name }) => name)
 
-    // downloads available translations
-    const locales = await pull.listLocales()
-    const localeToRawTranslations = await pull.downloadTranslations(locales)
-    const keysInPhrase = new Set<string>()
-    Object.keys(localeToRawTranslations).forEach((locale) =>
-        Object.keys(JSON.parse(localeToRawTranslations[locale])).forEach((key) => keysInPhrase.add(key))
-    )
+    return keysInPhraseList
 
-    // use set for deduplication
-    const keysInPhraseList = [...keysInPhrase]
+    // TODO deduplication of keys, report duplicates?
 
-    return keysInPhraseList
+    // Old (slow) logic:
+    // const pull = new PullHelper({ ...config, fileFormat: "react_simple_json" })
+
+    // // downloads available translations
+    // const locales = await pull.listLocales()
+    // const localeToRawTranslations = await pull.downloadTranslations(locales)
+    // const keysInPhrase = new Set<string>()
+    // Object.keys(localeToRawTranslations).forEach((locale) =>
+    //     Object.keys(JSON.parse(localeToRawTranslations[locale])).forEach((key) => keysInPhrase.add(key))
+    // )
+
+    // // use set for deduplication
+    // const keysInPhraseList = [...keysInPhrase]
+
+    // return keysInPhraseList
 }
 
 function applyTransform(input: string[], transformer: transformTranslationKeyFn) {
@@ -108,6 +117,7 @@ export default async function runExecutor(options: Partial<NonSensitiveArgs>, co
 
     // extract and prepare keys from phrase
     let phraseTranslationKeys
+    const phraseDuplicatedKeys = []
     {
         const phraseKeyFilter = options.phraseKeyFilter
             ? await loadTransformer<filterKeyFn>(resolve(context.root, options.phraseKeyFilter))
@@ -118,10 +128,31 @@ export default async function runExecutor(options: Partial<NonSensitiveArgs>, co
             : defaultTransformKeyFn
 
         const allPhraseTranslationKeys = await getKeysFromPhrase(config)
-        phraseTranslationKeys = filterList(allPhraseTranslationKeys, phraseKeyFilter)
 
-        if (allPhraseTranslationKeys.length !== phraseTranslationKeys.length) {
-            const bogusKeys = allPhraseTranslationKeys.filter((key) => !phraseTranslationKeys.includes(key))
+        // check for duplicate keys (keys are not unique in phrase, so this can happen when a key is renamed)
+        const deduplicatedPhraseTranslationKeys = allPhraseTranslationKeys.filter((key, index) => {
+            if (allPhraseTranslationKeys.indexOf(key) !== index) {
+                phraseDuplicatedKeys.push(key)
+                return false
+            } else {
+                return true
+            }
+        })
+        console.log(deduplicatedPhraseTranslationKeys)
+
+        // report duplicated keys
+        if (phraseDuplicatedKeys.length > 0) {
+            const phraseDuplicateKeysFilename = `${outputPath}/${projectName}.phrase-duplicate-keys.json`
+            writeFileSync(phraseDuplicateKeysFilename, JSON.stringify(phraseDuplicatedKeys, null, 2), {
+                flag: "w",
+            })
+            console.log(`Duplicated keys im phrase written to: ${phraseDuplicateKeysFilename}`)
+        }
+
+        phraseTranslationKeys = filterList(deduplicatedPhraseTranslationKeys, phraseKeyFilter)
+
+        if (deduplicatedPhraseTranslationKeys.length !== phraseTranslationKeys.length) {
+            const bogusKeys = deduplicatedPhraseTranslationKeys.filter((key) => !phraseTranslationKeys.includes(key))
             const phraseBugKeysFilename = `${outputPath}/${projectName}.filtered-phrase-keys.json`
             writeFileSync(phraseBugKeysFilename, JSON.stringify(bogusKeys, null, 2), {
                 flag: "w",

packages/nx-phrase/src/executors/pull/executor.spec.ts

@@ -4,8 +4,8 @@ import { ExecutorContext } from "@nrwl/devkit"
 import nock from "nock"
 
 import executor from "./executor"
-import { NonSensitiveArgs } from "../lib/types"
-import { PhraseClient } from "../lib/phrase"
+import { NonSensitiveArgs } from "../../lib/types"
+import { PhraseClient } from "../../lib/phrase"
 import { readFileSync } from "fs"
 
 const options: Partial<NonSensitiveArgs> = {
@@ -21,7 +21,7 @@ function nockForProject(projectId = "project_id") {
     nock("https://api.phrase.com/v2")
         .get(`/projects/${projectId}/locales`)
         .matchHeader("Authorization", /token .*/)
-        .query({ per_page: 20 })
+        .query({ per_page: 20, page: 1 })
         .replyWithFile(200, `${TEST_ASSETS_DIR}/localesListResponse.json`)
 
         .get(/\/projects\/[^/]+\/locales\/[^/]+\/download/)

packages/nx-phrase/src/executors/pull/executor.ts

@@ -1,8 +1,8 @@
 import { ExecutorContext } from "@nrwl/devkit"
 
-import { getConfig, InternalPhraseConfig } from "../lib/config"
-import { pull } from "../lib/pull"
-import { NonSensitiveArgs } from "../lib/types"
+import { getConfig, InternalPhraseConfig } from "../../lib/config"
+import { pull } from "../../lib/pull"
+import { NonSensitiveArgs } from "../../lib/types"
 
 export const pullRequiredConfigs = ["projectId", "output"]
 

packages/nx-phrase/src/executors/push/executor.spec.ts

@@ -4,7 +4,7 @@ import { ExecutorContext } from "@nrwl/devkit"
 import nock from "nock"
 
 import executor from "./executor"
-import { NonSensitiveArgs } from "../lib/types"
+import { NonSensitiveArgs } from "../../lib/types"
 
 const TEST_ASSETS_DIR = resolve(__dirname, "../../../test")
 

packages/nx-phrase/src/executors/push/executor.ts

@@ -1,8 +1,8 @@
 import { ExecutorContext } from "@nrwl/devkit"
 
-import { getConfig, InternalPhraseConfig } from "../lib/config"
-import { push } from "../lib/push"
-import { NonSensitiveArgs } from "../lib/types"
+import { getConfig, InternalPhraseConfig } from "../../lib/config"
+import { push } from "../../lib/push"
+import { NonSensitiveArgs } from "../../lib/types"
 
 export const pushRequiredConfigs = ["projectId", "uploadLanguageId"]
 

packages/nx-phrase/src/generators/configuration/generator.ts

@@ -8,7 +8,7 @@ import {
     readJson,
 } from "@nrwl/devkit"
 import { join } from "path"
-import { normalizeOptions, NPM_SCOPE } from "../../utils"
+import { normalizeOptions, NPM_SCOPE } from "../../lib/utils"
 import { NxPhraseGeneratorSchema } from "./schema"
 
 export default async function (tree: Tree, options: NxPhraseGeneratorSchema) {

packages/nx-phrase/src/executors/lib/config.ts → packages/nx-phrase/src/lib/config.ts

@@ -5,7 +5,7 @@ import { NonSensitiveArgs } from "./types"
 import { ExecutorContext } from "@nrwl/devkit"
 import { load } from "js-yaml"
 import { PhraseClientConfig } from "./phrase"
-import { BuildExecutorSchema } from "../build/schema"
+import { BuildExecutorSchema } from "../executors/build/schema"
 
 interface ConfigFileFormat {
     phrase: {

packages/nx-phrase/src/executors/lib/delete.ts → packages/nx-phrase/src/lib/delete.ts

@@ -12,7 +12,7 @@ export async function deleteKeys(file: string, config: InternalPhraseConfig): Pr
     const phrase = new PhraseClient(config.phraseClientConfig)
     for (const key of keysToDelete) {
         // resolve keyIds for key name
-        const keyIds = await phrase.resolveKeyIdByName({
+        const keyIds = await phrase.keysSearchByName({
             projectId: config.projectId,
             keyName: key,
             branch: config.branch,