@vocab/phrase - npm Package Compare versions

Comparing version 0.0.0-phrase-pull-dev-language-202281412540 to 0.0.0-push-split-translation-files-20230508031119

dist/declarations/src/file.d.ts

@@ -0,0 +0,0 @@ /// <reference types="node" />

dist/declarations/src/index.d.ts

 export { pull } from './pull-translations';
 export { push } from './push-translations';

dist/declarations/src/logger.d.ts

 import debug from 'debug';
 export declare const trace: debug.Debugger;
 export declare const log: (...params: unknown[]) => void;

dist/declarations/src/phrase-api.d.ts

@@ -1,7 +0,12 @@
-import { TranslationsByKey } from './../../types/src/index';
-import type { TranslationsByLanguage } from '@vocab/types';
+import type { TranslationsByLanguage } from '@vocab/core';
 import fetch from 'node-fetch';
-export declare function callPhrase(relativePath: string, options?: Parameters<typeof fetch>[1]): Promise<any>;
+export declare function callPhrase<T = any>(relativePath: string, options?: Parameters<typeof fetch>[1]): Promise<T>;
 export declare function pullAllTranslations(branch: string): Promise<TranslationsByLanguage>;
-export declare function pushTranslationsByLocale(contents: TranslationsByKey, locale: string, branch: string): Promise<void>;
+export declare function pushTranslations(translationsByLanguage: TranslationsByLanguage, { devLanguage, branch }: {
+    devLanguage: string;
+    branch: string;
+}): Promise<{
+    uploadIds: string[];
+}>;
+export declare function deleteUnusedKeys(uploadId: string, branch: string): Promise<void>;
 export declare function ensureBranch(branch: string): Promise<void>;

dist/declarations/src/pull-translations.d.ts

@@ -1,6 +0,7 @@
-import type { UserConfig } from '@vocab/types';
+import type { UserConfig } from '@vocab/core';
 interface PullOptions {
     branch?: string;
+    deleteUnusedKeys?: boolean;
 }
 export declare function pull({ branch }: PullOptions, config: UserConfig): Promise<void>;
 export {};

dist/declarations/src/push-translations.d.ts

@@ -1,9 +0,11 @@
-import { UserConfig } from '@vocab/types';
+import type { UserConfig } from '@vocab/core';
 interface PushOptions {
     branch: string;
+    deleteUnusedKeys?: boolean;
 }
 /**
- * Uploading to the Phrase API for each language. Adding a unique namespace to each key using file path they key came from
+ * Uploads translations to the Phrase API for each language.
+ * A unique namespace is appended to each key using the file path the key came from.
  */
-export declare function push({ branch }: PushOptions, config: UserConfig): Promise<void>;
+export declare function push({ branch, deleteUnusedKeys }: PushOptions, config: UserConfig): Promise<void>;
 export {};

dist/vocab-phrase.cjs.dev.js

@@ -12,2 +12,3 @@ 'use strict';
 var debug = require('debug');
+var sync = require('csv-stringify/sync');
 
@@ -25,16 +26,62 @@ function _interopDefault (e) { return e && e.__esModule ? e : { 'default': e }; }
 
-const trace = debug__default['default'](`vocab:phrase`);
+const trace = debug__default["default"](`vocab:phrase`);
 const log = (...params) => {
   // eslint-disable-next-line no-console
-  console.log(chalk__default['default'].yellow('Vocab'), ...params);
+  console.log(chalk__default["default"].yellow('Vocab'), ...params);
 };
 
+function translationsToCsv(translations, devLanguage) {
+  const languages = Object.keys(translations);
+  const altLanguages = languages.filter(language => language !== devLanguage);
+  // Ensure languages are ordered for locale mapping
+  // Might not need this anymore?
+  // const orderedLanguages = [devLanguage, ...altLanguages];
+
+  const devLanguageTranslations = translations[devLanguage];
+  const csvFilesByLanguage = Object.fromEntries(languages.map(language => [language, []]));
+  Object.entries(devLanguageTranslations).map(([key, {
+    message,
+    description,
+    tags
+  }]) => {
+    const sharedData = [key, description, tags === null || tags === void 0 ? void 0 : tags.join(',')];
+    const devLanguageRow = [...sharedData, message];
+    csvFilesByLanguage[devLanguage].push(devLanguageRow);
+    altLanguages.map(language => {
+      var _translations$languag, _translations$languag2;
+      const altTranslationMessage = (_translations$languag = translations[language]) === null || _translations$languag === void 0 ? void 0 : (_translations$languag2 = _translations$languag[key]) === null || _translations$languag2 === void 0 ? void 0 : _translations$languag2.message;
+      if (altTranslationMessage) {
+        csvFilesByLanguage[language].push([...sharedData, altTranslationMessage]);
+      }
+    });
+  });
+  const csvFilesWithKeys = Object.fromEntries(Object.entries(csvFilesByLanguage).filter(([_, csvFile]) => csvFile.length > 0));
+  const csvFileStrings = Object.fromEntries(Object.entries(csvFilesWithKeys).map(([language, csvFile]) => {
+    const csvFileString = sync.stringify(csvFile, {
+      delimiter: ',',
+      header: false
+    });
+    return [language, csvFileString];
+  }));
+
+  // Column indices start at 1
+  const keyIndex = 1;
+  const commentIndex = keyIndex + 1;
+  const tagColumn = commentIndex + 1;
+  return {
+    csvFileStrings,
+    keyIndex,
+    commentIndex,
+    tagColumn
+  };
+}
+
+/* eslint-disable no-console */
 function _callPhrase(path, options = {}) {
   const phraseApiToken = process.env.PHRASE_API_TOKEN;
-
   if (!phraseApiToken) {
     throw new Error('Missing PHRASE_API_TOKEN');
   }
-
-  return fetch__default['default'](path, { ...options,
+  return fetch__default["default"](path, {
+    ...options,
     headers: {
@@ -48,4 +95,6 @@ Authorization: `token ${phraseApiToken}`,
     console.log(`${path}: ${response.status} - ${response.statusText}`);
-    console.log(`Rate Limit: ${response.headers.get('X-Rate-Limit-Remaining')} of ${response.headers.get('X-Rate-Limit-Limit')} remaining. (${response.headers.get('X-Rate-Limit-Reset')} seconds remaining})`);
-    console.log('\nLink:', response.headers.get('Link'), '\n'); // Print All Headers:
+    const secondsUntilLimitReset = Math.ceil(Number.parseFloat(response.headers.get('X-Rate-Limit-Reset') || '0') - Date.now() / 1000);
+    console.log(`Rate Limit: ${response.headers.get('X-Rate-Limit-Remaining')} of ${response.headers.get('X-Rate-Limit-Limit')} remaining. (${secondsUntilLimitReset} seconds remaining)`);
+    trace('\nLink:', response.headers.get('Link'), '\n');
+    // Print All Headers:
     // console.log(Array.from(r.headers.entries()));
@@ -55,20 +104,14 @@
       var _response$headers$get;
-
       const result = await response.json();
-      console.log(`Internal Result (Length: ${result.length})\n`);
-
+      trace(`Internal Result (Length: ${result.length})\n`);
       if ((!options.method || options.method === 'GET') && (_response$headers$get = response.headers.get('Link')) !== null && _response$headers$get !== void 0 && _response$headers$get.includes('rel=next')) {
         var _response$headers$get2, _response$headers$get3;
-
         const [, nextPageUrl] = (_response$headers$get2 = (_response$headers$get3 = response.headers.get('Link')) === null || _response$headers$get3 === void 0 ? void 0 : _response$headers$get3.match(/<([^>]*)>; rel=next/)) !== null && _response$headers$get2 !== void 0 ? _response$headers$get2 : [];
-
         if (!nextPageUrl) {
-          throw new Error('Cant parse next page URL');
+          throw new Error("Can't parse next page URL");
         }
-
-        console.log('Results recieved with next page: ', nextPageUrl);
+        console.log('Results received with next page: ', nextPageUrl);
         const nextPageResult = await _callPhrase(nextPageUrl, options);
         return [...result, ...nextPageResult];
       }
-
       return result;
@@ -81,10 +124,7 @@ } catch (e) {
 }
-
 async function callPhrase(relativePath, options = {}) {
   const projectId = process.env.PHRASE_PROJECT_ID;
-
   if (!projectId) {
     throw new Error('Missing PHRASE_PROJECT_ID');
   }
-
   return _callPhrase(`https://api.phrase.com/v2/projects/${projectId}/${relativePath}`, options).then(result => {
@@ -94,3 +134,2 @@ if (Array.isArray(result)) {
     }
-
     return result;
@@ -105,3 +144,2 @@ }).catch(error => {
   const translations = {};
-
   for (const r of phraseResult) {
@@ -111,3 +149,2 @@ if (!translations[r.locale.code]) {
     }
-
     translations[r.locale.code][r.key.name] = {
@@ -117,22 +154,66 @@ message: r.content
   }
-
   return translations;
 }
-async function pushTranslationsByLocale(contents, locale, branch) {
-  const formData = new FormData__default['default']();
-  const fileContents = Buffer.from(JSON.stringify(contents));
-  formData.append('file', fileContents, {
-    contentType: 'application/json',
-    filename: `${locale}.json`
+async function pushTranslations(translationsByLanguage, {
+  devLanguage,
+  branch
+}) {
+  const {
+    csvFileStrings,
+    keyIndex,
+    commentIndex,
+    tagColumn
+  } = translationsToCsv(translationsByLanguage, devLanguage);
+  const uploadIds = [];
+  for (const [language, csvFileString] of Object.entries(csvFileStrings)) {
+    const formData = new FormData__default["default"]();
+    const fileContents = Buffer.from(csvFileString);
+    formData.append('file', fileContents, {
+      contentType: 'text/csv',
+      filename: 'translations.csv'
+    });
+    formData.append('file_format', 'csv');
+    formData.append('branch', branch);
+    formData.append('update_translations', 'true');
+    formData.append('update_descriptions', 'true');
+    formData.append('locale_id', language);
+    formData.append('format_options[key_index]', keyIndex);
+    formData.append('format_options[comment_index]', commentIndex);
+    formData.append('format_options[tag_column]', tagColumn);
+    formData.append('format_options[enable_pluralization]', 'false');
+    log(`Uploading translations for language ${language}`);
+    const result = await callPhrase(`uploads`, {
+      method: 'POST',
+      body: formData
+    });
+    trace('Upload result:\n', result);
+    if (result && 'id' in result) {
+      log('Upload ID:', result.id, '\n');
+      log('Successfully Uploaded\n');
+    } else {
+      log(`Error uploading: ${result === null || result === void 0 ? void 0 : result.message}\n`);
+      log('Response:', result);
+      throw new Error('Error uploading');
+    }
+    uploadIds.push(result.id);
+  }
+  return {
+    uploadIds
+  };
+}
+async function deleteUnusedKeys(uploadId, branch) {
+  const query = `unmentioned_in_upload:${uploadId}`;
+  const {
+    records_affected
+  } = await callPhrase('keys', {
+    method: 'DELETE',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      branch,
+      q: query
+    })
   });
-  formData.append('file_format', 'json');
-  formData.append('locale_id', locale);
-  formData.append('branch', branch);
-  formData.append('update_translations', 'true');
-  trace('Starting to upload:', locale);
-  await callPhrase(`uploads`, {
-    method: 'POST',
-    body: formData
-  });
-  log('Successfully Uploaded:', locale, '\n');
+  log('Successfully deleted', records_affected, 'unused keys from branch', branch);
 }
@@ -149,3 +230,3 @@ async function ensureBranch(branch) {
   });
-  trace('Created branch:', branch);
+  log('Created branch:', branch);
 }
@@ -162,30 +243,23 @@
   const phraseLanguages = Object.keys(allPhraseTranslations);
-  const phraseLanguagesWithTranslations = phraseLanguages.filter(language => {
-    const phraseTranslationsForLanguage = allPhraseTranslations[language];
-    return Object.keys(phraseTranslationsForLanguage).length > 0;
-  });
-  trace(`Found Phrase translations for languages ${phraseLanguagesWithTranslations.join(', ')}`);
-
-  if (!phraseLanguagesWithTranslations.includes(config.devLanguage)) {
-    throw new Error(`Phrase did not return any translations for dev language "${config.devLanguage}".\nEnsure you have configured your Phrase project for your dev language, and have pushed your translations.`);
+  trace(`Found Phrase translations for languages ${phraseLanguages.join(', ')}`);
+  if (!phraseLanguages.includes(config.devLanguage)) {
+    throw new Error(`Phrase did not return any translations for the configured development language "${config.devLanguage}".\nPlease ensure this language is present in your Phrase project's configuration.`);
   }
-
   const allVocabTranslations = await core.loadAllTranslations({
     fallbacks: 'none',
-    includeNodeModules: false
+    includeNodeModules: false,
+    withTags: true
   }, config);
-
   for (const loadedTranslation of allVocabTranslations) {
     const devTranslations = loadedTranslation.languages[config.devLanguage];
-
     if (!devTranslations) {
       throw new Error('No dev language translations loaded');
     }
-
-    const defaultValues = { ...devTranslations
+    const defaultValues = {
+      ...devTranslations
     };
     const localKeys = Object.keys(defaultValues);
-
     for (const key of localKeys) {
-      defaultValues[key] = { ...defaultValues[key],
+      defaultValues[key] = {
+        ...defaultValues[key],
         ...allPhraseTranslations[config.devLanguage][core.getUniqueKey(key, loadedTranslation.namespace)]
@@ -195,16 +269,17 @@ };
 
+    // Only write a `_meta` field if necessary
+    if (Object.keys(loadedTranslation.metadata).length > 0) {
+      defaultValues._meta = loadedTranslation.metadata;
+    }
     await writeFile(loadedTranslation.filePath, `${JSON.stringify(defaultValues, null, 2)}\n`);
-
     for (const alternativeLanguage of alternativeLanguages) {
       if (alternativeLanguage in allPhraseTranslations) {
-        const altTranslations = { ...loadedTranslation.languages[alternativeLanguage]
+        const altTranslations = {
+          ...loadedTranslation.languages[alternativeLanguage]
         };
         const phraseAltTranslations = allPhraseTranslations[alternativeLanguage];
-
         for (const key of localKeys) {
           var _phraseAltTranslation;
-
           const phraseKey = core.getUniqueKey(key, loadedTranslation.namespace);
           const phraseTranslationMessage = (_phraseAltTranslation = phraseAltTranslations[phraseKey]) === null || _phraseAltTranslation === void 0 ? void 0 : _phraseAltTranslation.message;
-
           if (!phraseTranslationMessage) {
@@ -214,10 +289,9 @@ trace(`Missing translation. No translation for key ${key} in phrase as ${phraseKey} in language ${alternativeLanguage}.`);
           }
-
-          altTranslations[key] = { ...altTranslations[key],
+          altTranslations[key] = {
+            ...altTranslations[key],
             message: phraseTranslationMessage
           };
         }
-
         const altTranslationFilePath = core.getAltLanguageFilePath(loadedTranslation.filePath, alternativeLanguage);
-        await mkdir(path__default['default'].dirname(altTranslationFilePath), {
+        await mkdir(path__default["default"].dirname(altTranslationFilePath), {
           recursive: true
@@ -232,10 +306,13 @@ });
 /**
- * Uploading to the Phrase API for each language. Adding a unique namespace to each key using file path they key came from
+ * Uploads translations to the Phrase API for each language.
+ * A unique namespace is appended to each key using the file path the key came from.
  */
 async function push({
-  branch
+  branch,
+  deleteUnusedKeys: deleteUnusedKeys$1
 }, config) {
   const allLanguageTranslations = await core.loadAllTranslations({
     fallbacks: 'none',
-    includeNodeModules: false
+    includeNodeModules: false,
+    withTags: true
   }, config);
@@ -247,25 +324,38 @@ trace(`Pushing translations to branch ${branch}`);
   const phraseTranslations = {};
-
   for (const loadedTranslation of allLanguageTranslations) {
     for (const language of allLanguages) {
       const localTranslations = loadedTranslation.languages[language];
-
       if (!localTranslations) {
         continue;
       }
-
       if (!phraseTranslations[language]) {
         phraseTranslations[language] = {};
       }
-
+      const {
+        metadata: {
+          tags: sharedTags = []
+        }
+      } = loadedTranslation;
       for (const localKey of Object.keys(localTranslations)) {
         const phraseKey = core.getUniqueKey(localKey, loadedTranslation.namespace);
-        phraseTranslations[language][phraseKey] = localTranslations[localKey];
+        const {
+          tags = [],
+          ...localTranslation
+        } = localTranslations[localKey];
+        if (language === config.devLanguage) {
+          localTranslation.tags = [...tags, ...sharedTags];
+        }
+        phraseTranslations[language][phraseKey] = localTranslation;
       }
     }
   }
-
-  for (const language of allLanguages) {
-    if (phraseTranslations[language]) {
-      await pushTranslationsByLocale(phraseTranslations[language], language, branch);
+  const {
+    uploadIds
+  } = await pushTranslations(phraseTranslations, {
+    devLanguage: config.devLanguage,
+    branch
+  });
+  if (deleteUnusedKeys$1) {
+    for (const uploadId of uploadIds) {
+      await deleteUnusedKeys(uploadId, branch);
     }
@@ -272,0 +362,0 @@ }

dist/vocab-phrase.cjs.prod.js

@@ -12,2 +12,3 @@ 'use strict';
 var debug = require('debug');
+var sync = require('csv-stringify/sync');
 
@@ -25,16 +26,62 @@ function _interopDefault (e) { return e && e.__esModule ? e : { 'default': e }; }
 
-const trace = debug__default['default'](`vocab:phrase`);
+const trace = debug__default["default"](`vocab:phrase`);
 const log = (...params) => {
   // eslint-disable-next-line no-console
-  console.log(chalk__default['default'].yellow('Vocab'), ...params);
+  console.log(chalk__default["default"].yellow('Vocab'), ...params);
 };
 
+function translationsToCsv(translations, devLanguage) {
+  const languages = Object.keys(translations);
+  const altLanguages = languages.filter(language => language !== devLanguage);
+  // Ensure languages are ordered for locale mapping
+  // Might not need this anymore?
+  // const orderedLanguages = [devLanguage, ...altLanguages];
+
+  const devLanguageTranslations = translations[devLanguage];
+  const csvFilesByLanguage = Object.fromEntries(languages.map(language => [language, []]));
+  Object.entries(devLanguageTranslations).map(([key, {
+    message,
+    description,
+    tags
+  }]) => {
+    const sharedData = [key, description, tags === null || tags === void 0 ? void 0 : tags.join(',')];
+    const devLanguageRow = [...sharedData, message];
+    csvFilesByLanguage[devLanguage].push(devLanguageRow);
+    altLanguages.map(language => {
+      var _translations$languag, _translations$languag2;
+      const altTranslationMessage = (_translations$languag = translations[language]) === null || _translations$languag === void 0 ? void 0 : (_translations$languag2 = _translations$languag[key]) === null || _translations$languag2 === void 0 ? void 0 : _translations$languag2.message;
+      if (altTranslationMessage) {
+        csvFilesByLanguage[language].push([...sharedData, altTranslationMessage]);
+      }
+    });
+  });
+  const csvFilesWithKeys = Object.fromEntries(Object.entries(csvFilesByLanguage).filter(([_, csvFile]) => csvFile.length > 0));
+  const csvFileStrings = Object.fromEntries(Object.entries(csvFilesWithKeys).map(([language, csvFile]) => {
+    const csvFileString = sync.stringify(csvFile, {
+      delimiter: ',',
+      header: false
+    });
+    return [language, csvFileString];
+  }));
+
+  // Column indices start at 1
+  const keyIndex = 1;
+  const commentIndex = keyIndex + 1;
+  const tagColumn = commentIndex + 1;
+  return {
+    csvFileStrings,
+    keyIndex,
+    commentIndex,
+    tagColumn
+  };
+}
+
+/* eslint-disable no-console */
 function _callPhrase(path, options = {}) {
   const phraseApiToken = process.env.PHRASE_API_TOKEN;
-
   if (!phraseApiToken) {
     throw new Error('Missing PHRASE_API_TOKEN');
   }
-
-  return fetch__default['default'](path, { ...options,
+  return fetch__default["default"](path, {
+    ...options,
     headers: {
@@ -48,4 +95,6 @@ Authorization: `token ${phraseApiToken}`,
     console.log(`${path}: ${response.status} - ${response.statusText}`);
-    console.log(`Rate Limit: ${response.headers.get('X-Rate-Limit-Remaining')} of ${response.headers.get('X-Rate-Limit-Limit')} remaining. (${response.headers.get('X-Rate-Limit-Reset')} seconds remaining})`);
-    console.log('\nLink:', response.headers.get('Link'), '\n'); // Print All Headers:
+    const secondsUntilLimitReset = Math.ceil(Number.parseFloat(response.headers.get('X-Rate-Limit-Reset') || '0') - Date.now() / 1000);
+    console.log(`Rate Limit: ${response.headers.get('X-Rate-Limit-Remaining')} of ${response.headers.get('X-Rate-Limit-Limit')} remaining. (${secondsUntilLimitReset} seconds remaining)`);
+    trace('\nLink:', response.headers.get('Link'), '\n');
+    // Print All Headers:
     // console.log(Array.from(r.headers.entries()));
@@ -55,20 +104,14 @@
       var _response$headers$get;
-
       const result = await response.json();
-      console.log(`Internal Result (Length: ${result.length})\n`);
-
+      trace(`Internal Result (Length: ${result.length})\n`);
       if ((!options.method || options.method === 'GET') && (_response$headers$get = response.headers.get('Link')) !== null && _response$headers$get !== void 0 && _response$headers$get.includes('rel=next')) {
         var _response$headers$get2, _response$headers$get3;
-
         const [, nextPageUrl] = (_response$headers$get2 = (_response$headers$get3 = response.headers.get('Link')) === null || _response$headers$get3 === void 0 ? void 0 : _response$headers$get3.match(/<([^>]*)>; rel=next/)) !== null && _response$headers$get2 !== void 0 ? _response$headers$get2 : [];
-
         if (!nextPageUrl) {
-          throw new Error('Cant parse next page URL');
+          throw new Error("Can't parse next page URL");
         }
-
-        console.log('Results recieved with next page: ', nextPageUrl);
+        console.log('Results received with next page: ', nextPageUrl);
         const nextPageResult = await _callPhrase(nextPageUrl, options);
         return [...result, ...nextPageResult];
       }
-
       return result;
@@ -81,10 +124,7 @@ } catch (e) {
 }
-
 async function callPhrase(relativePath, options = {}) {
   const projectId = process.env.PHRASE_PROJECT_ID;
-
   if (!projectId) {
     throw new Error('Missing PHRASE_PROJECT_ID');
   }
-
   return _callPhrase(`https://api.phrase.com/v2/projects/${projectId}/${relativePath}`, options).then(result => {
@@ -94,3 +134,2 @@ if (Array.isArray(result)) {
     }
-
     return result;
@@ -105,3 +144,2 @@ }).catch(error => {
   const translations = {};
-
   for (const r of phraseResult) {
@@ -111,3 +149,2 @@ if (!translations[r.locale.code]) {
     }
-
     translations[r.locale.code][r.key.name] = {
@@ -117,22 +154,66 @@ message: r.content
   }
-
   return translations;
 }
-async function pushTranslationsByLocale(contents, locale, branch) {
-  const formData = new FormData__default['default']();
-  const fileContents = Buffer.from(JSON.stringify(contents));
-  formData.append('file', fileContents, {
-    contentType: 'application/json',
-    filename: `${locale}.json`
+async function pushTranslations(translationsByLanguage, {
+  devLanguage,
+  branch
+}) {
+  const {
+    csvFileStrings,
+    keyIndex,
+    commentIndex,
+    tagColumn
+  } = translationsToCsv(translationsByLanguage, devLanguage);
+  const uploadIds = [];
+  for (const [language, csvFileString] of Object.entries(csvFileStrings)) {
+    const formData = new FormData__default["default"]();
+    const fileContents = Buffer.from(csvFileString);
+    formData.append('file', fileContents, {
+      contentType: 'text/csv',
+      filename: 'translations.csv'
+    });
+    formData.append('file_format', 'csv');
+    formData.append('branch', branch);
+    formData.append('update_translations', 'true');
+    formData.append('update_descriptions', 'true');
+    formData.append('locale_id', language);
+    formData.append('format_options[key_index]', keyIndex);
+    formData.append('format_options[comment_index]', commentIndex);
+    formData.append('format_options[tag_column]', tagColumn);
+    formData.append('format_options[enable_pluralization]', 'false');
+    log(`Uploading translations for language ${language}`);
+    const result = await callPhrase(`uploads`, {
+      method: 'POST',
+      body: formData
+    });
+    trace('Upload result:\n', result);
+    if (result && 'id' in result) {
+      log('Upload ID:', result.id, '\n');
+      log('Successfully Uploaded\n');
+    } else {
+      log(`Error uploading: ${result === null || result === void 0 ? void 0 : result.message}\n`);
+      log('Response:', result);
+      throw new Error('Error uploading');
+    }
+    uploadIds.push(result.id);
+  }
+  return {
+    uploadIds
+  };
+}
+async function deleteUnusedKeys(uploadId, branch) {
+  const query = `unmentioned_in_upload:${uploadId}`;
+  const {
+    records_affected
+  } = await callPhrase('keys', {
+    method: 'DELETE',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      branch,
+      q: query
+    })
   });
-  formData.append('file_format', 'json');
-  formData.append('locale_id', locale);
-  formData.append('branch', branch);
-  formData.append('update_translations', 'true');
-  trace('Starting to upload:', locale);
-  await callPhrase(`uploads`, {
-    method: 'POST',
-    body: formData
-  });
-  log('Successfully Uploaded:', locale, '\n');
+  log('Successfully deleted', records_affected, 'unused keys from branch', branch);
 }
@@ -149,3 +230,3 @@ async function ensureBranch(branch) {
   });
-  trace('Created branch:', branch);
+  log('Created branch:', branch);
 }
@@ -162,30 +243,23 @@
   const phraseLanguages = Object.keys(allPhraseTranslations);
-  const phraseLanguagesWithTranslations = phraseLanguages.filter(language => {
-    const phraseTranslationsForLanguage = allPhraseTranslations[language];
-    return Object.keys(phraseTranslationsForLanguage).length > 0;
-  });
-  trace(`Found Phrase translations for languages ${phraseLanguagesWithTranslations.join(', ')}`);
-
-  if (!phraseLanguagesWithTranslations.includes(config.devLanguage)) {
-    throw new Error(`Phrase did not return any translations for dev language "${config.devLanguage}".\nEnsure you have configured your Phrase project for your dev language, and have pushed your translations.`);
+  trace(`Found Phrase translations for languages ${phraseLanguages.join(', ')}`);
+  if (!phraseLanguages.includes(config.devLanguage)) {
+    throw new Error(`Phrase did not return any translations for the configured development language "${config.devLanguage}".\nPlease ensure this language is present in your Phrase project's configuration.`);
   }
-
   const allVocabTranslations = await core.loadAllTranslations({
     fallbacks: 'none',
-    includeNodeModules: false
+    includeNodeModules: false,
+    withTags: true
   }, config);
-
   for (const loadedTranslation of allVocabTranslations) {
     const devTranslations = loadedTranslation.languages[config.devLanguage];
-
     if (!devTranslations) {
       throw new Error('No dev language translations loaded');
     }
-
-    const defaultValues = { ...devTranslations
+    const defaultValues = {
+      ...devTranslations
     };
     const localKeys = Object.keys(defaultValues);
-
     for (const key of localKeys) {
-      defaultValues[key] = { ...defaultValues[key],
+      defaultValues[key] = {
+        ...defaultValues[key],
         ...allPhraseTranslations[config.devLanguage][core.getUniqueKey(key, loadedTranslation.namespace)]
@@ -195,16 +269,17 @@ };
 
+    // Only write a `_meta` field if necessary
+    if (Object.keys(loadedTranslation.metadata).length > 0) {
+      defaultValues._meta = loadedTranslation.metadata;
+    }
     await writeFile(loadedTranslation.filePath, `${JSON.stringify(defaultValues, null, 2)}\n`);
-
     for (const alternativeLanguage of alternativeLanguages) {
       if (alternativeLanguage in allPhraseTranslations) {
-        const altTranslations = { ...loadedTranslation.languages[alternativeLanguage]
+        const altTranslations = {
+          ...loadedTranslation.languages[alternativeLanguage]
         };
         const phraseAltTranslations = allPhraseTranslations[alternativeLanguage];
-
         for (const key of localKeys) {
           var _phraseAltTranslation;
-
           const phraseKey = core.getUniqueKey(key, loadedTranslation.namespace);
           const phraseTranslationMessage = (_phraseAltTranslation = phraseAltTranslations[phraseKey]) === null || _phraseAltTranslation === void 0 ? void 0 : _phraseAltTranslation.message;
-
           if (!phraseTranslationMessage) {
@@ -214,10 +289,9 @@ trace(`Missing translation. No translation for key ${key} in phrase as ${phraseKey} in language ${alternativeLanguage}.`);
           }
-
-          altTranslations[key] = { ...altTranslations[key],
+          altTranslations[key] = {
+            ...altTranslations[key],
             message: phraseTranslationMessage
           };
         }
-
         const altTranslationFilePath = core.getAltLanguageFilePath(loadedTranslation.filePath, alternativeLanguage);
-        await mkdir(path__default['default'].dirname(altTranslationFilePath), {
+        await mkdir(path__default["default"].dirname(altTranslationFilePath), {
           recursive: true
@@ -232,10 +306,13 @@ });
 /**
- * Uploading to the Phrase API for each language. Adding a unique namespace to each key using file path they key came from
+ * Uploads translations to the Phrase API for each language.
+ * A unique namespace is appended to each key using the file path the key came from.
  */
 async function push({
-  branch
+  branch,
+  deleteUnusedKeys: deleteUnusedKeys$1
 }, config) {
   const allLanguageTranslations = await core.loadAllTranslations({
     fallbacks: 'none',
-    includeNodeModules: false
+    includeNodeModules: false,
+    withTags: true
   }, config);
@@ -247,25 +324,38 @@ trace(`Pushing translations to branch ${branch}`);
   const phraseTranslations = {};
-
   for (const loadedTranslation of allLanguageTranslations) {
     for (const language of allLanguages) {
       const localTranslations = loadedTranslation.languages[language];
-
       if (!localTranslations) {
         continue;
       }
-
       if (!phraseTranslations[language]) {
         phraseTranslations[language] = {};
       }
-
+      const {
+        metadata: {
+          tags: sharedTags = []
+        }
+      } = loadedTranslation;
       for (const localKey of Object.keys(localTranslations)) {
         const phraseKey = core.getUniqueKey(localKey, loadedTranslation.namespace);
-        phraseTranslations[language][phraseKey] = localTranslations[localKey];
+        const {
+          tags = [],
+          ...localTranslation
+        } = localTranslations[localKey];
+        if (language === config.devLanguage) {
+          localTranslation.tags = [...tags, ...sharedTags];
+        }
+        phraseTranslations[language][phraseKey] = localTranslation;
       }
     }
   }
-
-  for (const language of allLanguages) {
-    if (phraseTranslations[language]) {
-      await pushTranslationsByLocale(phraseTranslations[language], language, branch);
+  const {
+    uploadIds
+  } = await pushTranslations(phraseTranslations, {
+    devLanguage: config.devLanguage,
+    branch
+  });
+  if (deleteUnusedKeys$1) {
+    for (const uploadId of uploadIds) {
+      await deleteUnusedKeys(uploadId, branch);
     }
@@ -272,0 +362,0 @@ }

dist/vocab-phrase.esm.js

@@ -8,2 +8,3 @@ import { promises } from 'fs';
 import debug from 'debug';
+import { stringify } from 'csv-stringify/sync';
 
@@ -19,10 +20,56 @@ const mkdir = promises.mkdir;
 
+function translationsToCsv(translations, devLanguage) {
+  const languages = Object.keys(translations);
+  const altLanguages = languages.filter(language => language !== devLanguage);
+  // Ensure languages are ordered for locale mapping
+  // Might not need this anymore?
+  // const orderedLanguages = [devLanguage, ...altLanguages];
+
+  const devLanguageTranslations = translations[devLanguage];
+  const csvFilesByLanguage = Object.fromEntries(languages.map(language => [language, []]));
+  Object.entries(devLanguageTranslations).map(([key, {
+    message,
+    description,
+    tags
+  }]) => {
+    const sharedData = [key, description, tags === null || tags === void 0 ? void 0 : tags.join(',')];
+    const devLanguageRow = [...sharedData, message];
+    csvFilesByLanguage[devLanguage].push(devLanguageRow);
+    altLanguages.map(language => {
+      var _translations$languag, _translations$languag2;
+      const altTranslationMessage = (_translations$languag = translations[language]) === null || _translations$languag === void 0 ? void 0 : (_translations$languag2 = _translations$languag[key]) === null || _translations$languag2 === void 0 ? void 0 : _translations$languag2.message;
+      if (altTranslationMessage) {
+        csvFilesByLanguage[language].push([...sharedData, altTranslationMessage]);
+      }
+    });
+  });
+  const csvFilesWithKeys = Object.fromEntries(Object.entries(csvFilesByLanguage).filter(([_, csvFile]) => csvFile.length > 0));
+  const csvFileStrings = Object.fromEntries(Object.entries(csvFilesWithKeys).map(([language, csvFile]) => {
+    const csvFileString = stringify(csvFile, {
+      delimiter: ',',
+      header: false
+    });
+    return [language, csvFileString];
+  }));
+
+  // Column indices start at 1
+  const keyIndex = 1;
+  const commentIndex = keyIndex + 1;
+  const tagColumn = commentIndex + 1;
+  return {
+    csvFileStrings,
+    keyIndex,
+    commentIndex,
+    tagColumn
+  };
+}
+
+/* eslint-disable no-console */
 function _callPhrase(path, options = {}) {
   const phraseApiToken = process.env.PHRASE_API_TOKEN;
-
   if (!phraseApiToken) {
     throw new Error('Missing PHRASE_API_TOKEN');
   }
-
-  return fetch(path, { ...options,
+  return fetch(path, {
+    ...options,
     headers: {
@@ -36,4 +83,6 @@ Authorization: `token ${phraseApiToken}`,
     console.log(`${path}: ${response.status} - ${response.statusText}`);
-    console.log(`Rate Limit: ${response.headers.get('X-Rate-Limit-Remaining')} of ${response.headers.get('X-Rate-Limit-Limit')} remaining. (${response.headers.get('X-Rate-Limit-Reset')} seconds remaining})`);
-    console.log('\nLink:', response.headers.get('Link'), '\n'); // Print All Headers:
+    const secondsUntilLimitReset = Math.ceil(Number.parseFloat(response.headers.get('X-Rate-Limit-Reset') || '0') - Date.now() / 1000);
+    console.log(`Rate Limit: ${response.headers.get('X-Rate-Limit-Remaining')} of ${response.headers.get('X-Rate-Limit-Limit')} remaining. (${secondsUntilLimitReset} seconds remaining)`);
+    trace('\nLink:', response.headers.get('Link'), '\n');
+    // Print All Headers:
     // console.log(Array.from(r.headers.entries()));
@@ -43,20 +92,14 @@
       var _response$headers$get;
-
       const result = await response.json();
-      console.log(`Internal Result (Length: ${result.length})\n`);
-
+      trace(`Internal Result (Length: ${result.length})\n`);
       if ((!options.method || options.method === 'GET') && (_response$headers$get = response.headers.get('Link')) !== null && _response$headers$get !== void 0 && _response$headers$get.includes('rel=next')) {
         var _response$headers$get2, _response$headers$get3;
-
         const [, nextPageUrl] = (_response$headers$get2 = (_response$headers$get3 = response.headers.get('Link')) === null || _response$headers$get3 === void 0 ? void 0 : _response$headers$get3.match(/<([^>]*)>; rel=next/)) !== null && _response$headers$get2 !== void 0 ? _response$headers$get2 : [];
-
         if (!nextPageUrl) {
-          throw new Error('Cant parse next page URL');
+          throw new Error("Can't parse next page URL");
         }
-
-        console.log('Results recieved with next page: ', nextPageUrl);
+        console.log('Results received with next page: ', nextPageUrl);
         const nextPageResult = await _callPhrase(nextPageUrl, options);
         return [...result, ...nextPageResult];
       }
-
       return result;
@@ -69,10 +112,7 @@ } catch (e) {
 }
-
 async function callPhrase(relativePath, options = {}) {
   const projectId = process.env.PHRASE_PROJECT_ID;
-
   if (!projectId) {
     throw new Error('Missing PHRASE_PROJECT_ID');
   }
-
   return _callPhrase(`https://api.phrase.com/v2/projects/${projectId}/${relativePath}`, options).then(result => {
@@ -82,3 +122,2 @@ if (Array.isArray(result)) {
     }
-
     return result;
@@ -93,3 +132,2 @@ }).catch(error => {
   const translations = {};
-
   for (const r of phraseResult) {
@@ -99,3 +137,2 @@ if (!translations[r.locale.code]) {
     }
-
     translations[r.locale.code][r.key.name] = {
@@ -105,22 +142,66 @@ message: r.content
   }
-
   return translations;
 }
-async function pushTranslationsByLocale(contents, locale, branch) {
-  const formData = new FormData();
-  const fileContents = Buffer.from(JSON.stringify(contents));
-  formData.append('file', fileContents, {
-    contentType: 'application/json',
-    filename: `${locale}.json`
+async function pushTranslations(translationsByLanguage, {
+  devLanguage,
+  branch
+}) {
+  const {
+    csvFileStrings,
+    keyIndex,
+    commentIndex,
+    tagColumn
+  } = translationsToCsv(translationsByLanguage, devLanguage);
+  const uploadIds = [];
+  for (const [language, csvFileString] of Object.entries(csvFileStrings)) {
+    const formData = new FormData();
+    const fileContents = Buffer.from(csvFileString);
+    formData.append('file', fileContents, {
+      contentType: 'text/csv',
+      filename: 'translations.csv'
+    });
+    formData.append('file_format', 'csv');
+    formData.append('branch', branch);
+    formData.append('update_translations', 'true');
+    formData.append('update_descriptions', 'true');
+    formData.append('locale_id', language);
+    formData.append('format_options[key_index]', keyIndex);
+    formData.append('format_options[comment_index]', commentIndex);
+    formData.append('format_options[tag_column]', tagColumn);
+    formData.append('format_options[enable_pluralization]', 'false');
+    log(`Uploading translations for language ${language}`);
+    const result = await callPhrase(`uploads`, {
+      method: 'POST',
+      body: formData
+    });
+    trace('Upload result:\n', result);
+    if (result && 'id' in result) {
+      log('Upload ID:', result.id, '\n');
+      log('Successfully Uploaded\n');
+    } else {
+      log(`Error uploading: ${result === null || result === void 0 ? void 0 : result.message}\n`);
+      log('Response:', result);
+      throw new Error('Error uploading');
+    }
+    uploadIds.push(result.id);
+  }
+  return {
+    uploadIds
+  };
+}
+async function deleteUnusedKeys(uploadId, branch) {
+  const query = `unmentioned_in_upload:${uploadId}`;
+  const {
+    records_affected
+  } = await callPhrase('keys', {
+    method: 'DELETE',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      branch,
+      q: query
+    })
   });
-  formData.append('file_format', 'json');
-  formData.append('locale_id', locale);
-  formData.append('branch', branch);
-  formData.append('update_translations', 'true');
-  trace('Starting to upload:', locale);
-  await callPhrase(`uploads`, {
-    method: 'POST',
-    body: formData
-  });
-  log('Successfully Uploaded:', locale, '\n');
+  log('Successfully deleted', records_affected, 'unused keys from branch', branch);
 }
@@ -137,3 +218,3 @@ async function ensureBranch(branch) {
   });
-  trace('Created branch:', branch);
+  log('Created branch:', branch);
 }
@@ -150,30 +231,23 @@
   const phraseLanguages = Object.keys(allPhraseTranslations);
-  const phraseLanguagesWithTranslations = phraseLanguages.filter(language => {
-    const phraseTranslationsForLanguage = allPhraseTranslations[language];
-    return Object.keys(phraseTranslationsForLanguage).length > 0;
-  });
-  trace(`Found Phrase translations for languages ${phraseLanguagesWithTranslations.join(', ')}`);
-
-  if (!phraseLanguagesWithTranslations.includes(config.devLanguage)) {
-    throw new Error(`Phrase did not return any translations for dev language "${config.devLanguage}".\nEnsure you have configured your Phrase project for your dev language, and have pushed your translations.`);
+  trace(`Found Phrase translations for languages ${phraseLanguages.join(', ')}`);
+  if (!phraseLanguages.includes(config.devLanguage)) {
+    throw new Error(`Phrase did not return any translations for the configured development language "${config.devLanguage}".\nPlease ensure this language is present in your Phrase project's configuration.`);
   }
-
   const allVocabTranslations = await loadAllTranslations({
     fallbacks: 'none',
-    includeNodeModules: false
+    includeNodeModules: false,
+    withTags: true
   }, config);
-
   for (const loadedTranslation of allVocabTranslations) {
     const devTranslations = loadedTranslation.languages[config.devLanguage];
-
     if (!devTranslations) {
       throw new Error('No dev language translations loaded');
     }
-
-    const defaultValues = { ...devTranslations
+    const defaultValues = {
+      ...devTranslations
     };
     const localKeys = Object.keys(defaultValues);
-
     for (const key of localKeys) {
-      defaultValues[key] = { ...defaultValues[key],
+      defaultValues[key] = {
+        ...defaultValues[key],
         ...allPhraseTranslations[config.devLanguage][getUniqueKey(key, loadedTranslation.namespace)]
@@ -183,16 +257,17 @@ };
 
+    // Only write a `_meta` field if necessary
+    if (Object.keys(loadedTranslation.metadata).length > 0) {
+      defaultValues._meta = loadedTranslation.metadata;
+    }
     await writeFile(loadedTranslation.filePath, `${JSON.stringify(defaultValues, null, 2)}\n`);
-
     for (const alternativeLanguage of alternativeLanguages) {
       if (alternativeLanguage in allPhraseTranslations) {
-        const altTranslations = { ...loadedTranslation.languages[alternativeLanguage]
+        const altTranslations = {
+          ...loadedTranslation.languages[alternativeLanguage]
         };
         const phraseAltTranslations = allPhraseTranslations[alternativeLanguage];
-
         for (const key of localKeys) {
           var _phraseAltTranslation;
-
           const phraseKey = getUniqueKey(key, loadedTranslation.namespace);
           const phraseTranslationMessage = (_phraseAltTranslation = phraseAltTranslations[phraseKey]) === null || _phraseAltTranslation === void 0 ? void 0 : _phraseAltTranslation.message;
-
           if (!phraseTranslationMessage) {
@@ -202,8 +277,7 @@ trace(`Missing translation. No translation for key ${key} in phrase as ${phraseKey} in language ${alternativeLanguage}.`);
           }
-
-          altTranslations[key] = { ...altTranslations[key],
+          altTranslations[key] = {
+            ...altTranslations[key],
             message: phraseTranslationMessage
           };
         }
-
         const altTranslationFilePath = getAltLanguageFilePath(loadedTranslation.filePath, alternativeLanguage);
@@ -220,10 +294,13 @@ await mkdir(path.dirname(altTranslationFilePath), {
 /**
- * Uploading to the Phrase API for each language. Adding a unique namespace to each key using file path they key came from
+ * Uploads translations to the Phrase API for each language.
+ * A unique namespace is appended to each key using the file path the key came from.
  */
 async function push({
-  branch
+  branch,
+  deleteUnusedKeys: deleteUnusedKeys$1
 }, config) {
   const allLanguageTranslations = await loadAllTranslations({
     fallbacks: 'none',
-    includeNodeModules: false
+    includeNodeModules: false,
+    withTags: true
   }, config);
@@ -235,25 +312,38 @@ trace(`Pushing translations to branch ${branch}`);
   const phraseTranslations = {};
-
   for (const loadedTranslation of allLanguageTranslations) {
     for (const language of allLanguages) {
       const localTranslations = loadedTranslation.languages[language];
-
       if (!localTranslations) {
         continue;
       }
-
       if (!phraseTranslations[language]) {
         phraseTranslations[language] = {};
       }
-
+      const {
+        metadata: {
+          tags: sharedTags = []
+        }
+      } = loadedTranslation;
       for (const localKey of Object.keys(localTranslations)) {
         const phraseKey = getUniqueKey(localKey, loadedTranslation.namespace);
-        phraseTranslations[language][phraseKey] = localTranslations[localKey];
+        const {
+          tags = [],
+          ...localTranslation
+        } = localTranslations[localKey];
+        if (language === config.devLanguage) {
+          localTranslation.tags = [...tags, ...sharedTags];
+        }
+        phraseTranslations[language][phraseKey] = localTranslation;
       }
     }
   }
-
-  for (const language of allLanguages) {
-    if (phraseTranslations[language]) {
-      await pushTranslationsByLocale(phraseTranslations[language], language, branch);
+  const {
+    uploadIds
+  } = await pushTranslations(phraseTranslations, {
+    devLanguage: config.devLanguage,
+    branch
+  });
+  if (deleteUnusedKeys$1) {
+    for (const uploadId of uploadIds) {
+      await deleteUnusedKeys(uploadId, branch);
     }
@@ -260,0 +350,0 @@ }

package.json

 {
   "name": "@vocab/phrase",
-  "version": "0.0.0-phrase-pull-dev-language-202281412540",
+  "version": "0.0.0-push-split-translation-files-20230508031119",
   "main": "dist/vocab-phrase.cjs.js",
@@ -9,5 +9,5 @@ "module": "dist/vocab-phrase.esm.js",
   "dependencies": {
-    "@vocab/core": "^1.0.0",
-    "@vocab/types": "^1.0.0",
+    "@vocab/core": "^1.3.1",
     "chalk": "^4.1.0",
+    "csv-stringify": "^6.2.3",
     "debug": "^4.3.1",
@@ -18,4 +18,8 @@ "form-data": "^3.0.0",
   "devDependencies": {
+    "@types/debug": "^4.1.5",
     "@types/node-fetch": "^2.5.7"
-  }
-}
+  },
+  "files": [
+    "dist"
+  ]
+}

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Worsened metrics

Total package byte prevSize

43336

decreased by-22.02%

Dev dependency count

2

increased by100%

Number of package files

14

decreased by-33.33%

Lines of code

1057

decreased by-16.57%

Number of medium supply chain risk alerts

4

increased by33.33%

Published versions last two months

0

decreased by-100%

Dependency changes

• + Addedcsv-stringify@^6.2.3

• + Addedcsv-stringify@6.5.2(transitive)

• - Removed@vocab/types@^1.0.0

• - Removed@vocab/types@1.3.7(transitive)

• Updated@vocab/core@^1.3.1