Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More
Socket
Sign inDemoInstall
Socket

@vocab/phrase

Package Overview
Dependencies
Maintainers
0
Versions
46
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@vocab/phrase - npm Package Compare versions

Comparing version 0.0.0-delete-unused-keys-20228144520 to 0.0.0-feature-ignore-flag-push-20241014224750

LICENSE

4

dist/declarations/src/index.d.ts

@@ -1,2 +0,2 @@

export { pull } from './pull-translations';
export { push } from './push-translations';
export { pull } from "./pull-translations.js";
export { push } from "./push-translations.js";

@@ -1,7 +0,8 @@

import type { UserConfig } from '@vocab/types';
import { type UserConfig } from '@vocab/core';
interface PullOptions {
branch?: string;
deleteUnusedKeys?: boolean;
errorOnNoGlobalKeyTranslation?: boolean;
}
export declare function pull({ branch }: PullOptions, config: UserConfig): Promise<void>;
export declare function pull({ branch, errorOnNoGlobalKeyTranslation }: PullOptions, config: UserConfig): Promise<void>;
export {};

@@ -1,10 +0,12 @@

import { UserConfig } from '@vocab/types';
import { type UserConfig } from '@vocab/core';
interface PushOptions {
branch: string;
deleteUnusedKeys?: boolean;
ignore?: string[];
}
/**
* Uploading to the Phrase API for each language. Adding a unique namespace to each key using file path they key came from
* Uploads translations to the Phrase API for each language.
* A unique namespace is appended to each key using the file path the key came from.
*/
export declare function push({ branch, deleteUnusedKeys }: PushOptions, config: UserConfig): Promise<void>;
export declare function push({ branch, deleteUnusedKeys, ignore }: PushOptions, config: UserConfig): Promise<void>;
export {};
export * from "./declarations/src/index";
//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidm9jYWItcGhyYXNlLmNqcy5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi9kZWNsYXJhdGlvbnMvc3JjL2luZGV4LmQudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEifQ==

@@ -8,6 +8,5 @@ 'use strict';

var core = require('@vocab/core');
var FormData = require('form-data');
var fetch = require('node-fetch');
var chalk = require('chalk');
var pc = require('picocolors');
var debug = require('debug');
var sync = require('csv-stringify/sync');

@@ -17,5 +16,3 @@ function _interopDefault (e) { return e && e.__esModule ? e : { 'default': e }; }

var path__default = /*#__PURE__*/_interopDefault(path);
var FormData__default = /*#__PURE__*/_interopDefault(FormData);
var fetch__default = /*#__PURE__*/_interopDefault(fetch);
var chalk__default = /*#__PURE__*/_interopDefault(chalk);
var pc__default = /*#__PURE__*/_interopDefault(pc);
var debug__default = /*#__PURE__*/_interopDefault(debug);

@@ -26,20 +23,65 @@

const trace = debug__default['default'](`vocab:phrase`);
const trace = debug__default["default"](`vocab:phrase`);
const log = (...params) => {
// eslint-disable-next-line no-console
console.log(chalk__default['default'].yellow('Vocab'), ...params);
console.log(pc__default["default"].yellow('Vocab'), ...params);
};
function translationsToCsv(translations, devLanguage) {
const languages = Object.keys(translations);
const altLanguages = languages.filter(language => language !== devLanguage);
const devLanguageTranslations = translations[devLanguage];
const csvFilesByLanguage = Object.fromEntries(languages.map(language => [language, []]));
Object.entries(devLanguageTranslations).map(([key, {
message,
description,
tags
}]) => {
const sharedData = [key, description, tags === null || tags === void 0 ? void 0 : tags.join(',')];
const devLanguageRow = [...sharedData, message];
csvFilesByLanguage[devLanguage].push(devLanguageRow);
altLanguages.map(language => {
var _translations$languag;
const altTranslationMessage = (_translations$languag = translations[language]) === null || _translations$languag === void 0 || (_translations$languag = _translations$languag[key]) === null || _translations$languag === void 0 ? void 0 : _translations$languag.message;
if (altTranslationMessage) {
csvFilesByLanguage[language].push([...sharedData, altTranslationMessage]);
}
});
});
const csvFileStrings = Object.fromEntries(Object.entries(csvFilesByLanguage)
// Ensure CSV files are only created if the language has at least 1 translation
.filter(([_, csvFile]) => csvFile.length > 0).map(([language, csvFile]) => {
const csvFileString = sync.stringify(csvFile, {
delimiter: ',',
header: false
});
return [language, csvFileString];
}));
// Column indices start at 1
const keyIndex = 1;
const commentIndex = keyIndex + 1;
const tagColumn = commentIndex + 1;
const messageIndex = tagColumn + 1;
return {
csvFileStrings,
keyIndex,
messageIndex,
commentIndex,
tagColumn
};
}
/* eslint-disable no-console */
function _callPhrase(path, options = {}) {
const phraseApiToken = process.env.PHRASE_API_TOKEN;
if (!phraseApiToken) {
throw new Error('Missing PHRASE_API_TOKEN');
}
return fetch__default['default'](path, { ...options,
return fetch(path, {
...options,
headers: {
Authorization: `token ${phraseApiToken}`,
// Provide identification via User Agent as requested in https://developers.phrase.com/api/#overview--identification-via-user-agent
'User-Agent': 'SEEK Demo Candidate App (jhope@seek.com.au)',
'User-Agent': 'Vocab Client (https://github.com/seek-oss/vocab)',
...options.headers

@@ -49,4 +91,6 @@ }

console.log(`${path}: ${response.status} - ${response.statusText}`);
console.log(`Rate Limit: ${response.headers.get('X-Rate-Limit-Remaining')} of ${response.headers.get('X-Rate-Limit-Limit')} remaining. (${response.headers.get('X-Rate-Limit-Reset')} seconds remaining})`);
console.log('\nLink:', response.headers.get('Link'), '\n'); // Print All Headers:
const secondsUntilLimitReset = Math.ceil(Number.parseFloat(response.headers.get('X-Rate-Limit-Reset') || '0') - Date.now() / 1000);
console.log(`Rate Limit: ${response.headers.get('X-Rate-Limit-Remaining')} of ${response.headers.get('X-Rate-Limit-Limit')} remaining. (${secondsUntilLimitReset} seconds remaining)`);
trace('\nLink:', response.headers.get('Link'), '\n');
// Print All Headers:
// console.log(Array.from(r.headers.entries()));

@@ -56,20 +100,14 @@

var _response$headers$get;
const result = await response.json();
console.log(`Internal Result (Length: ${result.length})\n`);
trace(`Internal Result (Length: ${result.length})\n`);
if ((!options.method || options.method === 'GET') && (_response$headers$get = response.headers.get('Link')) !== null && _response$headers$get !== void 0 && _response$headers$get.includes('rel=next')) {
var _response$headers$get2, _response$headers$get3;
const [, nextPageUrl] = (_response$headers$get2 = (_response$headers$get3 = response.headers.get('Link')) === null || _response$headers$get3 === void 0 ? void 0 : _response$headers$get3.match(/<([^>]*)>; rel=next/)) !== null && _response$headers$get2 !== void 0 ? _response$headers$get2 : [];
if (!nextPageUrl) {
throw new Error('Cant parse next page URL');
throw new Error("Can't parse next page URL");
}
console.log('Results recieved with next page: ', nextPageUrl);
console.log('Results received with next page: ', nextPageUrl);
const nextPageResult = await _callPhrase(nextPageUrl, options);
return [...result, ...nextPageResult];
}
return result;

@@ -82,10 +120,7 @@ } catch (e) {

}
async function callPhrase(relativePath, options = {}) {
const projectId = process.env.PHRASE_PROJECT_ID;
if (!projectId) {
throw new Error('Missing PHRASE_PROJECT_ID');
}
return _callPhrase(`https://api.phrase.com/v2/projects/${projectId}/${relativePath}`, options).then(result => {

@@ -95,3 +130,2 @@ if (Array.isArray(result)) {

}
return result;

@@ -106,3 +140,2 @@ }).catch(error => {

const translations = {};
for (const r of phraseResult) {

@@ -112,3 +145,2 @@ if (!translations[r.locale.code]) {

}
translations[r.locale.code][r.key.name] = {

@@ -118,32 +150,57 @@ message: r.content

}
return translations;
}
async function pushTranslationsByLocale(contents, locale, branch) {
const formData = new FormData__default['default']();
const fileContents = Buffer.from(JSON.stringify(contents));
formData.append('file', fileContents, {
contentType: 'application/json',
filename: `${locale}.json`
});
formData.append('file_format', 'json');
formData.append('locale_id', locale);
formData.append('branch', branch);
formData.append('update_translations', 'true');
trace('Starting to upload:', locale);
async function pushTranslations(translationsByLanguage, {
devLanguage,
branch
}) {
const {
id
} = await callPhrase(`uploads`, {
method: 'POST',
body: formData
});
log('Upload ID:', id, '\n');
log('Successfully Uploaded:', locale, '\n');
csvFileStrings,
keyIndex,
commentIndex,
tagColumn,
messageIndex
} = translationsToCsv(translationsByLanguage, devLanguage);
let devLanguageUploadId = '';
for (const [language, csvFileString] of Object.entries(csvFileStrings)) {
const formData = new FormData();
formData.append('file', new Blob([csvFileString], {
type: 'text/csv'
}), `${language}.translations.csv`);
formData.append('file_format', 'csv');
formData.append('branch', branch);
formData.append('update_translations', 'true');
formData.append('update_descriptions', 'true');
formData.append(`locale_mapping[${language}]`, messageIndex.toString());
formData.append('format_options[key_index]', keyIndex.toString());
formData.append('format_options[comment_index]', commentIndex.toString());
formData.append('format_options[tag_column]', tagColumn.toString());
formData.append('format_options[enable_pluralization]', 'false');
log(`Uploading translations for language ${language}`);
const result = await callPhrase(`uploads`, {
method: 'POST',
body: formData
});
trace('Upload result:\n', result);
if (result && 'id' in result) {
log('Upload ID:', result.id, '\n');
log('Successfully Uploaded\n');
} else {
log(`Error uploading: ${result === null || result === void 0 ? void 0 : result.message}\n`);
log('Response:', result);
throw new Error('Error uploading');
}
if (language === devLanguage) {
devLanguageUploadId = result.id;
}
}
return {
uploadId: id
devLanguageUploadId
};
}
async function deleteUnusedKeys(uploadId, locale, branch) {
async function deleteUnusedKeys(uploadId, branch) {
const query = `unmentioned_in_upload:${uploadId}`;
const result = await callPhrase('keys', {
const {
records_affected
} = await callPhrase('keys', {
method: 'DELETE',

@@ -155,7 +212,6 @@ headers: {

branch,
locale_id: locale,
q: query
})
});
log('Successfully deleted', result.records_affected, 'unused keys from branch', branch);
log('Successfully deleted', records_affected, 'unused keys from branch', branch);
}

@@ -172,7 +228,8 @@ async function ensureBranch(branch) {

});
trace('Created branch:', branch);
log('Created branch:', branch);
}
async function pull({
branch = 'local-development'
branch = 'local-development',
errorOnNoGlobalKeyTranslation
}, config) {

@@ -184,50 +241,58 @@ trace(`Pulling translations from branch ${branch}`);

trace(`Pulling translations from Phrase for languages ${config.devLanguage} and ${alternativeLanguages.join(', ')}`);
const phraseLanguages = Object.keys(allPhraseTranslations);
trace(`Found Phrase translations for languages ${phraseLanguages.join(', ')}`);
if (!phraseLanguages.includes(config.devLanguage)) {
throw new Error(`Phrase did not return any translations for the configured development language "${config.devLanguage}".\nPlease ensure this language is present in your Phrase project's configuration.`);
}
const allVocabTranslations = await core.loadAllTranslations({
fallbacks: 'none',
includeNodeModules: false
includeNodeModules: false,
withTags: true
}, config);
for (const loadedTranslation of allVocabTranslations) {
const devTranslations = loadedTranslation.languages[config.devLanguage];
if (!devTranslations) {
throw new Error('No dev language translations loaded');
}
const defaultValues = { ...devTranslations
const defaultValues = {
...devTranslations
};
const localKeys = Object.keys(defaultValues);
for (const key of localKeys) {
defaultValues[key] = { ...defaultValues[key],
...allPhraseTranslations[config.devLanguage][core.getUniqueKey(key, loadedTranslation.namespace)]
var _defaultValues$key$gl;
defaultValues[key] = {
...defaultValues[key],
...allPhraseTranslations[config.devLanguage][(_defaultValues$key$gl = defaultValues[key].globalKey) !== null && _defaultValues$key$gl !== void 0 ? _defaultValues$key$gl : core.getUniqueKey(key, loadedTranslation.namespace)]
};
}
// Only write a `_meta` field if necessary
if (Object.keys(loadedTranslation.metadata).length > 0) {
defaultValues._meta = loadedTranslation.metadata;
}
await writeFile(loadedTranslation.filePath, `${JSON.stringify(defaultValues, null, 2)}\n`);
for (const alternativeLanguage of alternativeLanguages) {
if (alternativeLanguage in allPhraseTranslations) {
const altTranslations = { ...loadedTranslation.languages[alternativeLanguage]
const altTranslations = {
...loadedTranslation.languages[alternativeLanguage]
};
const phraseAltTranslations = allPhraseTranslations[alternativeLanguage];
for (const key of localKeys) {
var _phraseAltTranslation;
const phraseKey = core.getUniqueKey(key, loadedTranslation.namespace);
var _defaultValues$key$gl2, _phraseAltTranslation;
const phraseKey = (_defaultValues$key$gl2 = defaultValues[key].globalKey) !== null && _defaultValues$key$gl2 !== void 0 ? _defaultValues$key$gl2 : core.getUniqueKey(key, loadedTranslation.namespace);
const phraseTranslationMessage = (_phraseAltTranslation = phraseAltTranslations[phraseKey]) === null || _phraseAltTranslation === void 0 ? void 0 : _phraseAltTranslation.message;
if (!phraseTranslationMessage) {
trace(`Missing translation. No translation for key ${key} in phrase as ${phraseKey} in language ${alternativeLanguage}.`);
if (errorOnNoGlobalKeyTranslation && defaultValues[key].globalKey) {
throw new Error(`Missing translation for global key ${key} in language ${alternativeLanguage}`);
}
continue;
}
altTranslations[key] = { ...altTranslations[key],
altTranslations[key] = {
...altTranslations[key],
message: phraseTranslationMessage
};
}
const altTranslationFilePath = core.getAltLanguageFilePath(loadedTranslation.filePath, alternativeLanguage);
await mkdir(path__default['default'].dirname(altTranslationFilePath), {
await mkdir(path__default["default"].dirname(altTranslationFilePath), {
recursive: true

@@ -242,12 +307,18 @@ });

/**
* Uploading to the Phrase API for each language. Adding a unique namespace to each key using file path they key came from
* Uploads translations to the Phrase API for each language.
* A unique namespace is appended to each key using the file path the key came from.
*/
async function push({
branch,
deleteUnusedKeys: deleteUnusedKeys$1
deleteUnusedKeys: deleteUnusedKeys$1,
ignore
}, config) {
const allLanguageTranslations = await core.loadAllTranslations({
fallbacks: 'none',
includeNodeModules: false
}, config);
includeNodeModules: false,
withTags: true
}, {
...config,
ignore: [...(config.ignore || []), ...(ignore || [])]
});
trace(`Pushing translations to branch ${branch}`);

@@ -258,32 +329,38 @@ const allLanguages = config.languages.map(v => v.name);

const phraseTranslations = {};
for (const loadedTranslation of allLanguageTranslations) {
for (const language of allLanguages) {
const localTranslations = loadedTranslation.languages[language];
if (!localTranslations) {
continue;
}
if (!phraseTranslations[language]) {
phraseTranslations[language] = {};
}
const {
metadata: {
tags: sharedTags = []
}
} = loadedTranslation;
for (const localKey of Object.keys(localTranslations)) {
const phraseKey = core.getUniqueKey(localKey, loadedTranslation.namespace);
phraseTranslations[language][phraseKey] = localTranslations[localKey];
const {
tags = [],
...localTranslation
} = localTranslations[localKey];
if (language === config.devLanguage) {
localTranslation.tags = [...tags, ...sharedTags];
}
const globalKey = loadedTranslation.languages[config.devLanguage][localKey].globalKey;
const phraseKey = globalKey !== null && globalKey !== void 0 ? globalKey : core.getUniqueKey(localKey, loadedTranslation.namespace);
phraseTranslations[language][phraseKey] = localTranslation;
}
}
}
for (const language of allLanguages) {
if (phraseTranslations[language]) {
const {
uploadId
} = await pushTranslationsByLocale(phraseTranslations[language], language, branch);
if (deleteUnusedKeys$1) {
await deleteUnusedKeys(uploadId, language, branch);
}
}
const {
devLanguageUploadId
} = await pushTranslations(phraseTranslations, {
devLanguage: config.devLanguage,
branch
});
if (deleteUnusedKeys$1) {
await deleteUnusedKeys(devLanguageUploadId, branch);
}

@@ -290,0 +367,0 @@ }

@@ -8,6 +8,5 @@ 'use strict';

var core = require('@vocab/core');
var FormData = require('form-data');
var fetch = require('node-fetch');
var chalk = require('chalk');
var pc = require('picocolors');
var debug = require('debug');
var sync = require('csv-stringify/sync');

@@ -17,5 +16,3 @@ function _interopDefault (e) { return e && e.__esModule ? e : { 'default': e }; }

var path__default = /*#__PURE__*/_interopDefault(path);
var FormData__default = /*#__PURE__*/_interopDefault(FormData);
var fetch__default = /*#__PURE__*/_interopDefault(fetch);
var chalk__default = /*#__PURE__*/_interopDefault(chalk);
var pc__default = /*#__PURE__*/_interopDefault(pc);
var debug__default = /*#__PURE__*/_interopDefault(debug);

@@ -26,20 +23,65 @@

const trace = debug__default['default'](`vocab:phrase`);
const trace = debug__default["default"](`vocab:phrase`);
const log = (...params) => {
// eslint-disable-next-line no-console
console.log(chalk__default['default'].yellow('Vocab'), ...params);
console.log(pc__default["default"].yellow('Vocab'), ...params);
};
function translationsToCsv(translations, devLanguage) {
const languages = Object.keys(translations);
const altLanguages = languages.filter(language => language !== devLanguage);
const devLanguageTranslations = translations[devLanguage];
const csvFilesByLanguage = Object.fromEntries(languages.map(language => [language, []]));
Object.entries(devLanguageTranslations).map(([key, {
message,
description,
tags
}]) => {
const sharedData = [key, description, tags === null || tags === void 0 ? void 0 : tags.join(',')];
const devLanguageRow = [...sharedData, message];
csvFilesByLanguage[devLanguage].push(devLanguageRow);
altLanguages.map(language => {
var _translations$languag;
const altTranslationMessage = (_translations$languag = translations[language]) === null || _translations$languag === void 0 || (_translations$languag = _translations$languag[key]) === null || _translations$languag === void 0 ? void 0 : _translations$languag.message;
if (altTranslationMessage) {
csvFilesByLanguage[language].push([...sharedData, altTranslationMessage]);
}
});
});
const csvFileStrings = Object.fromEntries(Object.entries(csvFilesByLanguage)
// Ensure CSV files are only created if the language has at least 1 translation
.filter(([_, csvFile]) => csvFile.length > 0).map(([language, csvFile]) => {
const csvFileString = sync.stringify(csvFile, {
delimiter: ',',
header: false
});
return [language, csvFileString];
}));
// Column indices start at 1
const keyIndex = 1;
const commentIndex = keyIndex + 1;
const tagColumn = commentIndex + 1;
const messageIndex = tagColumn + 1;
return {
csvFileStrings,
keyIndex,
messageIndex,
commentIndex,
tagColumn
};
}
/* eslint-disable no-console */
function _callPhrase(path, options = {}) {
const phraseApiToken = process.env.PHRASE_API_TOKEN;
if (!phraseApiToken) {
throw new Error('Missing PHRASE_API_TOKEN');
}
return fetch__default['default'](path, { ...options,
return fetch(path, {
...options,
headers: {
Authorization: `token ${phraseApiToken}`,
// Provide identification via User Agent as requested in https://developers.phrase.com/api/#overview--identification-via-user-agent
'User-Agent': 'SEEK Demo Candidate App (jhope@seek.com.au)',
'User-Agent': 'Vocab Client (https://github.com/seek-oss/vocab)',
...options.headers

@@ -49,4 +91,6 @@ }

console.log(`${path}: ${response.status} - ${response.statusText}`);
console.log(`Rate Limit: ${response.headers.get('X-Rate-Limit-Remaining')} of ${response.headers.get('X-Rate-Limit-Limit')} remaining. (${response.headers.get('X-Rate-Limit-Reset')} seconds remaining})`);
console.log('\nLink:', response.headers.get('Link'), '\n'); // Print All Headers:
const secondsUntilLimitReset = Math.ceil(Number.parseFloat(response.headers.get('X-Rate-Limit-Reset') || '0') - Date.now() / 1000);
console.log(`Rate Limit: ${response.headers.get('X-Rate-Limit-Remaining')} of ${response.headers.get('X-Rate-Limit-Limit')} remaining. (${secondsUntilLimitReset} seconds remaining)`);
trace('\nLink:', response.headers.get('Link'), '\n');
// Print All Headers:
// console.log(Array.from(r.headers.entries()));

@@ -56,20 +100,14 @@

var _response$headers$get;
const result = await response.json();
console.log(`Internal Result (Length: ${result.length})\n`);
trace(`Internal Result (Length: ${result.length})\n`);
if ((!options.method || options.method === 'GET') && (_response$headers$get = response.headers.get('Link')) !== null && _response$headers$get !== void 0 && _response$headers$get.includes('rel=next')) {
var _response$headers$get2, _response$headers$get3;
const [, nextPageUrl] = (_response$headers$get2 = (_response$headers$get3 = response.headers.get('Link')) === null || _response$headers$get3 === void 0 ? void 0 : _response$headers$get3.match(/<([^>]*)>; rel=next/)) !== null && _response$headers$get2 !== void 0 ? _response$headers$get2 : [];
if (!nextPageUrl) {
throw new Error('Cant parse next page URL');
throw new Error("Can't parse next page URL");
}
console.log('Results recieved with next page: ', nextPageUrl);
console.log('Results received with next page: ', nextPageUrl);
const nextPageResult = await _callPhrase(nextPageUrl, options);
return [...result, ...nextPageResult];
}
return result;

@@ -82,10 +120,7 @@ } catch (e) {

}
async function callPhrase(relativePath, options = {}) {
const projectId = process.env.PHRASE_PROJECT_ID;
if (!projectId) {
throw new Error('Missing PHRASE_PROJECT_ID');
}
return _callPhrase(`https://api.phrase.com/v2/projects/${projectId}/${relativePath}`, options).then(result => {

@@ -95,3 +130,2 @@ if (Array.isArray(result)) {

}
return result;

@@ -106,3 +140,2 @@ }).catch(error => {

const translations = {};
for (const r of phraseResult) {

@@ -112,3 +145,2 @@ if (!translations[r.locale.code]) {

}
translations[r.locale.code][r.key.name] = {

@@ -118,32 +150,57 @@ message: r.content

}
return translations;
}
async function pushTranslationsByLocale(contents, locale, branch) {
const formData = new FormData__default['default']();
const fileContents = Buffer.from(JSON.stringify(contents));
formData.append('file', fileContents, {
contentType: 'application/json',
filename: `${locale}.json`
});
formData.append('file_format', 'json');
formData.append('locale_id', locale);
formData.append('branch', branch);
formData.append('update_translations', 'true');
trace('Starting to upload:', locale);
async function pushTranslations(translationsByLanguage, {
devLanguage,
branch
}) {
const {
id
} = await callPhrase(`uploads`, {
method: 'POST',
body: formData
});
log('Upload ID:', id, '\n');
log('Successfully Uploaded:', locale, '\n');
csvFileStrings,
keyIndex,
commentIndex,
tagColumn,
messageIndex
} = translationsToCsv(translationsByLanguage, devLanguage);
let devLanguageUploadId = '';
for (const [language, csvFileString] of Object.entries(csvFileStrings)) {
const formData = new FormData();
formData.append('file', new Blob([csvFileString], {
type: 'text/csv'
}), `${language}.translations.csv`);
formData.append('file_format', 'csv');
formData.append('branch', branch);
formData.append('update_translations', 'true');
formData.append('update_descriptions', 'true');
formData.append(`locale_mapping[${language}]`, messageIndex.toString());
formData.append('format_options[key_index]', keyIndex.toString());
formData.append('format_options[comment_index]', commentIndex.toString());
formData.append('format_options[tag_column]', tagColumn.toString());
formData.append('format_options[enable_pluralization]', 'false');
log(`Uploading translations for language ${language}`);
const result = await callPhrase(`uploads`, {
method: 'POST',
body: formData
});
trace('Upload result:\n', result);
if (result && 'id' in result) {
log('Upload ID:', result.id, '\n');
log('Successfully Uploaded\n');
} else {
log(`Error uploading: ${result === null || result === void 0 ? void 0 : result.message}\n`);
log('Response:', result);
throw new Error('Error uploading');
}
if (language === devLanguage) {
devLanguageUploadId = result.id;
}
}
return {
uploadId: id
devLanguageUploadId
};
}
async function deleteUnusedKeys(uploadId, locale, branch) {
async function deleteUnusedKeys(uploadId, branch) {
const query = `unmentioned_in_upload:${uploadId}`;
const result = await callPhrase('keys', {
const {
records_affected
} = await callPhrase('keys', {
method: 'DELETE',

@@ -155,7 +212,6 @@ headers: {

branch,
locale_id: locale,
q: query
})
});
log('Successfully deleted', result.records_affected, 'unused keys from branch', branch);
log('Successfully deleted', records_affected, 'unused keys from branch', branch);
}

@@ -172,7 +228,8 @@ async function ensureBranch(branch) {

});
trace('Created branch:', branch);
log('Created branch:', branch);
}
async function pull({
branch = 'local-development'
branch = 'local-development',
errorOnNoGlobalKeyTranslation
}, config) {

@@ -184,50 +241,58 @@ trace(`Pulling translations from branch ${branch}`);

trace(`Pulling translations from Phrase for languages ${config.devLanguage} and ${alternativeLanguages.join(', ')}`);
const phraseLanguages = Object.keys(allPhraseTranslations);
trace(`Found Phrase translations for languages ${phraseLanguages.join(', ')}`);
if (!phraseLanguages.includes(config.devLanguage)) {
throw new Error(`Phrase did not return any translations for the configured development language "${config.devLanguage}".\nPlease ensure this language is present in your Phrase project's configuration.`);
}
const allVocabTranslations = await core.loadAllTranslations({
fallbacks: 'none',
includeNodeModules: false
includeNodeModules: false,
withTags: true
}, config);
for (const loadedTranslation of allVocabTranslations) {
const devTranslations = loadedTranslation.languages[config.devLanguage];
if (!devTranslations) {
throw new Error('No dev language translations loaded');
}
const defaultValues = { ...devTranslations
const defaultValues = {
...devTranslations
};
const localKeys = Object.keys(defaultValues);
for (const key of localKeys) {
defaultValues[key] = { ...defaultValues[key],
...allPhraseTranslations[config.devLanguage][core.getUniqueKey(key, loadedTranslation.namespace)]
var _defaultValues$key$gl;
defaultValues[key] = {
...defaultValues[key],
...allPhraseTranslations[config.devLanguage][(_defaultValues$key$gl = defaultValues[key].globalKey) !== null && _defaultValues$key$gl !== void 0 ? _defaultValues$key$gl : core.getUniqueKey(key, loadedTranslation.namespace)]
};
}
// Only write a `_meta` field if necessary
if (Object.keys(loadedTranslation.metadata).length > 0) {
defaultValues._meta = loadedTranslation.metadata;
}
await writeFile(loadedTranslation.filePath, `${JSON.stringify(defaultValues, null, 2)}\n`);
for (const alternativeLanguage of alternativeLanguages) {
if (alternativeLanguage in allPhraseTranslations) {
const altTranslations = { ...loadedTranslation.languages[alternativeLanguage]
const altTranslations = {
...loadedTranslation.languages[alternativeLanguage]
};
const phraseAltTranslations = allPhraseTranslations[alternativeLanguage];
for (const key of localKeys) {
var _phraseAltTranslation;
const phraseKey = core.getUniqueKey(key, loadedTranslation.namespace);
var _defaultValues$key$gl2, _phraseAltTranslation;
const phraseKey = (_defaultValues$key$gl2 = defaultValues[key].globalKey) !== null && _defaultValues$key$gl2 !== void 0 ? _defaultValues$key$gl2 : core.getUniqueKey(key, loadedTranslation.namespace);
const phraseTranslationMessage = (_phraseAltTranslation = phraseAltTranslations[phraseKey]) === null || _phraseAltTranslation === void 0 ? void 0 : _phraseAltTranslation.message;
if (!phraseTranslationMessage) {
trace(`Missing translation. No translation for key ${key} in phrase as ${phraseKey} in language ${alternativeLanguage}.`);
if (errorOnNoGlobalKeyTranslation && defaultValues[key].globalKey) {
throw new Error(`Missing translation for global key ${key} in language ${alternativeLanguage}`);
}
continue;
}
altTranslations[key] = { ...altTranslations[key],
altTranslations[key] = {
...altTranslations[key],
message: phraseTranslationMessage
};
}
const altTranslationFilePath = core.getAltLanguageFilePath(loadedTranslation.filePath, alternativeLanguage);
await mkdir(path__default['default'].dirname(altTranslationFilePath), {
await mkdir(path__default["default"].dirname(altTranslationFilePath), {
recursive: true

@@ -242,12 +307,18 @@ });

/**
* Uploading to the Phrase API for each language. Adding a unique namespace to each key using file path they key came from
* Uploads translations to the Phrase API for each language.
* A unique namespace is appended to each key using the file path the key came from.
*/
async function push({
branch,
deleteUnusedKeys: deleteUnusedKeys$1
deleteUnusedKeys: deleteUnusedKeys$1,
ignore
}, config) {
const allLanguageTranslations = await core.loadAllTranslations({
fallbacks: 'none',
includeNodeModules: false
}, config);
includeNodeModules: false,
withTags: true
}, {
...config,
ignore: [...(config.ignore || []), ...(ignore || [])]
});
trace(`Pushing translations to branch ${branch}`);

@@ -258,32 +329,38 @@ const allLanguages = config.languages.map(v => v.name);

const phraseTranslations = {};
for (const loadedTranslation of allLanguageTranslations) {
for (const language of allLanguages) {
const localTranslations = loadedTranslation.languages[language];
if (!localTranslations) {
continue;
}
if (!phraseTranslations[language]) {
phraseTranslations[language] = {};
}
const {
metadata: {
tags: sharedTags = []
}
} = loadedTranslation;
for (const localKey of Object.keys(localTranslations)) {
const phraseKey = core.getUniqueKey(localKey, loadedTranslation.namespace);
phraseTranslations[language][phraseKey] = localTranslations[localKey];
const {
tags = [],
...localTranslation
} = localTranslations[localKey];
if (language === config.devLanguage) {
localTranslation.tags = [...tags, ...sharedTags];
}
const globalKey = loadedTranslation.languages[config.devLanguage][localKey].globalKey;
const phraseKey = globalKey !== null && globalKey !== void 0 ? globalKey : core.getUniqueKey(localKey, loadedTranslation.namespace);
phraseTranslations[language][phraseKey] = localTranslation;
}
}
}
for (const language of allLanguages) {
if (phraseTranslations[language]) {
const {
uploadId
} = await pushTranslationsByLocale(phraseTranslations[language], language, branch);
if (deleteUnusedKeys$1) {
await deleteUnusedKeys(uploadId, language, branch);
}
}
const {
devLanguageUploadId
} = await pushTranslations(phraseTranslations, {
devLanguage: config.devLanguage,
branch
});
if (deleteUnusedKeys$1) {
await deleteUnusedKeys(devLanguageUploadId, branch);
}

@@ -290,0 +367,0 @@ }

import { promises } from 'fs';
import path from 'path';
import { getAltLanguages, loadAllTranslations, getUniqueKey, getAltLanguageFilePath } from '@vocab/core';
import FormData from 'form-data';
import fetch from 'node-fetch';
import chalk from 'chalk';
import pc from 'picocolors';
import debug from 'debug';
import { stringify } from 'csv-stringify/sync';

@@ -15,17 +14,62 @@ const mkdir = promises.mkdir;

// eslint-disable-next-line no-console
console.log(chalk.yellow('Vocab'), ...params);
console.log(pc.yellow('Vocab'), ...params);
};
function translationsToCsv(translations, devLanguage) {
const languages = Object.keys(translations);
const altLanguages = languages.filter(language => language !== devLanguage);
const devLanguageTranslations = translations[devLanguage];
const csvFilesByLanguage = Object.fromEntries(languages.map(language => [language, []]));
Object.entries(devLanguageTranslations).map(([key, {
message,
description,
tags
}]) => {
const sharedData = [key, description, tags === null || tags === void 0 ? void 0 : tags.join(',')];
const devLanguageRow = [...sharedData, message];
csvFilesByLanguage[devLanguage].push(devLanguageRow);
altLanguages.map(language => {
var _translations$languag;
const altTranslationMessage = (_translations$languag = translations[language]) === null || _translations$languag === void 0 || (_translations$languag = _translations$languag[key]) === null || _translations$languag === void 0 ? void 0 : _translations$languag.message;
if (altTranslationMessage) {
csvFilesByLanguage[language].push([...sharedData, altTranslationMessage]);
}
});
});
const csvFileStrings = Object.fromEntries(Object.entries(csvFilesByLanguage)
// Ensure CSV files are only created if the language has at least 1 translation
.filter(([_, csvFile]) => csvFile.length > 0).map(([language, csvFile]) => {
const csvFileString = stringify(csvFile, {
delimiter: ',',
header: false
});
return [language, csvFileString];
}));
// Column indices start at 1
const keyIndex = 1;
const commentIndex = keyIndex + 1;
const tagColumn = commentIndex + 1;
const messageIndex = tagColumn + 1;
return {
csvFileStrings,
keyIndex,
messageIndex,
commentIndex,
tagColumn
};
}
/* eslint-disable no-console */
function _callPhrase(path, options = {}) {
const phraseApiToken = process.env.PHRASE_API_TOKEN;
if (!phraseApiToken) {
throw new Error('Missing PHRASE_API_TOKEN');
}
return fetch(path, { ...options,
return fetch(path, {
...options,
headers: {
Authorization: `token ${phraseApiToken}`,
// Provide identification via User Agent as requested in https://developers.phrase.com/api/#overview--identification-via-user-agent
'User-Agent': 'SEEK Demo Candidate App (jhope@seek.com.au)',
'User-Agent': 'Vocab Client (https://github.com/seek-oss/vocab)',
...options.headers

@@ -35,4 +79,6 @@ }

console.log(`${path}: ${response.status} - ${response.statusText}`);
console.log(`Rate Limit: ${response.headers.get('X-Rate-Limit-Remaining')} of ${response.headers.get('X-Rate-Limit-Limit')} remaining. (${response.headers.get('X-Rate-Limit-Reset')} seconds remaining})`);
console.log('\nLink:', response.headers.get('Link'), '\n'); // Print All Headers:
const secondsUntilLimitReset = Math.ceil(Number.parseFloat(response.headers.get('X-Rate-Limit-Reset') || '0') - Date.now() / 1000);
console.log(`Rate Limit: ${response.headers.get('X-Rate-Limit-Remaining')} of ${response.headers.get('X-Rate-Limit-Limit')} remaining. (${secondsUntilLimitReset} seconds remaining)`);
trace('\nLink:', response.headers.get('Link'), '\n');
// Print All Headers:
// console.log(Array.from(r.headers.entries()));

@@ -42,20 +88,14 @@

var _response$headers$get;
const result = await response.json();
console.log(`Internal Result (Length: ${result.length})\n`);
trace(`Internal Result (Length: ${result.length})\n`);
if ((!options.method || options.method === 'GET') && (_response$headers$get = response.headers.get('Link')) !== null && _response$headers$get !== void 0 && _response$headers$get.includes('rel=next')) {
var _response$headers$get2, _response$headers$get3;
const [, nextPageUrl] = (_response$headers$get2 = (_response$headers$get3 = response.headers.get('Link')) === null || _response$headers$get3 === void 0 ? void 0 : _response$headers$get3.match(/<([^>]*)>; rel=next/)) !== null && _response$headers$get2 !== void 0 ? _response$headers$get2 : [];
if (!nextPageUrl) {
throw new Error('Cant parse next page URL');
throw new Error("Can't parse next page URL");
}
console.log('Results recieved with next page: ', nextPageUrl);
console.log('Results received with next page: ', nextPageUrl);
const nextPageResult = await _callPhrase(nextPageUrl, options);
return [...result, ...nextPageResult];
}
return result;

@@ -68,10 +108,7 @@ } catch (e) {

}
async function callPhrase(relativePath, options = {}) {
const projectId = process.env.PHRASE_PROJECT_ID;
if (!projectId) {
throw new Error('Missing PHRASE_PROJECT_ID');
}
return _callPhrase(`https://api.phrase.com/v2/projects/${projectId}/${relativePath}`, options).then(result => {

@@ -81,3 +118,2 @@ if (Array.isArray(result)) {

}
return result;

@@ -92,3 +128,2 @@ }).catch(error => {

const translations = {};
for (const r of phraseResult) {

@@ -98,3 +133,2 @@ if (!translations[r.locale.code]) {

}
translations[r.locale.code][r.key.name] = {

@@ -104,32 +138,57 @@ message: r.content

}
return translations;
}
async function pushTranslationsByLocale(contents, locale, branch) {
const formData = new FormData();
const fileContents = Buffer.from(JSON.stringify(contents));
formData.append('file', fileContents, {
contentType: 'application/json',
filename: `${locale}.json`
});
formData.append('file_format', 'json');
formData.append('locale_id', locale);
formData.append('branch', branch);
formData.append('update_translations', 'true');
trace('Starting to upload:', locale);
async function pushTranslations(translationsByLanguage, {
devLanguage,
branch
}) {
const {
id
} = await callPhrase(`uploads`, {
method: 'POST',
body: formData
});
log('Upload ID:', id, '\n');
log('Successfully Uploaded:', locale, '\n');
csvFileStrings,
keyIndex,
commentIndex,
tagColumn,
messageIndex
} = translationsToCsv(translationsByLanguage, devLanguage);
let devLanguageUploadId = '';
for (const [language, csvFileString] of Object.entries(csvFileStrings)) {
const formData = new FormData();
formData.append('file', new Blob([csvFileString], {
type: 'text/csv'
}), `${language}.translations.csv`);
formData.append('file_format', 'csv');
formData.append('branch', branch);
formData.append('update_translations', 'true');
formData.append('update_descriptions', 'true');
formData.append(`locale_mapping[${language}]`, messageIndex.toString());
formData.append('format_options[key_index]', keyIndex.toString());
formData.append('format_options[comment_index]', commentIndex.toString());
formData.append('format_options[tag_column]', tagColumn.toString());
formData.append('format_options[enable_pluralization]', 'false');
log(`Uploading translations for language ${language}`);
const result = await callPhrase(`uploads`, {
method: 'POST',
body: formData
});
trace('Upload result:\n', result);
if (result && 'id' in result) {
log('Upload ID:', result.id, '\n');
log('Successfully Uploaded\n');
} else {
log(`Error uploading: ${result === null || result === void 0 ? void 0 : result.message}\n`);
log('Response:', result);
throw new Error('Error uploading');
}
if (language === devLanguage) {
devLanguageUploadId = result.id;
}
}
return {
uploadId: id
devLanguageUploadId
};
}
async function deleteUnusedKeys(uploadId, locale, branch) {
async function deleteUnusedKeys(uploadId, branch) {
const query = `unmentioned_in_upload:${uploadId}`;
const result = await callPhrase('keys', {
const {
records_affected
} = await callPhrase('keys', {
method: 'DELETE',

@@ -141,7 +200,6 @@ headers: {

branch,
locale_id: locale,
q: query
})
});
log('Successfully deleted', result.records_affected, 'unused keys from branch', branch);
log('Successfully deleted', records_affected, 'unused keys from branch', branch);
}

@@ -158,7 +216,8 @@ async function ensureBranch(branch) {

});
trace('Created branch:', branch);
log('Created branch:', branch);
}
async function pull({
branch = 'local-development'
branch = 'local-development',
errorOnNoGlobalKeyTranslation
}, config) {

@@ -170,48 +229,56 @@ trace(`Pulling translations from branch ${branch}`);

trace(`Pulling translations from Phrase for languages ${config.devLanguage} and ${alternativeLanguages.join(', ')}`);
const phraseLanguages = Object.keys(allPhraseTranslations);
trace(`Found Phrase translations for languages ${phraseLanguages.join(', ')}`);
if (!phraseLanguages.includes(config.devLanguage)) {
throw new Error(`Phrase did not return any translations for the configured development language "${config.devLanguage}".\nPlease ensure this language is present in your Phrase project's configuration.`);
}
const allVocabTranslations = await loadAllTranslations({
fallbacks: 'none',
includeNodeModules: false
includeNodeModules: false,
withTags: true
}, config);
for (const loadedTranslation of allVocabTranslations) {
const devTranslations = loadedTranslation.languages[config.devLanguage];
if (!devTranslations) {
throw new Error('No dev language translations loaded');
}
const defaultValues = { ...devTranslations
const defaultValues = {
...devTranslations
};
const localKeys = Object.keys(defaultValues);
for (const key of localKeys) {
defaultValues[key] = { ...defaultValues[key],
...allPhraseTranslations[config.devLanguage][getUniqueKey(key, loadedTranslation.namespace)]
var _defaultValues$key$gl;
defaultValues[key] = {
...defaultValues[key],
...allPhraseTranslations[config.devLanguage][(_defaultValues$key$gl = defaultValues[key].globalKey) !== null && _defaultValues$key$gl !== void 0 ? _defaultValues$key$gl : getUniqueKey(key, loadedTranslation.namespace)]
};
}
// Only write a `_meta` field if necessary
if (Object.keys(loadedTranslation.metadata).length > 0) {
defaultValues._meta = loadedTranslation.metadata;
}
await writeFile(loadedTranslation.filePath, `${JSON.stringify(defaultValues, null, 2)}\n`);
for (const alternativeLanguage of alternativeLanguages) {
if (alternativeLanguage in allPhraseTranslations) {
const altTranslations = { ...loadedTranslation.languages[alternativeLanguage]
const altTranslations = {
...loadedTranslation.languages[alternativeLanguage]
};
const phraseAltTranslations = allPhraseTranslations[alternativeLanguage];
for (const key of localKeys) {
var _phraseAltTranslation;
const phraseKey = getUniqueKey(key, loadedTranslation.namespace);
var _defaultValues$key$gl2, _phraseAltTranslation;
const phraseKey = (_defaultValues$key$gl2 = defaultValues[key].globalKey) !== null && _defaultValues$key$gl2 !== void 0 ? _defaultValues$key$gl2 : getUniqueKey(key, loadedTranslation.namespace);
const phraseTranslationMessage = (_phraseAltTranslation = phraseAltTranslations[phraseKey]) === null || _phraseAltTranslation === void 0 ? void 0 : _phraseAltTranslation.message;
if (!phraseTranslationMessage) {
trace(`Missing translation. No translation for key ${key} in phrase as ${phraseKey} in language ${alternativeLanguage}.`);
if (errorOnNoGlobalKeyTranslation && defaultValues[key].globalKey) {
throw new Error(`Missing translation for global key ${key} in language ${alternativeLanguage}`);
}
continue;
}
altTranslations[key] = { ...altTranslations[key],
altTranslations[key] = {
...altTranslations[key],
message: phraseTranslationMessage
};
}
const altTranslationFilePath = getAltLanguageFilePath(loadedTranslation.filePath, alternativeLanguage);

@@ -228,12 +295,18 @@ await mkdir(path.dirname(altTranslationFilePath), {

/**
* Uploading to the Phrase API for each language. Adding a unique namespace to each key using file path they key came from
* Uploads translations to the Phrase API for each language.
* A unique namespace is appended to each key using the file path the key came from.
*/
async function push({
branch,
deleteUnusedKeys: deleteUnusedKeys$1
deleteUnusedKeys: deleteUnusedKeys$1,
ignore
}, config) {
const allLanguageTranslations = await loadAllTranslations({
fallbacks: 'none',
includeNodeModules: false
}, config);
includeNodeModules: false,
withTags: true
}, {
...config,
ignore: [...(config.ignore || []), ...(ignore || [])]
});
trace(`Pushing translations to branch ${branch}`);

@@ -244,32 +317,38 @@ const allLanguages = config.languages.map(v => v.name);

const phraseTranslations = {};
for (const loadedTranslation of allLanguageTranslations) {
for (const language of allLanguages) {
const localTranslations = loadedTranslation.languages[language];
if (!localTranslations) {
continue;
}
if (!phraseTranslations[language]) {
phraseTranslations[language] = {};
}
const {
metadata: {
tags: sharedTags = []
}
} = loadedTranslation;
for (const localKey of Object.keys(localTranslations)) {
const phraseKey = getUniqueKey(localKey, loadedTranslation.namespace);
phraseTranslations[language][phraseKey] = localTranslations[localKey];
const {
tags = [],
...localTranslation
} = localTranslations[localKey];
if (language === config.devLanguage) {
localTranslation.tags = [...tags, ...sharedTags];
}
const globalKey = loadedTranslation.languages[config.devLanguage][localKey].globalKey;
const phraseKey = globalKey !== null && globalKey !== void 0 ? globalKey : getUniqueKey(localKey, loadedTranslation.namespace);
phraseTranslations[language][phraseKey] = localTranslation;
}
}
}
for (const language of allLanguages) {
if (phraseTranslations[language]) {
const {
uploadId
} = await pushTranslationsByLocale(phraseTranslations[language], language, branch);
if (deleteUnusedKeys$1) {
await deleteUnusedKeys(uploadId, language, branch);
}
}
const {
devLanguageUploadId
} = await pushTranslations(phraseTranslations, {
devLanguage: config.devLanguage,
branch
});
if (deleteUnusedKeys$1) {
await deleteUnusedKeys(devLanguageUploadId, branch);
}

@@ -276,0 +355,0 @@ }

{
"name": "@vocab/phrase",
"version": "0.0.0-delete-unused-keys-20228144520",
"version": "0.0.0-feature-ignore-flag-push-20241014224750",
"repository": {
"type": "git",
"url": "https://github.com/seek-oss/vocab.git",
"directory": "packages/phrase"
},
"engines": {
"node": ">=18"
},
"main": "dist/vocab-phrase.cjs.js",

@@ -9,12 +17,14 @@ "module": "dist/vocab-phrase.esm.js",

"dependencies": {
"@vocab/core": "^1.0.0",
"@vocab/types": "^1.0.0",
"chalk": "^4.1.0",
"csv-stringify": "^6.2.3",
"debug": "^4.3.1",
"form-data": "^3.0.0",
"node-fetch": "^2.6.1"
"picocolors": "^1.0.0",
"@vocab/core": "^1.6.2"
},
"devDependencies": {
"@types/node-fetch": "^2.5.7"
}
}
"@types/debug": "^4.1.5",
"@types/node": "^18.11.9"
},
"files": [
"dist"
]
}
SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

Packages

npm

Stay in touch

Get open source security insights delivered straight into your inbox.


  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc