Big News: Socket raises $60M Series C at a $1B valuation to secure software supply chains for AI-driven development.Announcement →
Sign In

@formatjs/intl-segmenter

Package Overview
Dependencies
Maintainers
3
Versions
53
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@formatjs/intl-segmenter - npm Package Compare versions

Comparing version
12.2.1
 to
12.2.2
+58
index.d.ts
//#region packages/intl-segmenter/segmenter.d.ts
type SegmentResult = {
segment: string;
breakingRule?: string;
nonBreakingRules?: string[];
} | undefined;
interface SegmenterOptions {
localeMatcher?: "lookup" | "best fit";
granularity?: "word" | "sentence" | "grapheme";
}
interface SegmenterResolvedOptions {
locale: string;
granularity: NonNullable<SegmenterOptions["granularity"]>;
}
declare const breaksAtResult: (breaks: boolean, matchingRule: string) => {
breaks: boolean;
matchingRule: string;
};
declare class Segmenter {
private readonly rules;
private readonly ruleSortedKeys;
private readonly mergedSegmentationTypeValue;
constructor(locales: string | string[] | undefined, options: SegmenterOptions);
breaksAt(position: number, input: string): ReturnType<typeof breaksAtResult>;
segment(input: string): SegmentIterator;
resolvedOptions(): SegmenterResolvedOptions;
static availableLocales: Set<string>;
static supportedLocalesOf(locales?: string | string[], options?: Pick<SegmenterOptions, "localeMatcher">): string[];
static readonly polyfilled = true;
}
declare class SegmentIterator implements Iterable<SegmentResult>, Iterator<SegmentResult> {
private readonly segmenter;
private lastSegmentIndex;
private input;
constructor(segmenter: Segmenter, input: string);
[Symbol.iterator](): SegmentIterator;
next(): {
done: boolean;
value: {
segment: string;
index: number;
input: string;
isWordLike?: boolean;
};
} | {
done: boolean;
value: undefined;
};
containing(positionInput: number): {
segment: string;
index: number;
input: string;
isWordLike?: boolean;
} | undefined;
}
//#endregion
export { type SegmentIterator, Segmenter, SegmenterOptions, SegmenterResolvedOptions };
//# sourceMappingURL=index.d.ts.map
index.js

Sorry, the diff of this file is too big to display

index.js.map

Sorry, the diff of this file is too big to display

polyfill-force.js.map

Sorry, the diff of this file is too big to display

polyfill.js.map

Sorry, the diff of this file is too big to display

+1
should-polyfill.js.map
{"version":3,"file":"should-polyfill.js","names":[],"sources":["../should-polyfill.ts"],"sourcesContent":["export function shouldPolyfill(): boolean {\n return !(Intl as any).Segmenter\n}\n"],"mappings":";AAAA,SAAgB,iBAA0B;AACxC,QAAO,CAAE,KAAa"}
+3
-3
package.json
{
"name": "@formatjs/intl-segmenter",
"description": "Polyfill for Intl.Segmenter",
"version": "12.2.1",
"version": "12.2.2",
"license": "MIT",

@@ -9,2 +9,3 @@ "author": "Matija Gaspar <matijagaspar@gmail.com>",

"exports": {
".": "./index.js",
"./polyfill.js": "./polyfill.js",

@@ -15,4 +16,3 @@ "./polyfill-force.js": "./polyfill-force.js",

"dependencies": {
"@formatjs/ecma402-abstract": "3.2.0",
"@formatjs/intl-localematcher": "0.8.2"
"@formatjs/intl-localematcher": "0.8.3"
},

@@ -19,0 +19,0 @@ "bugs": "https://github.com/formatjs/formatjs/issues",

+1
-1
polyfill-force.d.ts

@@ -1,1 +0,1 @@

export {};
export { };
+1
-1
polyfill.d.ts

@@ -1,1 +0,1 @@

export {};
export { };
+5
-1
should-polyfill.d.ts

@@ -1,1 +0,5 @@

export declare function shouldPolyfill(): boolean;
//#region packages/intl-segmenter/should-polyfill.d.ts
declare function shouldPolyfill(): boolean;
//#endregion
export { shouldPolyfill };
//# sourceMappingURL=should-polyfill.d.ts.map
+6
-1
should-polyfill.js

@@ -1,3 +0,8 @@

export function shouldPolyfill() {
//#region packages/intl-segmenter/should-polyfill.ts
function shouldPolyfill() {
return !Intl.Segmenter;
}
//#endregion
export { shouldPolyfill };
//# sourceMappingURL=should-polyfill.js.map
src/cldr-segmentation-rules.generated.d.ts

Sorry, the diff of this file is too big to display

src/cldr-segmentation-rules.generated.js

Sorry, the diff of this file is too big to display

-2
src/segmentation-utils.d.ts
export declare const replaceVariables: (variables: Record<string, string>, input: string) => string;
export declare const isSurrogate: (str: string, pos: number) => boolean;
-29
src/segmentation-utils.js
export const replaceVariables = (variables, input) => {
const findVarRegex = /\$[A-Za-z0-9_]+/gm;
return input.replaceAll(findVarRegex, (match) => {
if (!(match in variables)) {
throw new Error(`No such variable ${match}`);
}
return variables[match];
});
};
export const isSurrogate = (str, pos) => {
return 55296 <= str.charCodeAt(pos - 1) && str.charCodeAt(pos - 1) <= 56319 && 56320 <= str.charCodeAt(pos) && str.charCodeAt(pos) <= 57343;
};
// alternative surrogate check mimicking the java implementation
// const TRAIL_SURROGATE_BITMASK = 0xfffffc00
// const TRAIL_SURROGATE_BITS = 0xdc00
// const LEAD_SURROGATE_BITMASK = 0xfffffc00
// const LEAD_SURROGATE_BITS = 0xd800
// const isSurrogate = (text: string, position: number) => {
// if (
// (text.charCodeAt(position - 1) & LEAD_SURROGATE_BITMASK) ==
// LEAD_SURROGATE_BITS &&
// (text.charCodeAt(position) & TRAIL_SURROGATE_BITMASK) ==
// TRAIL_SURROGATE_BITS
// ) {
// return true
// } else {
// return false
// }
// }
-55
src/segmenter.d.ts
type SegmentResult = {
segment: string;
breakingRule?: string;
nonBreakingRules?: string[];
} | undefined;
export interface SegmenterOptions {
localeMatcher?: "lookup" | "best fit";
granularity?: "word" | "sentence" | "grapheme";
}
export interface SegmenterResolvedOptions {
locale: string;
granularity: NonNullable<SegmenterOptions["granularity"]>;
}
declare const breaksAtResult: (breaks: boolean, matchingRule: string) => {
breaks: boolean;
matchingRule: string;
};
export declare class Segmenter {
private readonly rules;
private readonly ruleSortedKeys;
private readonly mergedSegmentationTypeValue;
constructor(locales: string | string[] | undefined, options: SegmenterOptions);
breaksAt(position: number, input: string): ReturnType<typeof breaksAtResult>;
segment(input: string): SegmentIterator;
resolvedOptions(): SegmenterResolvedOptions;
static availableLocales: Set<string>;
static supportedLocalesOf(locales?: string | string[], options?: Pick<SegmenterOptions, "localeMatcher">): string[];
static readonly polyfilled = true;
}
declare class SegmentIterator implements Iterable<SegmentResult>, Iterator<SegmentResult> {
private readonly segmenter;
private lastSegmentIndex;
private input;
constructor(segmenter: Segmenter, input: string);
[Symbol.iterator](): SegmentIterator;
next(): {
done: boolean;
value: {
segment: string;
index: number;
input: string;
isWordLike?: boolean;
};
} | {
done: boolean;
value: undefined;
};
containing(positionInput: number): {
segment: string;
index: number;
input: string;
isWordLike?: boolean;
} | undefined;
}
export type { SegmentIterator };
-350
src/segmenter.js
import { CanonicalizeLocaleList, GetOption, GetOptionsObject, SupportedLocales, getInternalSlot, getMultiInternalSlots, setInternalSlot } from "@formatjs/ecma402-abstract";
import { ResolveLocale } from "@formatjs/intl-localematcher";
import { SegmentationRules } from "./cldr-segmentation-rules.generated.js";
import { isSurrogate, replaceVariables } from "./segmentation-utils.js";
// Cached regex patterns for word character detection
// Note: Unicode property escape regex is created at runtime in try-catch
// to avoid compile-time errors when targeting ES5
const WORD_CHARACTERS_BASIC_REGEX = /\w/;
// Lazy-initialized Unicode word character regex (null if not supported)
let WORD_CHARACTERS_UNICODE_REGEX = undefined;
/**
* Adds $ to before rules and ^ to after rules for strictness
* Replaces variables
* Initializes the RegExp
*
* @param rule raw rule string from cldr-segmentation-rules.generated
* @param variables
* @param after appends ^ if true and $ if false
* @returns
*/
const generateRuleRegex = (rule, variables, after) => {
return new RegExp(`${after ? "^" : ""}${replaceVariables(variables, rule)}${after ? "" : "$"}`);
};
const prepareLocaleSegmentationRules = (segmentationTypeValue) => {
const preparedRules = {};
for (const ruleNr of Object.keys(segmentationTypeValue.segmentRules)) {
const ruleValue = segmentationTypeValue.segmentRules[ruleNr];
const preparedRule = { breaks: ruleValue.breaks };
if ("before" in ruleValue && ruleValue.before) {
preparedRule.before = generateRuleRegex(ruleValue.before, segmentationTypeValue.variables, false);
}
if ("after" in ruleValue && ruleValue.after) {
preparedRule.after = generateRuleRegex(ruleValue.after, segmentationTypeValue.variables, true);
}
preparedRules[ruleNr] = preparedRule;
}
return preparedRules;
};
const breaksAtResult = (breaks, matchingRule) => ({
breaks,
matchingRule
});
export class Segmenter {
rules;
ruleSortedKeys;
mergedSegmentationTypeValue;
constructor(locales, options) {
if (new.target === undefined) {
throw TypeError(`Constructor Intl.Segmenter requires 'new'`);
}
const requestedLocales = CanonicalizeLocaleList(locales);
options = GetOptionsObject(options);
const opt = Object.create(null);
const matcher = GetOption(options, "localeMatcher", "string", ["lookup", "best fit"], "best fit");
opt.localeMatcher = matcher;
const granularity = GetOption(options, "granularity", "string", [
"word",
"sentence",
"grapheme"
], "grapheme");
setSlot(this, "granularity", granularity);
//TODO: figure out correct availible locales
const r = ResolveLocale(Segmenter.availableLocales, requestedLocales, opt, [], {}, () => "");
setSlot(this, "locale", r.locale);
//root rules based on granularity
this.mergedSegmentationTypeValue = SegmentationRules.root[granularity];
//merge root rules with locale ones if locale is specified
if (r.locale.length) {
const localeOverrides = SegmentationRules[r.locale];
if (granularity in localeOverrides) {
const localeSegmentationTypeValue = localeOverrides[granularity];
this.mergedSegmentationTypeValue.variables = {
...this.mergedSegmentationTypeValue.variables,
...localeSegmentationTypeValue.variables
};
this.mergedSegmentationTypeValue.segmentRules = {
...this.mergedSegmentationTypeValue.segmentRules,
...localeSegmentationTypeValue.segmentRules
};
this.mergedSegmentationTypeValue.suppressions = [...this.mergedSegmentationTypeValue.suppressions, ...localeSegmentationTypeValue.suppressions];
}
}
//prepare rules
this.rules = prepareLocaleSegmentationRules(this.mergedSegmentationTypeValue);
//order rule keys
this.ruleSortedKeys = Object.keys(this.rules).sort((a, b) => Number(a) - Number(b));
}
breaksAt(position, input) {
const ruleSortedKeys = this.ruleSortedKeys;
const rules = this.rules;
const mergedSegmentationTypeValue = this.mergedSegmentationTypeValue;
//artificial rule 0.2
if (position === 0) {
return breaksAtResult(true, "0.2");
}
if (position === input.length) {
//rule 0.3
return breaksAtResult(true, "0.3");
}
//artificial rule 0.1: js specific, due to es5 regex not being unicode aware
//number 0.1 chosen to mimic java implementation, but needs to execute after 0.2 and 0.3 to be inside the string bounds
if (isSurrogate(input, position)) {
return breaksAtResult(false, "0.1");
}
const stringBeforeBreak = input.substring(0, position);
const stringAfterBreak = input.substring(position);
//artificial rule 0.4: handle suppressions
if ("suppressions" in mergedSegmentationTypeValue) {
for (const suppressions of mergedSegmentationTypeValue.suppressions) {
if (stringBeforeBreak.trim().endsWith(suppressions)) {
return breaksAtResult(false, "0.4");
}
}
}
// loop through rules and find a match
for (const ruleKey of ruleSortedKeys) {
const { before, after, breaks } = rules[ruleKey];
// for debugging
// if (ruleKey === '16' && position === 4) {
// console.log({before, after, stringBeforeBreak, stringAfterBreak})
// }
if (before) {
if (!before.test(stringBeforeBreak)) {
//didn't match the before part, therfore skipping
continue;
}
}
if (after) {
if (!after.test(stringAfterBreak)) {
//didn't match the after part, therfore skipping
continue;
}
}
return breaksAtResult(breaks, ruleKey);
}
//artificial rule 999: if no rule matched is Any ÷ Any so return true
return breaksAtResult(true, "999");
}
segment(input) {
checkReceiver(this, "segment");
return new SegmentIterator(this, input);
}
resolvedOptions() {
checkReceiver(this, "resolvedOptions");
return { ...getMultiInternalSlots(__INTERNAL_SLOT_MAP__, this, "locale", "granularity") };
}
static availableLocales = new Set(Object.keys(SegmentationRules).filter((key) => key !== "root"));
static supportedLocalesOf(locales, options) {
return SupportedLocales(Segmenter.availableLocales, CanonicalizeLocaleList(locales), options);
}
static polyfilled = true;
}
/**
* Determines if a segment is word-like according to Unicode Word Break rules.
*
* A segment is considered word-like if it contains alphabetic characters,
* numbers, or ideographs. Segments containing only whitespace, punctuation,
* or symbols are not word-like.
*
* Per Unicode Word Break (UAX #29) and native Intl.Segmenter implementations,
* this matches segments that contain characters from word character classes:
* ALetter, Hebrew_Letter, Numeric, Katakana, Hiragana, and Ideographic.
*
* @param segment - The text segment to check
* @param matchingRule - The word break rule that created this segment
* @returns true if the segment is word-like
*/
function isSegmentWordLike(segment, matchingRule) {
// Primary check: Does the segment contain word characters?
// Word-like segments contain letters (including ideographs), numbers,
// or connecting characters like apostrophes within words
//
// Regex matches:
// - Letters: \p{L} (all Unicode letters)
// - Numbers: \p{N} (all Unicode numbers)
// - Marks: \p{M} (combining marks, typically part of letters)
//
// Note: Using Unicode property escapes which work in modern JS engines
// and are necessary for proper internationalization
// Lazy-initialize Unicode regex on first use
if (WORD_CHARACTERS_UNICODE_REGEX === undefined) {
try {
// Create Unicode property escape regex at runtime to avoid compile-time TS1501 error
WORD_CHARACTERS_UNICODE_REGEX = new RegExp("[\\p{L}\\p{N}\\p{M}]", "u");
} catch {
// Environment doesn't support Unicode property escapes
WORD_CHARACTERS_UNICODE_REGEX = null;
}
}
let hasWordCharacters;
if (WORD_CHARACTERS_UNICODE_REGEX) {
// Check if segment contains word characters using Unicode property escapes
// This matches the behavior of native Intl.Segmenter in Chrome/Firefox
hasWordCharacters = WORD_CHARACTERS_UNICODE_REGEX.test(segment);
} else {
// Fallback for environments without Unicode property escapes
// Match basic word characters: letters, numbers, underscores
hasWordCharacters = WORD_CHARACTERS_BASIC_REGEX.test(segment);
}
// If segment contains word characters, it's word-like
if (hasWordCharacters) {
return true;
}
// If no word characters, check if it's definitely not word-like via rules
// Non-word-like rules per Unicode Word Break specification (UAX #29):
// https://unicode.org/reports/tr29/#Word_Boundaries
//
// WB3a (3.1): Break before newlines (sot ÷ (Newline | CR | LF))
// WB3b (3.2): Break after newlines ((Newline | CR | LF) ÷ eot)
// WB3d (3.4): Keep horizontal whitespace together (WSegSpace × WSegSpace)
//
// These rules specifically identify non-word segments like line breaks and whitespace
const definitelyNotWordLikeRules = [
"3.1",
"3.2",
"3.4"
];
if (definitelyNotWordLikeRules.includes(matchingRule)) {
return false;
}
// For segments without word characters and not matching specific non-word rules,
// return false (e.g., punctuation, symbols, whitespace via rule 999)
return false;
}
const createSegmentDataObject = (segmenter, segment, index, input, matchingRule) => {
const returnValue = {
segment,
index,
input
};
if (getSlot(segmenter, "granularity") === "word") {
returnValue.isWordLike = isSegmentWordLike(segment, matchingRule);
}
return returnValue;
};
class SegmentIterator {
segmenter;
lastSegmentIndex;
input;
constructor(segmenter, input) {
this.segmenter = segmenter;
this.lastSegmentIndex = 0;
if (typeof input == "symbol") {
throw TypeError(`Input must not be a symbol`);
}
this.input = String(input);
}
[Symbol.iterator]() {
return new SegmentIterator(this.segmenter, this.input);
}
next() {
//using only the relevant bit of the string
let checkString = this.input.substring(this.lastSegmentIndex);
//loop from the start of the checkString, until exactly length (breaksAt returns break at pos=== lenght)
for (let position = 1; position <= checkString.length; position++) {
const { breaks, matchingRule } = this.segmenter.breaksAt(position, checkString);
if (breaks) {
const segment = checkString.substring(0, position);
const index = this.lastSegmentIndex;
this.lastSegmentIndex += position;
return {
done: false,
value: createSegmentDataObject(this.segmenter, segment, index, this.input, matchingRule)
};
}
}
//no segment was found by the loop, therefore the segmentation is done
return {
done: true,
value: undefined
};
}
containing(positionInput) {
if (typeof positionInput === "bigint") {
throw TypeError("Index must not be a BigInt");
}
let position = Number(positionInput);
//https://tc39.es/ecma262/#sec-tointegerorinfinity
// 2. If number is NaN, +0𝔽, or -0𝔽, return 0.
if (isNaN(position) || !position) {
position = 0;
}
// 5. Let integer be floor(abs(ℝ(number))).
// 6. If number < -0𝔽, set integer to -integer.
position = Math.floor(Math.abs(position)) * (position < 0 ? -1 : 1);
if (position < 0 || position >= this.input.length) {
return undefined;
}
//find previous break point
let previousBreakPoint = 0;
if (position === 0) {
previousBreakPoint = 0;
} else {
const checkString = this.input;
for (let cursor = position; cursor >= 0; cursor--) {
const { breaks } = this.segmenter.breaksAt(cursor, checkString);
if (breaks) {
previousBreakPoint = cursor;
break;
}
}
}
let checkString = this.input.substring(previousBreakPoint);
//find next break point
for (let cursor = 1; cursor <= checkString.length; cursor++) {
const { breaks, matchingRule } = this.segmenter.breaksAt(cursor, checkString);
if (breaks) {
const segment = checkString.substring(0, cursor);
return createSegmentDataObject(this.segmenter, segment, previousBreakPoint, this.input, matchingRule);
}
}
}
}
const __INTERNAL_SLOT_MAP__ = new WeakMap();
function getSlot(instance, key) {
return getInternalSlot(__INTERNAL_SLOT_MAP__, instance, key);
}
function setSlot(instance, key, value) {
setInternalSlot(__INTERNAL_SLOT_MAP__, instance, key, value);
}
function checkReceiver(receiver, methodName) {
if (!(receiver instanceof Segmenter)) {
throw TypeError(`Method Intl.Segmenter.prototype.${methodName} called on incompatible receiver`);
}
}
try {
// IE11 does not have Symbol
if (typeof Symbol !== "undefined") {
Object.defineProperty(Segmenter.prototype, Symbol.toStringTag, {
value: "Intl.Segmenter",
writable: false,
enumerable: false,
configurable: true
});
}
//github.com/tc39/test262/blob/main/test/intl402/Segmenter/constructor/length.js
Object.defineProperty(Segmenter.prototype.constructor, "length", {
value: 0,
writable: false,
enumerable: false,
configurable: true
});
// https://github.com/tc39/test262/blob/main/test/intl402/Segmenter/constructor/supportedLocalesOf/length.js
Object.defineProperty(Segmenter.supportedLocalesOf, "length", {
value: 1,
writable: false,
enumerable: false,
configurable: true
});
} catch {}
-2
test262-main.d.ts
import "./polyfill-force.js";
export {};
-3
test262-main.js
// @generated
// @ts-nocheck
import "./polyfill-force.js";
polyfill-force.js

Sorry, the diff of this file is too big to display

polyfill.iife.js

Sorry, the diff of this file is too big to display

polyfill.js

Sorry, the diff of this file is too big to display