ploon - npm Package Compare versions

Comparing version 1.0.2 to 1.0.3

dist/decode/decoder.d.ts

 /**
  * PLOON Decoder
  * Reconstructs objects from PLOON records
+ *
+ * PLOON Record Format:
+ * - Array record: "4:1|value1|value2"  (depth:index|values)
+ * - Object record: "4 |value1|value2"  (depth |values)
+ * - Empty marker: "4:1|value1|"        (trailing | means empty marker at end)
  */
@@ -5,0 +10,0 @@ import type { JsonValue, PloonConfig } from '../types';

dist/decode/parser.d.ts

@@ -15,2 +15,4 @@ /**
     isObject: boolean;
+    isPrimitiveArray?: boolean;
+    isOptional?: boolean;
     nested?: ParsedSchema;
@@ -17,0 +19,0 @@ }

dist/encode/encoder.d.ts

 /**
  * PLOON Encoder
  * Main encoding logic - converts data to PLOON format
+ *
+ * PLOON Record Format:
+ * - Array record: "4:1|value1|value2"  (depth:index|values)
+ * - Object record: "4 |value1|value2"  (depth |values)
+ * - Empty marker: "4:1|value1|"        (trailing | means empty marker at end)
  */
@@ -5,0 +10,0 @@ import type { JsonValue, PloonConfig } from '../types';

dist/encode/normalize.d.ts

@@ -32,1 +32,9 @@ /**
 export declare function getAllKeys(objects: JsonObject[]): string[];
+/**
+ * Analyze fields in array of objects to determine which are required vs optional
+ * A field is optional if it's missing from any object OR empty (array/object with no content) in any object
+ * A field is required only if it appears in ALL objects AND always has content
+ */
+export declare function analyzeFields(objects: JsonObject[]): Map<string, {
+    isOptional: boolean;
+}>;

dist/index.js

@@ -54,2 +54,26 @@ import { XMLBuilder, XMLParser, XMLValidator } from "fast-xml-parser";
 }
+/**
+* Analyze fields in array of objects to determine which are required vs optional
+* A field is optional if it's missing from any object OR empty (array/object with no content) in any object
+* A field is required only if it appears in ALL objects AND always has content
+*/
+function analyzeFields(objects) {
+	const fieldMetadata = /* @__PURE__ */ new Map();
+	if (objects.length === 0) return fieldMetadata;
+	const fieldCounts = /* @__PURE__ */ new Map();
+	const fieldEverEmpty = /* @__PURE__ */ new Map();
+	for (const obj of objects) for (const key of Object.keys(obj)) {
+		fieldCounts.set(key, (fieldCounts.get(key) || 0) + 1);
+		const value = obj[key];
+		const isEmpty = isJsonArray(value) && value.length === 0 || isJsonObject(value) && !isJsonArray(value) && Object.keys(value).length === 0;
+		if (isEmpty) fieldEverEmpty.set(key, true);
+	}
+	const totalObjects = objects.length;
+	for (const [fieldName, count] of fieldCounts.entries()) {
+		const isMissingSomewhere = count < totalObjects;
+		const isEmptySomewhere = fieldEverEmpty.get(fieldName) || false;
+		fieldMetadata.set(fieldName, { isOptional: isMissingSomewhere || isEmptySomewhere });
+	}
+	return fieldMetadata;
+}
 
@@ -98,9 +122,15 @@ //#endregion
 	for (const field of fields) if (field.type === "array" && field.nested && isJsonArray(field.nested)) {
-		const nestedSchema = generateNestedArraySchema(field.name, field.nested, config);
+		const nestedSchema = generateNestedArraySchema(field.name, field.nested, field.isOptional, config);
 		fieldStrings.push(nestedSchema);
 	} else if (field.type === "object" && field.nested && isJsonObject(field.nested)) {
-		const nestedSchema = generateNestedObjectSchema(field.name, field.nested, config);
+		const nestedSchema = generateNestedObjectSchema(field.name, field.nested, field.isOptional, config);
 		fieldStrings.push(nestedSchema);
-	} else fieldStrings.push(field.name);
-	const fieldsStr = fieldStrings.join(",");
+	} else if (field.type === "primitiveArray") {
+		const fieldName = formatFieldName(field.name, field.isOptional, config);
+		fieldStrings.push(`${fieldName}${config.arraySizeMarker}${config.fieldsOpen}${config.fieldsClose}`);
+	} else {
+		const fieldName = formatFieldName(field.name, field.isOptional, config);
+		fieldStrings.push(fieldName);
+	}
+	const fieldsStr = fieldStrings.join(config.schemaFieldSeparator);
 	return `${schemaOpen}${name}${arraySizeMarker}${count}${schemaClose}${fieldsOpen}${fieldsStr}${fieldsClose}`;
@@ -111,15 +141,23 @@ }
 */
-function generateNestedArraySchema(name, array, config) {
+function generateNestedArraySchema(name, array, isOptional, config) {
 	const { arraySizeMarker, fieldsOpen, fieldsClose } = config;
+	const count = array.length;
 	const fields = analyzeArrayFields(array);
 	const fieldStrings = [];
 	for (const field of fields) if (field.type === "array" && field.nested && isJsonArray(field.nested)) {
-		const nestedSchema = generateNestedArraySchema(field.name, field.nested, config);
+		const nestedSchema = generateNestedArraySchema(field.name, field.nested, field.isOptional, config);
 		fieldStrings.push(nestedSchema);
 	} else if (field.type === "object" && field.nested && isJsonObject(field.nested)) {
-		const nestedSchema = generateNestedObjectSchema(field.name, field.nested, config);
+		const nestedSchema = generateNestedObjectSchema(field.name, field.nested, field.isOptional, config);
 		fieldStrings.push(nestedSchema);
-	} else fieldStrings.push(field.name);
-	const fieldsStr = fieldStrings.join(",");
-	return `${name}${arraySizeMarker}${fieldsOpen}${fieldsStr}${fieldsClose}`;
+	} else if (field.type === "primitiveArray") {
+		const fieldName = formatFieldName(field.name, field.isOptional, config);
+		fieldStrings.push(`${fieldName}${arraySizeMarker}${fieldsOpen}${fieldsClose}`);
+	} else {
+		const fieldName = formatFieldName(field.name, field.isOptional, config);
+		fieldStrings.push(fieldName);
+	}
+	const fieldsStr = fieldStrings.join(config.schemaFieldSeparator);
+	const formattedName = formatFieldName(name, isOptional, config);
+	return `${formattedName}${arraySizeMarker}${count}${fieldsOpen}${fieldsStr}${fieldsClose}`;
 }
@@ -130,19 +168,55 @@ /**
 */
-function generateNestedObjectSchema(name, obj, config) {
+function generateNestedObjectSchema(name, obj, isOptional, config) {
 	const keys = Object.keys(obj).sort();
-	const fieldStrings = [];
+	const primitiveKeys = [];
+	const requiredComplexKeys = [];
+	const optionalComplexKeys = [];
 	for (const key of keys) {
 		const value = obj[key];
-		if (isJsonArray(value) && value.length > 0) {
-			const nestedSchema = generateNestedArraySchema(key, value, config);
+		if (isJsonArray(value)) {
+			const isPrimArray = value.length === 0 || !isArrayOfObjects(value);
+			if (isPrimArray) primitiveKeys.push(key);
+			else requiredComplexKeys.push(key);
+		} else if (isJsonObject(value) && !isJsonArray(value)) if (Object.keys(value).length === 0) optionalComplexKeys.push(key);
+		else requiredComplexKeys.push(key);
+		else primitiveKeys.push(key);
+	}
+	const fieldStrings = [];
+	for (const key of primitiveKeys) {
+		const value = obj[key];
+		if (isJsonArray(value)) fieldStrings.push(`${key}${config.arraySizeMarker}${config.fieldsOpen}${config.fieldsClose}`);
+		else fieldStrings.push(key);
+	}
+	for (const key of requiredComplexKeys) {
+		const value = obj[key];
+		if (isJsonArray(value)) {
+			const nestedSchema = generateNestedArraySchema(key, value, false, config);
 			fieldStrings.push(nestedSchema);
 		} else if (isJsonObject(value) && !isJsonArray(value)) {
-			const nestedSchema = generateNestedObjectSchema(key, value, config);
+			const nestedSchema = generateNestedObjectSchema(key, value, false, config);
 			fieldStrings.push(nestedSchema);
-		} else fieldStrings.push(key);
+		}
 	}
-	const fieldsStr = fieldStrings.join(",");
-	return `${name}{${fieldsStr}}`;
+	for (const key of optionalComplexKeys) {
+		const value = obj[key];
+		if (isJsonArray(value)) {
+			const nestedSchema = generateNestedArraySchema(key, value, true, config);
+			fieldStrings.push(nestedSchema);
+		} else if (isJsonObject(value) && !isJsonArray(value)) {
+			const nestedSchema = generateNestedObjectSchema(key, value, true, config);
+			fieldStrings.push(nestedSchema);
+		}
+	}
+	const fieldsStr = fieldStrings.join(config.schemaFieldSeparator);
+	const formattedName = formatFieldName(name, isOptional, config);
+	return `${formattedName}{${fieldsStr}}`;
 }
 /**
+* Format field name with optional marker suffix if needed
+*/
+function formatFieldName(name, isOptional, config) {
+	if (isOptional) return `${name}${config.optionalFieldMarker}`;
+	return name;
+}
+/**
 * Analyze array to determine fields and nested arrays/objects
@@ -154,4 +228,7 @@ */
 		const keys = getAllKeys(array);
+		const fieldMetadata = analyzeFields(array);
 		const fields = [];
 		for (const key of keys) {
+			const metadata = fieldMetadata.get(key);
+			const isOptional = metadata?.isOptional ?? false;
 			const nestedArray = findNestedArray(array, key);
@@ -162,2 +239,3 @@ if (nestedArray) {
 					type: "array",
+					isOptional,
 					nested: nestedArray
@@ -172,2 +250,3 @@ });
 					type: "object",
+					isOptional,
 					nested: nestedObject
@@ -177,8 +256,27 @@ });
 			}
+			const firstValue = array.find((obj) => key in obj)?.[key];
+			if (isJsonArray(firstValue)) {
+				const isPrimArray = firstValue.length === 0 || !isArrayOfObjects(firstValue);
+				if (isPrimArray) {
+					fields.push({
+						name: key,
+						type: "primitiveArray",
+						isOptional
+					});
+					continue;
+				}
+			}
 			fields.push({
 				name: key,
-				type: "primitive"
+				type: "primitive",
+				isOptional
 			});
 		}
-		return fields;
+		return fields.sort((a, b) => {
+			const aIsPrimitive = a.type === "primitive" || a.type === "primitiveArray";
+			const bIsPrimitive = b.type === "primitive" || b.type === "primitiveArray";
+			if (aIsPrimitive !== bIsPrimitive) return aIsPrimitive ? -1 : 1;
+			if (a.isOptional !== b.isOptional) return a.isOptional ? 1 : -1;
+			return 0;
+		});
 	}
@@ -189,8 +287,21 @@ return [];
 * Find nested array for a given key across all objects
+* Only returns array if MAJORITY of present values are arrays (handles type inconsistencies)
 */
 function findNestedArray(objects, key) {
+	let arrayCount = 0;
+	let primitiveCount = 0;
+	let objectCount = 0;
+	let foundArray;
 	for (const obj of objects) {
+		if (!(key in obj)) continue;
 		const value = obj[key];
-		if (isJsonArray(value) && value.length > 0) return value;
+		if (isJsonArray(value)) {
+			arrayCount++;
+			if (!foundArray || foundArray.length === 0 && value.length > 0) foundArray = value;
+		} else if (isJsonObject(value)) objectCount++;
+		else primitiveCount++;
 	}
+	if (arrayCount > primitiveCount && arrayCount > objectCount) {
+		if (foundArray && foundArray.length > 0 && isArrayOfObjects(foundArray)) return foundArray;
+	}
 	return void 0;
@@ -200,8 +311,19 @@ }
 * Find nested object for a given key across all objects
+* Only returns object if MAJORITY of present values are objects (handles type inconsistencies)
 */
 function findNestedObject(objects, key) {
+	let arrayCount = 0;
+	let primitiveCount = 0;
+	let objectCount = 0;
+	let foundObject;
 	for (const obj of objects) {
+		if (!(key in obj)) continue;
 		const value = obj[key];
-		if (isJsonObject(value) && !isJsonArray(value)) return value;
+		if (isJsonObject(value) && !isJsonArray(value)) {
+			objectCount++;
+			if (!foundObject || Object.keys(foundObject).length === 0 && Object.keys(value).length > 0) foundObject = value;
+		} else if (isJsonArray(value)) arrayCount++;
+		else primitiveCount++;
 	}
+	if (objectCount > primitiveCount && objectCount > arrayCount) return foundObject;
 	return void 0;
@@ -293,2 +415,3 @@ }
 	else escaped = escaped.replace(new RegExp(escapeRegex(recordSeparator), "g"), `${escapeChar}${recordSeparator}`);
+	escaped = escaped.replace(/,/g, `${escapeChar},`);
 	return escaped;
@@ -305,2 +428,3 @@ }
 	else unescaped = unescaped.replace(new RegExp(`\\${escapeChar}${escapeRegex(recordSeparator)}`, "g"), recordSeparator);
+	unescaped = unescaped.replace(new RegExp(`\\${escapeChar},`, "g"), ",");
 	unescaped = unescaped.replace(new RegExp(`\\${escapeChar}\\${escapeChar}`, "g"), escapeChar);
@@ -312,5 +436,5 @@ return unescaped;
 */
-function formatValue(value, config) {
+function formatValue(value, config, isOptional) {
 	if (value === null) return "null";
-	if (value === void 0) return "null";
+	if (value === void 0) return isOptional ? "" : "null";
 	const str = String(value);
@@ -320,2 +444,16 @@ return escapeValue(str, config);
 /**
+* Format a primitive array for inline encoding (comma-separated)
+* Handles null values and respects preserveEmptyFields config
+*/
+function formatPrimitiveArray(array, config, preserveEmpty) {
+	let elements = array.map((item) => {
+		if (item === null) return "";
+		if (item === void 0) return "";
+		const str = String(item);
+		return escapeValue(str, config);
+	});
+	if (!preserveEmpty) elements = elements.filter((el) => el !== "");
+	return elements.join(",");
+}
+/**
 * Escape regex special characters
@@ -328,2 +466,3 @@ */
 * Split a string by delimiter, respecting escapes
+* Only unescapes the delimiter and escape char itself, preserves other escape sequences
 */
@@ -335,4 +474,10 @@ function splitEscaped(str, delimiter, escapeChar) {
 	while (i < str.length) if (str[i] === escapeChar && i + 1 < str.length) {
-		current += str[i + 1];
-		i += 2;
+		const nextChar = str[i + 1];
+		if (nextChar === delimiter || nextChar === escapeChar) {
+			current += nextChar;
+			i += 2;
+		} else {
+			current += str[i] + nextChar;
+			i += 2;
+		}
 	} else if (str.slice(i, i + delimiter.length) === delimiter) {
@@ -379,15 +524,62 @@ parts.push(current);
 	const records = [];
-	if (!isArrayOfObjects(array)) return records;
+	if (!isArrayOfObjects(array)) {
+		const { fieldDelimiter } = config;
+		array.forEach((value, index) => {
+			const itemIndex = index + 1;
+			pathWriter.push(itemIndex);
+			const path = pathWriter.getCurrentPath();
+			const formattedValue = formatValue(value, config);
+			const record = `${path}${fieldDelimiter}${formattedValue}`;
+			records.push(record);
+			pathWriter.pop();
+		});
+		return records;
+	}
 	const keys = getAllKeys(array);
+	const fieldMetadata = analyzeFields(array);
+	const fields = keys.map((key) => {
+		const value = array.find((obj) => obj[key] !== void 0)?.[key];
+		const isOptional = fieldMetadata.get(key)?.isOptional ?? false;
+		if (isJsonArray(value)) if (value.length > 0 && isArrayOfObjects(value)) return {
+			name: key,
+			type: "array",
+			isOptional,
+			nested: value
+		};
+		else return {
+			name: key,
+			type: "primitiveArray",
+			isOptional
+		};
+		else if (isJsonObject(value) && !isJsonArray(value)) return {
+			name: key,
+			type: "object",
+			isOptional,
+			nested: value
+		};
+		else return {
+			name: key,
+			type: "primitive",
+			isOptional
+		};
+	});
+	fields.sort((a, b) => {
+		const aIsPrimitive = a.type === "primitive" || a.type === "primitiveArray";
+		const bIsPrimitive = b.type === "primitive" || b.type === "primitiveArray";
+		if (aIsPrimitive !== bIsPrimitive) return aIsPrimitive ? -1 : 1;
+		if (a.isOptional !== b.isOptional) return a.isOptional ? 1 : -1;
+		return 0;
+	});
 	array.forEach((obj, index) => {
 		const itemIndex = index + 1;
 		pathWriter.push(itemIndex);
-		const record = encodeObject(obj, keys, pathWriter, config);
+		const record = encodeObject(obj, fields, pathWriter, config);
 		records.push(record);
-		for (const key of keys) {
-			const value = obj[key];
-			if (isJsonArray(value) && value.length > 0) {
+		for (const field of fields) {
+			if (field.type === "primitive") continue;
+			const value = obj[field.name];
+			if (field.type === "array" && isJsonArray(value)) {
 				const nestedRecords = encodeArray(value, pathWriter, config);
 				records.push(...nestedRecords);
-			} else if (isJsonObject(value) && !isJsonArray(value)) {
+			} else if (field.type === "object" && isJsonObject(value) && !isJsonArray(value)) {
 				const nestedRecords = encodeNestedObject(value, pathWriter, config);
@@ -404,12 +596,51 @@ records.push(...nestedRecords);
 */
-function encodeObject(obj, keys, pathWriter, config) {
+function encodeObject(obj, fields, pathWriter, config) {
 	const { fieldDelimiter } = config;
 	const path = pathWriter.getCurrentPath();
 	const values = [path];
-	for (const key of keys) {
-		const value = obj[key];
-		if (isJsonArray(value)) continue;
-		if (isJsonObject(value) && !isJsonArray(value)) continue;
-		values.push(formatValue(value, config));
+	const primitiveFields = fields.filter((f) => f.type === "primitive" || f.type === "primitiveArray");
+	const complexFields = fields.filter((f) => f.type === "array" || f.type === "object");
+	for (const field of primitiveFields) {
+		const value = obj[field.name];
+		if (field.isOptional && !(field.name in obj)) values.push("");
+		else if (field.type === "primitiveArray" && Array.isArray(value)) if (value.length === 0) values.push(",");
+		else values.push(formatPrimitiveArray(value, config, config.preserveEmptyFields));
+		else values.push(formatValue(value, config, false));
 	}
+	/**
+	* Optional Complex Field Marker Strategy:
+	*
+	* Optional arrays/objects are placed at the END of the schema in alphabetical order.
+	* Their data comes as child records, but we need to tell the decoder which fields
+	* are absent vs present.
+	*
+	* Encoding rules:
+	* 1. Find the LAST optional field that has data (non-empty or non-absent)
+	* 2. For each optional field UP TO that last one:
+	*    - If ABSENT or EMPTY: write || marker
+	*    - If HAS DATA: write nothing (data comes as child records)
+	* 3. After the last non-empty field: stop (trailing absent fields omitted)
+	*
+	* Example with fields [_collections?, collections?, is_yalla?, variants?]:
+	*   Hit has collections=38, is_yalla=2  (no _collections, no variants)
+	*   → Last non-empty is is_yalla at index 2
+	*   → Write: || (for _collections) then stop
+	*   → Child records: 38 for collections, 2 for is_yalla
+	*/
+	let lastPresentOptionalIndex = -1;
+	for (let i = 0; i < complexFields.length; i++) {
+		const field = complexFields[i];
+		if (!field.isOptional) continue;
+		if (field.name in obj) lastPresentOptionalIndex = i;
+	}
+	for (let i = 0; i < complexFields.length; i++) {
+		const field = complexFields[i];
+		if (!field.isOptional) continue;
+		if (i > lastPresentOptionalIndex) break;
+		if (field.name in obj) {
+			const value = obj[field.name];
+			const isEmpty = field.type === "array" && isJsonArray(value) && value.length === 0 || field.type === "object" && isJsonObject(value) && !isJsonArray(value) && Object.keys(value).length === 0;
+			if (isEmpty) values.push("");
+		}
+	}
 	return values.join(fieldDelimiter);
@@ -428,13 +659,41 @@ }
 	const values = [path];
+	const primitiveKeys = [];
+	const complexKeys = [];
 	for (const key of keys) {
 		const value = obj[key];
-		if (isJsonArray(value) || isJsonObject(value) && !isJsonArray(value)) continue;
-		values.push(formatValue(value, config));
+		if (isJsonArray(value)) {
+			const isPrimArray = value.length === 0 || !isArrayOfObjects(value);
+			if (isPrimArray) primitiveKeys.push(key);
+			else complexKeys.push(key);
+		} else if (isJsonObject(value) && !isJsonArray(value)) complexKeys.push(key);
+		else primitiveKeys.push(key);
 	}
+	for (const key of primitiveKeys) {
+		const value = obj[key];
+		if (isJsonArray(value)) if (value.length === 0) values.push(",");
+		else values.push(formatPrimitiveArray(value, config, config.preserveEmptyFields));
+		else values.push(formatValue(value, config));
+	}
+	let lastNonEmptyIndex = -1;
+	for (let i = 0; i < complexKeys.length; i++) {
+		const key = complexKeys[i];
+		const value = obj[key];
+		const isEmpty = isJsonArray(value) && value.length === 0 || isJsonObject(value) && !isJsonArray(value) && Object.keys(value).length === 0;
+		if (!isEmpty) lastNonEmptyIndex = i;
+	}
+	for (let i = 0; i <= lastNonEmptyIndex; i++) {
+		const key = complexKeys[i];
+		const value = obj[key];
+		const isEmpty = isJsonArray(value) && value.length === 0 || isJsonObject(value) && !isJsonArray(value) && Object.keys(value).length === 0;
+		if (isEmpty) values.push("");
+	}
 	records.push(values.join(fieldDelimiter));
 	for (const key of keys) {
 		const value = obj[key];
-		if (isJsonArray(value) && value.length > 0) {
-			const nestedRecords = encodeArray(value, pathWriter, config);
-			records.push(...nestedRecords);
+		if (isJsonArray(value)) {
+			const isPrimArray = value.length === 0 || !isArrayOfObjects(value);
+			if (!isPrimArray) {
+				const nestedRecords = encodeArray(value, pathWriter, config);
+				records.push(...nestedRecords);
+			}
 		} else if (isJsonObject(value) && !isJsonArray(value)) {
@@ -464,3 +723,7 @@ const nestedRecords = encodeNestedObject(value, pathWriter, config);
 	fieldsClose: ")",
-	nestedSeparator: "|"
+	nestedSeparator: "|",
+	schemaFieldSeparator: "|",
+	schemaWhitespace: " ",
+	optionalFieldMarker: "?",
+	preserveEmptyFields: true
 };
@@ -530,7 +793,10 @@ /**
 		"fieldsClose",
-		"nestedSeparator"
+		"nestedSeparator",
+		"schemaFieldSeparator",
+		"schemaWhitespace",
+		"optionalFieldMarker"
 	];
 	for (const field of singleCharFields) {
 		const value = config[field];
-		if (value.length !== 1) errors.push(`${field} must be exactly 1 character, got: "${value}"`);
+		if (typeof value === "string" && value.length !== 1) errors.push(`${field} must be exactly 1 character, got: "${value}"`);
 	}
@@ -552,2 +818,5 @@ if (config.recordSeparator.length === 0) errors.push("recordSeparator cannot be empty");
 	addChar(config.nestedSeparator, "nestedSeparator");
+	addChar(config.schemaFieldSeparator, "schemaFieldSeparator");
+	addChar(config.schemaWhitespace, "schemaWhitespace");
+	addChar(config.optionalFieldMarker, "optionalFieldMarker");
 	for (const [char, fields] of chars.entries()) if (fields.length > 1) errors.push(`Character "${char}" is used for multiple purposes: ${fields.join(", ")}`);
@@ -667,3 +936,3 @@ if (errors.length > 0) throw new Error(`Invalid PLOON configuration:\n${errors.join("\n")}`);
 	while (pos < fieldsStr.length) {
-		while (pos < fieldsStr.length && fieldsStr[pos] === " ") pos++;
+		while (pos < fieldsStr.length && fieldsStr[pos] === config.schemaWhitespace) pos++;
 		if (pos >= fieldsStr.length) break;
@@ -673,3 +942,3 @@ const { field, nextPos } = parseNextField(fieldsStr, pos, config);
 		pos = nextPos;
-		while (pos < fieldsStr.length && (fieldsStr[pos] === "," || fieldsStr[pos] === " ")) pos++;
+		while (pos < fieldsStr.length && (fieldsStr[pos] === config.schemaFieldSeparator || fieldsStr[pos] === config.schemaWhitespace)) pos++;
 	}
@@ -683,3 +952,3 @@ return fields;
 function parseNextField(str, start, config) {
-	const { arraySizeMarker, fieldsOpen, fieldsClose } = config;
+	const { arraySizeMarker, fieldsOpen, fieldsClose, schemaFieldSeparator } = config;
 	let depth = 0;
@@ -693,3 +962,3 @@ let braceDepth = 0;
 		else if (str[pos] === "}") braceDepth--;
-		else if (str[pos] === "," && depth === 0 && braceDepth === 0) break;
+		else if (str[pos] === schemaFieldSeparator && depth === 0 && braceDepth === 0) break;
 		pos++;
@@ -709,8 +978,11 @@ }
 *   - Array: "colors#(name,hex)"
+*   - Array (optional): "colors?#(name,hex)"
 *   - Object: "customer{id,name,address{street,city}}"
+*   - Object (optional): "customer?{id,name}"
+*   - Primitive (optional): "email?"
 */
 function parseFieldDefinition(fieldStr, config) {
-	const { arraySizeMarker, fieldsOpen, fieldsClose } = config;
-	const arrayMarkerIndex = fieldStr.indexOf(arraySizeMarker + fieldsOpen);
-	const objectBraceIndex = fieldStr.indexOf("{");
+	const { arraySizeMarker, fieldsOpen, fieldsClose, optionalFieldMarker } = config;
+	let arrayMarkerIndex = fieldStr.indexOf(arraySizeMarker);
+	let objectBraceIndex = fieldStr.indexOf("{");
 	let hasArray = arrayMarkerIndex !== -1;
@@ -720,3 +992,24 @@ let hasObject = objectBraceIndex !== -1;
 	else hasArray = false;
+	let isOptional = false;
+	let fieldStrWithoutMarker = fieldStr;
 	if (hasObject) {
+		const beforeBrace = fieldStr.slice(0, objectBraceIndex);
+		if (beforeBrace.endsWith(optionalFieldMarker)) {
+			isOptional = true;
+			fieldStrWithoutMarker = beforeBrace.slice(0, -optionalFieldMarker.length) + fieldStr.slice(objectBraceIndex);
+		}
+	} else if (hasArray) {
+		const beforeMarker = fieldStr.slice(0, arrayMarkerIndex);
+		if (beforeMarker.endsWith(optionalFieldMarker)) {
+			isOptional = true;
+			fieldStrWithoutMarker = beforeMarker.slice(0, -optionalFieldMarker.length) + fieldStr.slice(arrayMarkerIndex);
+		}
+	} else if (fieldStr.endsWith(optionalFieldMarker)) {
+		isOptional = true;
+		fieldStrWithoutMarker = fieldStr.slice(0, -optionalFieldMarker.length);
+	}
+	fieldStr = fieldStrWithoutMarker;
+	if (hasArray) arrayMarkerIndex = fieldStr.indexOf(arraySizeMarker);
+	if (hasObject) objectBraceIndex = fieldStr.indexOf("{");
+	if (hasObject) {
 		const name = fieldStr.slice(0, objectBraceIndex).trim();
@@ -731,2 +1024,3 @@ const closePos = findMatchingCloseBrace(fieldStr, objectBraceIndex);
 			isObject: true,
+			isOptional,
 			nested: {
@@ -741,7 +1035,15 @@ rootName: name,
 		const name = fieldStr.slice(0, arrayMarkerIndex).trim();
-		const openPos = arrayMarkerIndex + arraySizeMarker.length;
+		let openPos = arrayMarkerIndex + arraySizeMarker.length;
+		let count = 0;
+		const openParenPos = fieldStr.indexOf(fieldsOpen, openPos);
+		if (openParenPos !== -1 && openParenPos > openPos) {
+			const countStr = fieldStr.slice(openPos, openParenPos);
+			count = parseInt(countStr, 10) || 0;
+			openPos = openParenPos;
+		}
 		const closePos = findMatchingCloseParen(fieldStr, openPos, fieldsOpen, fieldsClose);
 		if (closePos === -1) throw new Error(`Unmatched parentheses in field: ${fieldStr}`);
-		const nestedFieldsStr = fieldStr.slice(openPos + 1, closePos);
+		const nestedFieldsStr = fieldStr.slice(openPos + 1, closePos).trim();
 		const nestedFields = parseFields(nestedFieldsStr, config);
+		const isPrimitiveArray = nestedFields.length === 0;
 		return {
@@ -751,5 +1053,7 @@ name,
 			isObject: false,
+			isPrimitiveArray,
+			isOptional,
 			nested: {
 				rootName: name,
-				count: 0,
+				count,
 				fields: nestedFields
@@ -762,3 +1066,4 @@ }
 		isArray: false,
-		isObject: false
+		isObject: false,
+		isOptional
 	};
@@ -823,3 +1128,3 @@ }
 /**
-* Reconstruct object tree from records using depth-first traversal
+* Reconstruct object tree from records using BFS (level-by-level)
 */
@@ -830,61 +1135,159 @@ function reconstruct(records, schema, config) {
 }
+function buildRoutingMap(childRecords, allFields, presenceMarkers, pathSeparator) {
+	const groups = [];
+	let currentBatch = [];
+	let lastIndex = -1;
+	let lastWasObject = false;
+	let fieldIdx = 0;
+	for (const item of childRecords) {
+		const { record } = item;
+		const parsed = parsePath(record.path, pathSeparator);
+		const currentIndex = parsed.index !== void 0 ? parsed.index : 0;
+		const isObject = parsed.isObject;
+		const fieldBoundary = currentBatch.length > 0 && currentIndex <= lastIndex || currentBatch.length > 0 && isObject !== lastWasObject || lastWasObject && isObject;
+		if (fieldBoundary) if (fieldIdx < allFields.length) {
+			const field = allFields[fieldIdx];
+			groups.push({
+				fieldName: field.name,
+				fieldIndex: fieldIdx,
+				isArray: field.isArray,
+				isObject: field.isObject,
+				records: currentBatch.map((it) => it.record),
+				recordIndices: currentBatch.map((it) => it.index)
+			});
+			fieldIdx++;
+			currentBatch = [];
+			if (fieldIdx >= allFields.length) break;
+		} else break;
+		currentBatch.push(item);
+		lastIndex = currentIndex;
+		lastWasObject = isObject;
+	}
+	if (currentBatch.length > 0 && fieldIdx < allFields.length) {
+		const field = allFields[fieldIdx];
+		groups.push({
+			fieldName: field.name,
+			fieldIndex: fieldIdx,
+			isArray: field.isArray,
+			isObject: field.isObject,
+			records: currentBatch.map((it) => it.record),
+			recordIndices: currentBatch.map((it) => it.index)
+		});
+	}
+	return groups;
+}
 /**
-* Build tree structure from flat records
-* Handles both array elements (depth:index) and nested objects (depth )
+* Pre-compute parent-child relationships for all records
+* Returns a map from record index to array of child record indices
 */
+function buildParentChildMap(records, pathSeparator) {
+	const parentChildMap = /* @__PURE__ */ new Map();
+	for (let i = 0; i < records.length; i++) {
+		const record = records[i];
+		const parsed = parsePath(record.path, pathSeparator);
+		const myDepth = parsed.depth;
+		const children = [];
+		for (let j = i + 1; j < records.length; j++) {
+			const childRecord = records[j];
+			const childParsed = parsePath(childRecord.path, pathSeparator);
+			const childDepth = childParsed.depth;
+			if (childDepth === myDepth + 1) children.push(j);
+			else if (childDepth <= myDepth) break;
+		}
+		parentChildMap.set(i, children);
+	}
+	return parentChildMap;
+}
+/**
+* Build tree structure from flat records using BFS (Breadth-First Search)
+* Process level by level with sequential consumption of children
+*/
 function buildTree(records, schema, config) {
 	const { pathSeparator } = config;
 	if (records.length === 0) return [];
+	const parentChildMap = buildParentChildMap(records, pathSeparator);
+	const recordsByDepth = /* @__PURE__ */ new Map();
+	for (const record of records) {
+		const parsed = parsePath(record.path, pathSeparator);
+		if (!recordsByDepth.has(parsed.depth)) recordsByDepth.set(parsed.depth, []);
+		recordsByDepth.get(parsed.depth).push(record);
+	}
 	const rootItems = [];
-	const stack = [];
-	for (const record of records) {
-		const parsedPath = parsePath(record.path, pathSeparator);
-		const { depth, isObject } = parsedPath;
-		while (stack.length >= depth) stack.pop();
-		if (isObject) {
-			const parentFields = depth === 1 ? schema.fields : stack[stack.length - 1]?.fields || [];
-			const parentObj = depth === 1 ? null : stack[stack.length - 1]?.obj;
-			const objectField = parentFields.find((f) => {
-				if (!f.isObject) return false;
-				if (parentObj) {
-					const currentValue = parentObj[f.name];
-					return !currentValue || typeof currentValue === "object" && Object.keys(currentValue).length === 0;
-				}
-				return true;
-			});
-			if (!objectField || !objectField.nested) continue;
-			const obj = createObjectFromRecord(record, objectField.nested.fields, config);
-			if (depth === 1) rootItems.push(obj);
-			else {
-				const parent = stack[stack.length - 1];
-				if (parent) parent.obj[objectField.name] = obj;
-			}
-			const arrayField = objectField.nested.fields.find((f) => f.isArray);
-			const childArray = arrayField ? obj[arrayField.name] : void 0;
-			const arrayNestedFields = arrayField?.nested?.fields || [];
-			stack.push({
+	const depth1Records = recordsByDepth.get(1) || [];
+	const currentLevelParents = [];
+	for (let i = 0; i < records.length; i++) {
+		const record = records[i];
+		const parsed = parsePath(record.path, pathSeparator);
+		if (parsed.depth === 1 && !parsed.isObject) {
+			const { obj } = createObjectFromRecord(record, schema.fields, config);
+			rootItems.push(obj);
+			currentLevelParents.push({
 				obj,
-				fields: objectField.nested.fields,
-				arrayFields: arrayNestedFields,
-				childArray
+				fields: schema.fields,
+				recordIndex: i
 			});
-		} else {
-			const fields = depth === 1 ? schema.fields : stack[stack.length - 1]?.arrayFields || getFieldsForDepth(stack, depth);
-			const obj = createObjectFromRecord(record, fields, config);
-			if (depth === 1) rootItems.push(obj);
-			else {
-				const parent = stack[stack.length - 1];
-				if (parent?.childArray) parent.childArray.push(obj);
+		}
+	}
+	let currentDepth = 1;
+	let parentsAtCurrentDepth = currentLevelParents;
+	while (parentsAtCurrentDepth.length > 0) {
+		const childDepth = currentDepth + 1;
+		const nextLevelParents = [];
+		for (const parent of parentsAtCurrentDepth) {
+			const { obj: parentObj, fields: parentFields, recordIndex: parentIdx } = parent;
+			const presenceMarkers = parentObj.__presenceMarkers__ || /* @__PURE__ */ new Set();
+			const childIndices = parentChildMap.get(parentIdx) || [];
+			const myChildrenWithIndices = childIndices.map((idx) => ({
+				record: records[idx],
+				index: idx
+			})).filter((item) => {
+				const parsed = parsePath(item.record.path, pathSeparator);
+				return parsed.depth === childDepth;
+			});
+			if (myChildrenWithIndices.length === 0) continue;
+			const complexFields = parentFields.filter((f) => !f.isPrimitiveArray && (f.isArray || f.isObject)).filter((f) => !presenceMarkers.has(f.name));
+			const routingMap = buildRoutingMap(myChildrenWithIndices, complexFields, presenceMarkers, pathSeparator);
+			if (routingMap.length === 0) continue;
+			for (const group of routingMap) {
+				const field = complexFields[group.fieldIndex];
+				if (group.isArray) {
+					if (!(field.name in parentObj)) parentObj[field.name] = [];
+					const targetArray = parentObj[field.name];
+					for (let i = 0; i < group.records.length; i++) {
+						const rec = group.records[i];
+						const recIdx = group.recordIndices[i];
+						const fields = field.nested?.fields || [];
+						if (fields.length === 0) {
+							const value = rec.values.length > 0 ? parseValue(rec.values[0], config) : null;
+							targetArray.push(value);
+						} else {
+							const { obj: arrayItem } = createObjectFromRecord(rec, fields, config);
+							targetArray.push(arrayItem);
+							nextLevelParents.push({
+								obj: arrayItem,
+								fields,
+								recordIndex: recIdx
+							});
+						}
+					}
+				} else if (group.isObject) {
+					if (field.nested && group.records.length > 0) {
+						const fields = field.nested.fields || [];
+						const rec = group.records[0];
+						const recIdx = group.recordIndices[0];
+						const { obj: nestedObj } = createObjectFromRecord(rec, fields, config);
+						parentObj[field.name] = nestedObj;
+						nextLevelParents.push({
+							obj: nestedObj,
+							fields,
+							recordIndex: recIdx
+						});
+					}
+				}
 			}
-			const arrayField = fields.find((f) => f.isArray);
-			const childArray = arrayField ? obj[arrayField.name] : void 0;
-			const arrayNestedFields = arrayField?.nested?.fields || [];
-			stack.push({
-				obj,
-				fields,
-				arrayFields: arrayNestedFields,
-				childArray
-			});
 		}
+		parentsAtCurrentDepth = nextLevelParents;
+		currentDepth++;
 	}
+	for (const item of rootItems) cleanupOptionalFields(item, schema.fields, config);
 	return rootItems;
@@ -894,22 +1297,98 @@ }
 * Create object from record based on schema fields
+* Returns both the object and presence markers for optional empty fields
 */
 function createObjectFromRecord(record, fields, config) {
 	const obj = {};
+	const presenceMarkers = /* @__PURE__ */ new Set();
+	const primitiveFields = fields.filter((f) => f.isPrimitiveArray || !f.isArray && !f.isObject);
+	const complexFields = fields.filter((f) => !f.isPrimitiveArray && (f.isArray || f.isObject));
+	const optionalComplexFields = complexFields.filter((f) => f.isOptional);
 	let valueIndex = 0;
-	for (const field of fields) if (field.isArray) obj[field.name] = [];
-	else if (field.isObject) obj[field.name] = {};
-	else if (valueIndex < record.values.length) {
-		obj[field.name] = parseValue(record.values[valueIndex], config);
+	for (const field of primitiveFields) {
+		if (valueIndex >= record.values.length) break;
+		const rawValue = record.values[valueIndex];
+		if (field.isOptional && rawValue === "") {
+			valueIndex++;
+			continue;
+		}
+		if (field.isPrimitiveArray) if (rawValue === ",") obj[field.name] = [];
+		else {
+			const elements = splitEscaped(rawValue, ",", config.escapeChar);
+			obj[field.name] = elements.map(parseValueFromUnescaped);
+		}
+		else obj[field.name] = parseValue(rawValue, config);
 		valueIndex++;
 	}
-	return obj;
+	/**
+	* Optional Complex Field Marker Decoding:
+	*
+	* When creating an object, we read || markers to determine which optional
+	* fields are absent/empty vs present.
+	*
+	* Decoding rules:
+	* 1. Read markers sequentially for optional complex fields
+	* 2. || marker means: field is absent or empty (create empty array/object)
+	* 3. No marker but more data exists: field has data (will be populated from child records)
+	* 4. No more markers/values: all remaining optional fields are absent (don't create them)
+	*
+	* Example: fields [_collections?, collections?, is_yalla?, variants?]
+	*   Marker: ||
+	*   Child records: 38 records, then 2 records
+	*   → _collections: empty (from marker)
+	*   → collections: populate from 38 child records
+	*   → is_yalla: populate from 2 child records
+	*   → variants: absent (no marker, no data after cutoff)
+	*/
+	let markersRead = 0;
+	let hitDataField = false;
+	for (let i = 0; i < optionalComplexFields.length; i++) {
+		const field = optionalComplexFields[i];
+		if (hitDataField) {
+			obj[field.name] = field.isArray ? [] : {};
+			continue;
+		}
+		if (valueIndex < record.values.length) {
+			const markerValue = record.values[valueIndex];
+			if (markerValue === "") {
+				presenceMarkers.add(field.name);
+				obj[field.name] = field.isArray ? [] : {};
+				valueIndex++;
+				markersRead++;
+				continue;
+			} else {
+				hitDataField = true;
+				obj[field.name] = field.isArray ? [] : {};
+				continue;
+			}
+		} else if (markersRead > 0) {
+			hitDataField = true;
+			obj[field.name] = field.isArray ? [] : {};
+		} else break;
+	}
+	for (const field of complexFields) if (!field.isOptional && !(field.name in obj)) obj[field.name] = field.isArray ? [] : {};
+	if (presenceMarkers.size > 0) obj.__presenceMarkers__ = presenceMarkers;
+	return {
+		obj,
+		presenceMarkers
+	};
 }
 /**
-* Get fields for a given depth from stack context
+* Recursively clean up empty optional fields that have no presence marker
+* This runs AFTER all child records have been processed
 */
-function getFieldsForDepth(stack, depth) {
-	if (depth <= 1 || stack.length === 0) return [];
-	const parentFrame = stack[stack.length - 1];
-	if (!parentFrame) return [];
-	return parentFrame.fields;
+function cleanupOptionalFields(obj, fields, config) {
+	const presenceMarkers = obj.__presenceMarkers__;
+	for (const field of fields) {
+		const value = obj[field.name];
+		if (field.isArray && !field.isPrimitiveArray && Array.isArray(value)) {
+			if (field.nested?.fields) {
+				for (const item of value) if (item && typeof item === "object" && !Array.isArray(item)) cleanupOptionalFields(item, field.nested.fields, config);
+			}
+			if (field.isOptional && value.length === 0 && !presenceMarkers?.has(field.name)) delete obj[field.name];
+		} else if (field.isObject && value && typeof value === "object" && !Array.isArray(value)) {
+			if (field.nested?.fields) cleanupOptionalFields(value, field.nested.fields, config);
+			if (field.isOptional && Object.keys(value).filter((k) => k !== "__presenceMarkers__").length === 0 && !presenceMarkers?.has(field.name)) delete obj[field.name];
+		}
+	}
+	delete obj.__presenceMarkers__;
 }
@@ -921,8 +1400,15 @@ /**
 	const unescaped = unescapeValue(value, config);
-	if (/^-?\d+(\.\d+)?$/.test(unescaped)) return parseFloat(unescaped);
-	if (unescaped === "null") return null;
-	if (unescaped === "true") return true;
-	if (unescaped === "false") return false;
-	return unescaped;
+	return parseValueFromUnescaped(unescaped);
 }
+/**
+* Parse an already-unescaped value to its JSON type
+* Used for primitive array elements that were unescaped by splitEscaped
+*/
+function parseValueFromUnescaped(value) {
+	if (/^-?\d+(\.\d+)?$/.test(value)) return parseFloat(value);
+	if (value === "null") return null;
+	if (value === "true") return true;
+	if (value === "false") return false;
+	return value;
+}
 
@@ -929,0 +1415,0 @@ //#endregion

dist/shared/string-utils.d.ts

@@ -17,6 +17,12 @@ /**
  */
-export declare function formatValue(value: unknown, config: PloonConfig): string;
+export declare function formatValue(value: unknown, config: PloonConfig, isOptional?: boolean): string;
 /**
+ * Format a primitive array for inline encoding (comma-separated)
+ * Handles null values and respects preserveEmptyFields config
+ */
+export declare function formatPrimitiveArray(array: unknown[], config: PloonConfig, preserveEmpty: boolean): string;
+/**
  * Split a string by delimiter, respecting escapes
+ * Only unescapes the delimiter and escape char itself, preserves other escape sequences
  */
 export declare function splitEscaped(str: string, delimiter: string, escapeChar: string): string[];

dist/types.d.ts

@@ -35,2 +35,10 @@ /**
     nestedSeparator: string;
+    /** Schema field separator (separates field names in schema) */
+    schemaFieldSeparator: string;
+    /** Schema whitespace (optional whitespace character in schema) */
+    schemaWhitespace: string;
+    /** Optional field marker (suffix for fields not present in all array elements) */
+    optionalFieldMarker: string;
+    /** Preserve empty arrays/objects with presence markers (fieldname() or fieldname{}) */
+    preserveEmptyFields: boolean;
 }
@@ -69,6 +77,7 @@ /**
  */
-export type FieldType = 'primitive' | 'array' | 'object';
+export type FieldType = 'primitive' | 'primitiveArray' | 'array' | 'object';
 export interface SchemaField {
     name: string;
     type: FieldType;
+    isOptional?: boolean;
     nested?: JsonArray | JsonObject;
@@ -75,0 +84,0 @@ fields?: SchemaField[];

package.json

 {
   "name": "ploon",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "Path-Level Object Oriented Notation - The Most Token-Efficient Format for Nested Hierarchical Data",
@@ -5,0 +5,0 @@ "type": "module",

No alert changes

Improved metrics

Total package byte prevSize

198211

48.21%

Lines of code

2078

33.46%

No dependency changes