@fluidframework/id-compressor - npm Package Compare versions

Comparing version 2.0.0-dev-rc.1.0.0.225277 to 2.0.0-dev-rc.1.0.0.228517

api-report/id-compressor.api.md

@@ -32,2 +32,3 @@ ## API Report File for "@fluidframework/id-compressor"
 export class IdCompressor implements IIdCompressor, IIdCompressorCore {
+    beginGhostSession(ghostSessionId: SessionId, ghostSessionCallback: () => void): void;
     // @deprecated (undocumented)
@@ -94,2 +95,3 @@ static create(logger?: ITelemetryBaseLogger): IIdCompressor & IIdCompressorCore;
 export interface IIdCompressorCore {
+    beginGhostSession(ghostSessionId: SessionId, ghostSessionCallback: () => void): any;
     finalizeCreationRange(range: IdCreationRange): void;
@@ -96,0 +98,0 @@ serialize(withSession: true): SerializedIdCompressorWithOngoingSession;

CHANGELOG.md

 # @fluidframework/id-compressor
 
+## 2.0.0-rc.1.0.0
+
+### Minor Changes
+
+-   id-compressor: Cluster allocation strategy updated ([#19066](https://github.com/microsoft/FluidFramework/issues/19066)) [0c36eb5f53](https://github.com/microsoft/FluidFramework/commits/0c36eb5f539362a8e27982e831a3ffe7999c1478)
+
+    This change adjusts the cluster allocation strategy for ghost sessions to exactly fill the cluster instead of needlessly allocating a large cluster.
+    It will also not make a cluster at all if IDs are not allocated.
+    This change adjusts a computation performed at a consensus point, and thus breaks any sessions collaborating across version numbers.
+    The version for the serialized format has been bumped to 2.0, and 1.0 documents will fail to load with the following error:
+    IdCompressor version 1.0 is no longer supported.
+
 ## 2.0.0-internal.8.0.0
@@ -4,0 +16,0 @@

dist/id-compressor-alpha.d.ts

@@ -68,2 +68,3 @@ import { ITelemetryBaseLogger } from '@fluidframework/core-interfaces';
     private telemetryEagerFinalIdCount;
+    private ongoingGhostSession?;
     private constructor();
@@ -79,2 +80,6 @@ /**
     generateCompressedId(): SessionSpaceCompressedId;
+    /**
+     * {@inheritdoc IIdCompressorCore.beginGhostSession}
+     */
+    beginGhostSession(ghostSessionId: SessionId, ghostSessionCallback: () => void): void;
     private generateNextLocalId;
@@ -94,2 +99,3 @@ takeNextCreationRange(): IdCreationRange;
     static deserialize(serialized: SerializedIdCompressorWithNoSession, newSessionId: SessionId): IdCompressor;
+    private static deserialize2_0;
     equals(other: IdCompressor, includeLocalState: boolean): boolean;
@@ -247,2 +253,23 @@ }
     /**
+     * Run a callback that is performed from the perspective of a special "ghost" session.
+     * Any ids generated by this session will be immediately finalized on the local client as if
+     * they were created by a remote client with `ghostSessionId`.
+     *
+     * *WARNING:* This API requires an external consensus mechanism to safely use:
+     * In an attached container (i.e. multiple clients may have the document loaded), all clients must guarantee that:
+     * - They invoke this API starting from the same finalized creation ranges
+     * - This API is invoked with the same ghost session id
+     * - `ghostSessionCallback` deterministically mints the same number of ids on each client within the ghost session
+     * Failure to meet these requirement will result in divergence across clients and eventual consistency errors.
+     * While the ghost sesion callback is running, IdCompressor does not support serialization.
+     * @remarks This API is primarily intended for data migration scenarios which are able to deterministically transform
+     * data in some format into data in a new format.
+     * The first requirement (that all clients must invoke the API with the same finalized creation ranges) is guaranteed
+     * for this scenario because the data transformation callback occurs at a specific ack within the op stream on all
+     * clients, and clients don't finalize creation ranges for local changes they might have at this point in time.
+     * @param ghostSessionId - The session id that minted ids generated within `ghostSessionCallback` should be attributed to.
+     * @param ghostSessionCallback - Callback which mints ids attributed to the ghost session.
+     */
+    beginGhostSession(ghostSessionId: SessionId, ghostSessionCallback: () => void): any;
+    /**
      * Returns a persistable form of the current state of this `IdCompressor` which can be rehydrated via `IdCompressor.deserialize()`.
@@ -249,0 +276,0 @@ * This includes finalized state as well as un-finalized state and is therefore suitable for use in offline scenarios.

dist/id-compressor-untrimmed.d.ts

@@ -76,2 +76,3 @@ import { ITelemetryBaseLogger } from '@fluidframework/core-interfaces';
     private telemetryEagerFinalIdCount;
+    private ongoingGhostSession?;
     private constructor();
@@ -87,2 +88,6 @@ /**
     generateCompressedId(): SessionSpaceCompressedId;
+    /**
+     * {@inheritdoc IIdCompressorCore.beginGhostSession}
+     */
+    beginGhostSession(ghostSessionId: SessionId, ghostSessionCallback: () => void): void;
     private generateNextLocalId;
@@ -102,2 +107,3 @@ takeNextCreationRange(): IdCreationRange;
     static deserialize(serialized: SerializedIdCompressorWithNoSession, newSessionId: SessionId): IdCompressor;
+    private static deserialize2_0;
     equals(other: IdCompressor, includeLocalState: boolean): boolean;
@@ -255,2 +261,23 @@ }
     /**
+     * Run a callback that is performed from the perspective of a special "ghost" session.
+     * Any ids generated by this session will be immediately finalized on the local client as if
+     * they were created by a remote client with `ghostSessionId`.
+     *
+     * *WARNING:* This API requires an external consensus mechanism to safely use:
+     * In an attached container (i.e. multiple clients may have the document loaded), all clients must guarantee that:
+     * - They invoke this API starting from the same finalized creation ranges
+     * - This API is invoked with the same ghost session id
+     * - `ghostSessionCallback` deterministically mints the same number of ids on each client within the ghost session
+     * Failure to meet these requirement will result in divergence across clients and eventual consistency errors.
+     * While the ghost sesion callback is running, IdCompressor does not support serialization.
+     * @remarks This API is primarily intended for data migration scenarios which are able to deterministically transform
+     * data in some format into data in a new format.
+     * The first requirement (that all clients must invoke the API with the same finalized creation ranges) is guaranteed
+     * for this scenario because the data transformation callback occurs at a specific ack within the op stream on all
+     * clients, and clients don't finalize creation ranges for local changes they might have at this point in time.
+     * @param ghostSessionId - The session id that minted ids generated within `ghostSessionCallback` should be attributed to.
+     * @param ghostSessionCallback - Callback which mints ids attributed to the ghost session.
+     */
+    beginGhostSession(ghostSessionId: SessionId, ghostSessionCallback: () => void): any;
+    /**
      * Returns a persistable form of the current state of this `IdCompressor` which can be rehydrated via `IdCompressor.deserialize()`.
@@ -257,0 +284,0 @@ * This includes finalized state as well as un-finalized state and is therefore suitable for use in offline scenarios.

dist/idCompressor.d.ts

@@ -37,2 +37,3 @@ /*!
     private telemetryEagerFinalIdCount;
+    private ongoingGhostSession?;
     private constructor();
@@ -48,2 +49,6 @@ /**
     generateCompressedId(): SessionSpaceCompressedId;
+    /**
+     * {@inheritdoc IIdCompressorCore.beginGhostSession}
+     */
+    beginGhostSession(ghostSessionId: SessionId, ghostSessionCallback: () => void): void;
     private generateNextLocalId;
@@ -63,2 +68,3 @@ takeNextCreationRange(): IdCreationRange;
     static deserialize(serialized: SerializedIdCompressorWithNoSession, newSessionId: SessionId): IdCompressor;
+    private static deserialize2_0;
     equals(other: IdCompressor, includeLocalState: boolean): boolean;
@@ -65,0 +71,0 @@ }

dist/packageVersion.d.ts

@@ -8,3 +8,3 @@ /*!
 export declare const pkgName = "@fluidframework/id-compressor";
-export declare const pkgVersion = "2.0.0-dev-rc.1.0.0.225277";
+export declare const pkgVersion = "2.0.0-dev-rc.1.0.0.228517";
 //# sourceMappingURL=packageVersion.d.ts.map

dist/types/idCompressor.d.ts

@@ -24,2 +24,23 @@ /*!
     /**
+     * Run a callback that is performed from the perspective of a special "ghost" session.
+     * Any ids generated by this session will be immediately finalized on the local client as if
+     * they were created by a remote client with `ghostSessionId`.
+     *
+     * *WARNING:* This API requires an external consensus mechanism to safely use:
+     * In an attached container (i.e. multiple clients may have the document loaded), all clients must guarantee that:
+     * - They invoke this API starting from the same finalized creation ranges
+     * - This API is invoked with the same ghost session id
+     * - `ghostSessionCallback` deterministically mints the same number of ids on each client within the ghost session
+     * Failure to meet these requirement will result in divergence across clients and eventual consistency errors.
+     * While the ghost sesion callback is running, IdCompressor does not support serialization.
+     * @remarks This API is primarily intended for data migration scenarios which are able to deterministically transform
+     * data in some format into data in a new format.
+     * The first requirement (that all clients must invoke the API with the same finalized creation ranges) is guaranteed
+     * for this scenario because the data transformation callback occurs at a specific ack within the op stream on all
+     * clients, and clients don't finalize creation ranges for local changes they might have at this point in time.
+     * @param ghostSessionId - The session id that minted ids generated within `ghostSessionCallback` should be attributed to.
+     * @param ghostSessionCallback - Callback which mints ids attributed to the ghost session.
+     */
+    beginGhostSession(ghostSessionId: SessionId, ghostSessionCallback: () => void): any;
+    /**
      * Returns a persistable form of the current state of this `IdCompressor` which can be rehydrated via `IdCompressor.deserialize()`.
@@ -26,0 +47,0 @@ * This includes finalized state as well as un-finalized state and is therefore suitable for use in offline scenarios.

package.json

 {
   "name": "@fluidframework/id-compressor",
-  "version": "2.0.0-dev-rc.1.0.0.225277",
+  "version": "2.0.0-dev-rc.1.0.0.228517",
   "description": "ID compressor",
@@ -14,3 +14,3 @@ "homepage": "https://fluidframework.com",
   "sideEffects": false,
-  "main": "dist/index.cjs",
+  "main": "dist/index.js",
   "module": "lib/index.mjs",
@@ -39,7 +39,7 @@ "types": "dist/index.d.ts",
   "dependencies": {
-    "@fluid-internal/client-utils": "2.0.0-dev-rc.1.0.0.225277",
-    "@fluidframework/core-interfaces": "2.0.0-dev-rc.1.0.0.225277",
-    "@fluidframework/core-utils": "2.0.0-dev-rc.1.0.0.225277",
-    "@fluidframework/telemetry-utils": "2.0.0-dev-rc.1.0.0.225277",
-    "sorted-btree": "^1.8.0",
+    "@fluid-internal/client-utils": "2.0.0-dev-rc.1.0.0.228517",
+    "@fluidframework/core-interfaces": "2.0.0-dev-rc.1.0.0.228517",
+    "@fluidframework/core-utils": "2.0.0-dev-rc.1.0.0.228517",
+    "@fluidframework/telemetry-utils": "2.0.0-dev-rc.1.0.0.228517",
+    "@tylerbu/sorted-btree-es6": "^1.8.0",
     "uuid": "^9.0.0"
@@ -49,10 +49,10 @@ },
     "@arethetypeswrong/cli": "^0.13.3",
-    "@fluid-private/stochastic-test-utils": "2.0.0-dev-rc.1.0.0.225277",
+    "@fluid-private/stochastic-test-utils": "2.0.0-dev-rc.1.0.0.228517",
     "@fluid-tools/benchmark": "^0.48.0",
-    "@fluid-tools/build-cli": "0.29.0-222379",
+    "@fluid-tools/build-cli": "^0.29.0",
     "@fluidframework/build-common": "^2.0.3",
-    "@fluidframework/build-tools": "0.29.0-222379",
-    "@fluidframework/eslint-config-fluid": "^3.1.0",
+    "@fluidframework/build-tools": "^0.29.0",
+    "@fluidframework/eslint-config-fluid": "^3.2.0",
     "@fluidframework/id-compressor-previous": "npm:@fluidframework/id-compressor@2.0.0-internal.8.0.0",
-    "@fluidframework/mocha-test-setup": "2.0.0-dev-rc.1.0.0.225277",
+    "@fluidframework/mocha-test-setup": "2.0.0-dev-rc.1.0.0.228517",
     "@microsoft/api-extractor": "^7.38.3",
@@ -99,3 +99,3 @@ "@types/mocha": "^9.1.1",
     "build:genver": "gen-version",
-    "build:test": "tsc-multi --config ./tsc-multi.test.json",
+    "build:test": "tsc --project ./src/test/tsconfig.json",
     "check:are-the-types-wrong": "attw --pack",
@@ -117,5 +117,5 @@ "check:release-tags": "api-extractor run --local --config ./api-extractor-lint.json",
     "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",
-    "tsc": "tsc-multi --config ../../../common/build/build-common/tsc-multi.cjs.json",
+    "tsc": "tsc",
     "tsc:watch": "tsc --watch"
   }
 }

README.md

@@ -33,8 +33,2 @@ # @fluidframework/id-compressor
 
-Note that when depending on a library version of the form `2.0.0-internal.x.y.z`, called the Fluid internal version scheme,
-you must use a `>= <` dependency range (such as `>=2.0.0-internal.x.y.z <2.0.0-internal.w.0.0` where `w` is `x+1`).
-Standard `^` and `~` ranges will not work as expected.
-See the [@fluid-tools/version-tools](https://github.com/microsoft/FluidFramework/blob/main/build-tools/packages/version-tools/README.md)
-package for more information including tools to convert between version schemes.
-
 <!-- prettier-ignore-end -->
@@ -41,0 +35,0 @@

src/idCompressor.ts

@@ -48,2 +48,3 @@ /*!
 	Sessions,
+	lastFinalizedFinal,
 } from "./sessions";
@@ -57,3 +58,3 @@ import { SessionSpaceNormalizer } from "./sessionSpaceNormalizer";
  */
-const currentWrittenVersion = 1;
+const currentWrittenVersion = 2.0;
 
@@ -108,2 +109,4 @@ /**
 	private telemetryEagerFinalIdCount = 0;
+	// The ongoing ghost session, if one exists.
+	private ongoingGhostSession?: { cluster?: IdCluster; ghostSessionId: SessionId };
 
@@ -169,24 +172,52 @@ // #endregion
 	public generateCompressedId(): SessionSpaceCompressedId {
-		this.localGenCount++;
-		const lastCluster = this.localSession.getLastCluster();
-		if (lastCluster === undefined) {
+		// This ghost session code inside this block should not be changed without a version bump (it is performed at a consensus point)
+		if (this.ongoingGhostSession) {
+			if (this.ongoingGhostSession.cluster === undefined) {
+				this.ongoingGhostSession.cluster = this.addEmptyCluster(
+					this.sessions.getOrCreate(this.ongoingGhostSession.ghostSessionId),
+					1,
+				);
+			} else {
+				this.ongoingGhostSession.cluster.capacity++;
+			}
+			this.ongoingGhostSession.cluster.count++;
+			return lastFinalizedFinal(
+				this.ongoingGhostSession.cluster,
+			) as unknown as SessionSpaceCompressedId;
+		} else {
+			this.localGenCount++;
+			const lastCluster = this.localSession.getLastCluster();
+			if (lastCluster === undefined) {
+				this.telemetryLocalIdCount++;
+				return this.generateNextLocalId();
+			}
+
+			// If there exists a cluster of final IDs already claimed by the local session that still has room in it,
+			// it is known prior to range sequencing what a local ID's corresponding final ID will be.
+			// In this case, it is safe to return the final ID immediately. This is guaranteed to be safe because
+			// any op that the local session sends that contains one of those final IDs are guaranteed to arrive to
+			// collaborators *after* the one containing the creation range.
+			const clusterOffset = this.localGenCount - genCountFromLocalId(lastCluster.baseLocalId);
+			if (lastCluster.capacity > clusterOffset) {
+				this.telemetryEagerFinalIdCount++;
+				// Space in the cluster: eager final
+				return ((lastCluster.baseFinalId as number) +
+					clusterOffset) as SessionSpaceCompressedId;
+			}
+			// No space in the cluster, return next local
 			this.telemetryLocalIdCount++;
 			return this.generateNextLocalId();
 		}
+	}
 
-		// If there exists a cluster of final IDs already claimed by the local session that still has room in it,
-		// it is known prior to range sequencing what a local ID's corresponding final ID will be.
-		// In this case, it is safe to return the final ID immediately. This is guaranteed to be safe because
-		// any op that the local session sends that contains one of those final IDs are guaranteed to arrive to
-		// collaborators *after* the one containing the creation range.
-		const clusterOffset = this.localGenCount - genCountFromLocalId(lastCluster.baseLocalId);
-		if (lastCluster.capacity > clusterOffset) {
-			this.telemetryEagerFinalIdCount++;
-			// Space in the cluster: eager final
-			return ((lastCluster.baseFinalId as number) +
-				clusterOffset) as SessionSpaceCompressedId;
+	/**
+	 * {@inheritdoc IIdCompressorCore.beginGhostSession}
+	 */
+	public beginGhostSession(ghostSessionId: SessionId, ghostSessionCallback: () => void) {
+		this.ongoingGhostSession = { ghostSessionId };
+		try {
+			ghostSessionCallback();
+		} finally {
+			this.ongoingGhostSession = undefined;
 		}
-		// No space in the cluster, return next local
-		this.telemetryLocalIdCount++;
-		return this.generateNextLocalId();
 	}
@@ -201,2 +232,6 @@
 	public takeNextCreationRange(): IdCreationRange {
+		assert(
+			!this.ongoingGhostSession,
+			0x8a6 /* IdCompressor should not be operated normally when in a ghost session */,
+		);
 		const count = this.localGenCount - (this.nextRangeBaseGenCount - 1);
@@ -235,2 +270,6 @@ if (count === 0) {
 	public finalizeCreationRange(range: IdCreationRange): void {
+		assert(
+			!this.ongoingGhostSession,
+			0x8a7 /* IdCompressor should not be operated normally when in a ghost session */,
+		);
 		// Check if the range has IDs
@@ -319,2 +358,6 @@ if (range.ids === undefined) {
 	private addEmptyCluster(session: Session, capacity: number): IdCluster {
+		assert(
+			!this.ongoingGhostSession?.cluster,
+			0x8a8 /* IdCompressor should not be operated normally when in a ghost session */,
+		);
 		const newCluster = session.addNewCluster(
@@ -483,2 +526,6 @@ this.finalSpace.getAllocatedIdLimit(),
 	public serialize(hasLocalState: boolean): SerializedIdCompressor {
+		assert(
+			!this.ongoingGhostSession,
+			0x8a9 /* IdCompressor should not be operated normally when in a ghost session */,
+		);
 		const { normalizer, finalSpace, sessions } = this;
@@ -569,3 +616,13 @@ const sessionIndexMap = new Map<Session, number>();
 		const version = readNumber(index);
-		assert(version === currentWrittenVersion, 0x75c /* Unknown serialized version. */);
+		switch (version) {
+			case 1.0:
+				throw new Error("IdCompressor version 1.0 is no longer supported.");
+			case 2.0:
+				return IdCompressor.deserialize2_0(index, sessionId);
+			default:
+				throw new Error("Unknown IdCompressor serialized version.");
+		}
+	}
+
+	private static deserialize2_0(index: Index, sessionId?: SessionId): IdCompressor {
 		const hasLocalState = readBoolean(index);
@@ -572,0 +629,0 @@ const sessionCount = readNumber(index);

src/packageVersion.ts

@@ -9,2 +9,2 @@ /*!
 export const pkgName = "@fluidframework/id-compressor";
-export const pkgVersion = "2.0.0-dev-rc.1.0.0.225277";
+export const pkgVersion = "2.0.0-dev-rc.1.0.0.228517";

src/sessions.ts

@@ -6,3 +6,3 @@ /*!
 
-import BTree from "sorted-btree";
+import { BTree } from "@tylerbu/sorted-btree-es6";
 import { assert } from "@fluidframework/core-utils";
@@ -9,0 +9,0 @@ import { SessionId, StableId } from "./types";

src/types/idCompressor.ts

@@ -32,2 +32,24 @@ /*!
 	/**
+	 * Run a callback that is performed from the perspective of a special "ghost" session.
+	 * Any ids generated by this session will be immediately finalized on the local client as if
+	 * they were created by a remote client with `ghostSessionId`.
+	 *
+	 * *WARNING:* This API requires an external consensus mechanism to safely use:
+	 * In an attached container (i.e. multiple clients may have the document loaded), all clients must guarantee that:
+	 * - They invoke this API starting from the same finalized creation ranges
+	 * - This API is invoked with the same ghost session id
+	 * - `ghostSessionCallback` deterministically mints the same number of ids on each client within the ghost session
+	 * Failure to meet these requirement will result in divergence across clients and eventual consistency errors.
+	 * While the ghost sesion callback is running, IdCompressor does not support serialization.
+	 * @remarks This API is primarily intended for data migration scenarios which are able to deterministically transform
+	 * data in some format into data in a new format.
+	 * The first requirement (that all clients must invoke the API with the same finalized creation ranges) is guaranteed
+	 * for this scenario because the data transformation callback occurs at a specific ack within the op stream on all
+	 * clients, and clients don't finalize creation ranges for local changes they might have at this point in time.
+	 * @param ghostSessionId - The session id that minted ids generated within `ghostSessionCallback` should be attributed to.
+	 * @param ghostSessionCallback - Callback which mints ids attributed to the ghost session.
+	 */
+	beginGhostSession(ghostSessionId: SessionId, ghostSessionCallback: () => void);
+
+	/**
 	 * Returns a persistable form of the current state of this `IdCompressor` which can be rehydrated via `IdCompressor.deserialize()`.
@@ -34,0 +56,0 @@ * This includes finalized state as well as un-finalized state and is therefore suitable for use in offline scenarios.

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

698619

increased by4.53%

Lines of code

6995

increased by3.34%

Worsened metrics

Number of package files

157

decreased by-0.63%

Number of lines in readme file

165

decreased by-3.51%

Dependency changes

• + Added@tylerbu/sorted-btree-es6@^1.8.0

• + Added@fluid-internal/client-utils@2.0.0-dev-rc.1.0.0.228517(transitive)

• + Added@fluidframework/core-interfaces@2.0.0-dev-rc.1.0.0.228517(transitive)

• + Added@fluidframework/core-utils@2.0.0-dev-rc.1.0.0.228517(transitive)

• + Added@fluidframework/telemetry-utils@2.0.0-dev-rc.1.0.0.228517(transitive)

• + Added@tylerbu/sorted-btree-es6@1.8.0(transitive)

• - Removedsorted-btree@^1.8.0

• - Removed@fluid-internal/client-utils@2.0.0-dev-rc.1.0.0.225277(transitive)

• - Removed@fluidframework/core-interfaces@2.0.0-dev-rc.1.0.0.225277(transitive)

• - Removed@fluidframework/core-utils@2.0.0-dev-rc.1.0.0.225277(transitive)

• - Removed@fluidframework/telemetry-utils@2.0.0-dev-rc.1.0.0.225277(transitive)

• - Removedsorted-btree@1.8.1(transitive)

• Updated@fluid-internal/client-utils@2.0.0-dev-rc.1.0.0.228517

• Updated@fluidframework/core-interfaces@2.0.0-dev-rc.1.0.0.228517

• Updated@fluidframework/core-utils@2.0.0-dev-rc.1.0.0.228517

• Updated@fluidframework/telemetry-utils@2.0.0-dev-rc.1.0.0.228517