@dstanesc/o-o-o-o-o-o-o - npm Package Compare versions

Comparing version 0.0.24 to 0.0.27

dist/relay-client.d.ts

@@ -16,2 +16,3 @@ import { BlockStore } from './block-store';
     blocksPush(bytes: Uint8Array): Promise<PlumbingBlocksPushResponse>;
+    blocksPull(links: string[]): Promise<Uint8Array | undefined>;
 }
@@ -34,3 +35,3 @@ type PlumbingStorePushResponse = {
     push(versionStoreRoot: Link, versionRoot?: Link): Promise<BasicPushResponse>;
-    pull(versionStoreId: string): Promise<{
+    pull(versionStoreId: string, localVersionStoreRoot?: Link): Promise<{
         versionStore: VersionStore;
@@ -37,0 +38,0 @@ graphStore: GraphStore;

dist/relay-client.js

@@ -10,3 +10,3 @@ import { blockIndexFactory } from './block-index';
     const plumbing = relayClientPlumbingFactory(config);
-    const { packVersionStore, restoreSingleIndex: restoreVersionStore, packGraphVersion, packRootIndex, packRandomBlocks, restoreGraphVersion, restoreRootIndex, } = graphPackerFactory(linkCodec);
+    const { packVersionStore, restoreSingleIndex: restoreVersionStore, packGraphVersion, packRootIndex, packRandomBlocks, restoreGraphVersion, restoreRootIndex, restoreRandomBlocks, } = graphPackerFactory(linkCodec);
     const push = async (versionStoreRoot, versionRoot) => {
@@ -67,3 +67,3 @@ var _a;
             else {
-                const remoteRootIndexBytes = await plumbing.indexPull(remoteVersionRoot.toString());
+                const remoteRootIndexBytes = await plumbing.indexPull(linkCodec.encodeString(remoteVersionRoot));
                 const { blocks: remoteRootIndexBlocks } = await restoreRootIndex(remoteRootIndexBytes, diffStore);
@@ -116,45 +116,192 @@ const localRootIndexBundle = await packRootIndex(localVersionRoot, blockStore);
     };
-    const pull = async (versionStoreId) => {
-        const storeBytes = await plumbing.storePull(chunkSize, versionStoreId);
-        const memoryStore = memoryBlockStoreFactory();
-        const { root: versionStoreRoot, index: versionStoreIndex, blocks: versionStoreBlocks, } = await restoreVersionStore(storeBytes, memoryStore);
-        const versionStoreTransient = await versionStoreFactory({
-            storeRoot: versionStoreRoot,
-            chunk,
-            linkCodec,
-            valueCodec,
-            blockStore: memoryStore,
-        });
-        const versions = versionStoreTransient.log();
-        for (const version of versions) {
+    const pull = async (versionStoreId, localVersionStoreRoot) => {
+        var _a, _b;
+        let remoteVersionStoreBytes;
+        if (incremental && localVersionStoreRoot !== undefined) {
             try {
-                await blockStore.get(version.root);
+                remoteVersionStoreBytes = await plumbing.storePull(chunkSize, versionStoreId);
             }
-            catch (e) {
-                const graphVersionBytes = await plumbing.graphPull(version.root.toString());
-                if (graphVersionBytes !== undefined) {
-                    await restoreGraphVersion(graphVersionBytes, memoryStore);
+            catch (error) {
+                if (axios.isAxiosError(error)) {
+                    const axiosError = error;
+                    if (((_a = axiosError.response) === null || _a === void 0 ? void 0 : _a.status) !== 404) {
+                        throw error;
+                    }
                 }
             }
+            if (remoteVersionStoreBytes !== undefined) {
+                const diffStore = memoryBlockStoreFactory();
+                const { root: remoteVersionStoreRoot } = await restoreVersionStore(remoteVersionStoreBytes, diffStore);
+                const remoteVersionStore = await versionStoreFactory({
+                    storeRoot: remoteVersionStoreRoot,
+                    chunk,
+                    linkCodec,
+                    valueCodec,
+                    blockStore: diffStore,
+                });
+                const remoteVersionRoot = remoteVersionStore.currentRoot();
+                const localVersionStore = await versionStoreFactory({
+                    storeRoot: localVersionStoreRoot,
+                    chunk,
+                    linkCodec,
+                    valueCodec,
+                    blockStore,
+                });
+                const localVersionRoot = localVersionStore.currentRoot();
+                if (linkCodec.encodeString(localVersionRoot) !==
+                    linkCodec.encodeString(remoteVersionRoot)) {
+                    const remoteRootIndexBytes = await plumbing.indexPull(linkCodec.encodeString(remoteVersionRoot));
+                    const { blocks: remoteRootIndexBlocks } = await restoreRootIndex(remoteRootIndexBytes, diffStore);
+                    const localRootIndexBundle = await packRootIndex(localVersionRoot, blockStore);
+                    const { blocks: localRootIndexBlocks } = await restoreRootIndex(localRootIndexBundle.bytes, diffStore);
+                    const requiredBlockIdentifiers = [];
+                    for (const block of remoteRootIndexBlocks) {
+                        const linkString = linkCodec.encodeString(block.cid);
+                        if (!localRootIndexBlocks
+                            .map((block) => linkCodec.encodeString(block.cid))
+                            .includes(linkString)) {
+                            requiredBlockIdentifiers.push(linkString);
+                        }
+                    }
+                    const blockIndexBuilder = blockIndexFactory({
+                        linkCodec,
+                        blockStore: diffStore,
+                    });
+                    const contentDiff = await blockIndexBuilder.diffRootIndex({
+                        currentRoot: localVersionRoot,
+                        otherRoot: remoteVersionRoot,
+                    });
+                    for (const link of contentDiff.added) {
+                        requiredBlockIdentifiers.push(linkCodec.encodeString(link));
+                    }
+                    const randomBlocksBundle = await plumbing.blocksPull(requiredBlockIdentifiers);
+                    if (randomBlocksBundle !== undefined) {
+                        const selectedBlocks = await restoreRandomBlocks(randomBlocksBundle, diffStore);
+                        const localVersionStoreBundle = await packVersionStore(localVersionStoreRoot, blockStore, chunk, valueCodec);
+                        const { root: storeRootExisting } = await restoreVersionStore(localVersionStoreBundle.bytes, diffStore);
+                        const graphStoreBundleExisting = await packGraphVersion(localVersionRoot, blockStore);
+                        const { root: versionRootExisting } = await restoreGraphVersion(graphStoreBundleExisting.bytes, diffStore);
+                        const versionStoreExisting = await versionStoreFactory({
+                            storeRoot: localVersionStoreRoot,
+                            versionRoot: localVersionRoot,
+                            chunk,
+                            linkCodec,
+                            valueCodec,
+                            blockStore: diffStore,
+                        });
+                        const { root: mergedRoot, index: mergedIndex, blocks: mergedBlocks, } = await versionStoreExisting.mergeVersions(remoteVersionStore);
+                        await diffStore.push(blockStore);
+                        const mergedVersionRoot = versionStoreExisting.currentRoot();
+                        const mergedVersionStoreRoot = versionStoreExisting.versionStoreRoot();
+                        const versionStore = await versionStoreFactory({
+                            storeRoot: mergedVersionStoreRoot,
+                            versionRoot: mergedVersionRoot,
+                            chunk,
+                            linkCodec,
+                            valueCodec,
+                            blockStore,
+                        });
+                        const graphStore = graphStoreFactory({
+                            chunk,
+                            linkCodec,
+                            valueCodec,
+                            blockStore,
+                        });
+                        const graph = new Graph(versionStore, graphStore);
+                        return {
+                            versionStore,
+                            graphStore,
+                            graph,
+                        };
+                    }
+                    else {
+                        throw new Error(`Failed to pull selected blocks: ${JSON.stringify(requiredBlockIdentifiers)}`);
+                    }
+                }
+                else {
+                    const versionStore = await versionStoreFactory({
+                        storeRoot: localVersionStoreRoot,
+                        versionRoot: localVersionRoot,
+                        chunk,
+                        linkCodec,
+                        valueCodec,
+                        blockStore,
+                    });
+                    const graphStore = graphStoreFactory({
+                        chunk,
+                        linkCodec,
+                        valueCodec,
+                        blockStore,
+                    });
+                    const graph = new Graph(versionStore, graphStore);
+                    return {
+                        versionStore,
+                        graphStore,
+                        graph,
+                    };
+                }
+            }
+            else {
+                return undefined;
+            }
         }
-        await memoryStore.push(blockStore);
-        const versionStore = await versionStoreFactory({
-            storeRoot: versionStoreRoot,
-            chunk,
-            linkCodec,
-            valueCodec,
-            blockStore,
-        });
-        const graphStore = graphStoreFactory({
-            chunk,
-            linkCodec,
-            valueCodec,
-            blockStore,
-        });
-        const graph = new Graph(versionStore, graphStore);
-        return {
-            versionStore,
-            graphStore,
-            graph,
-        };
+        else {
+            try {
+                remoteVersionStoreBytes = await plumbing.storePull(chunkSize, versionStoreId);
+            }
+            catch (error) {
+                if (axios.isAxiosError(error)) {
+                    const axiosError = error;
+                    if (((_b = axiosError.response) === null || _b === void 0 ? void 0 : _b.status) !== 404) {
+                        throw error;
+                    }
+                }
+            }
+            if (remoteVersionStoreBytes !== undefined) {
+                const transientStore = memoryBlockStoreFactory();
+                const { root: versionStoreRoot, index: versionStoreIndex, blocks: versionStoreBlocks, } = await restoreVersionStore(remoteVersionStoreBytes, transientStore);
+                const versionStoreTransient = await versionStoreFactory({
+                    storeRoot: versionStoreRoot,
+                    chunk,
+                    linkCodec,
+                    valueCodec,
+                    blockStore: transientStore,
+                });
+                const versions = versionStoreTransient.log();
+                for (const version of versions) {
+                    try {
+                        await blockStore.get(version.root);
+                    }
+                    catch (e) {
+                        const graphVersionBytes = await plumbing.graphPull(version.root.toString());
+                        if (graphVersionBytes !== undefined) {
+                            await restoreGraphVersion(graphVersionBytes, transientStore);
+                        }
+                    }
+                }
+                await transientStore.push(blockStore);
+                const versionStore = await versionStoreFactory({
+                    storeRoot: versionStoreRoot,
+                    chunk,
+                    linkCodec,
+                    valueCodec,
+                    blockStore,
+                });
+                const graphStore = graphStoreFactory({
+                    chunk,
+                    linkCodec,
+                    valueCodec,
+                    blockStore,
+                });
+                const graph = new Graph(versionStore, graphStore);
+                return {
+                    versionStore,
+                    graphStore,
+                    graph,
+                };
+            }
+            else {
+                return undefined;
+            }
+        }
     };
@@ -243,2 +390,16 @@ return { push, pull };
     };
+    const blocksPull = async (links) => {
+        const response = await httpClient.put('/blocks/pull', { links }, {
+            responseType: 'arraybuffer',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+        });
+        if (response.data) {
+            const bytes = new Uint8Array(response.data);
+            return bytes;
+        }
+        else
+            return undefined;
+    };
     return {
@@ -252,2 +413,3 @@ storePush,
         blocksPush,
+        blocksPull,
     };
@@ -254,0 +416,0 @@ };

package.json

 {
     "name": "@dstanesc/o-o-o-o-o-o-o",
     "description": "O-O-O-O-O-O-O is a collection of content addressed persistent data structures",
-    "version": "0.0.24",
+    "version": "0.0.27",
     "homepage": "https://github.com/dstanesc/O-O-O-O-O-O-O",
@@ -6,0 +6,0 @@ "repository": "https://github.com/dstanesc/O-O-O-O-O-O-O",

src/relay-client.ts

@@ -34,2 +34,3 @@ import { blockIndexFactory } from './block-index'
     blocksPush(bytes: Uint8Array): Promise<PlumbingBlocksPushResponse>
+    blocksPull(links: string[]): Promise<Uint8Array | undefined>
 }
@@ -58,3 +59,4 @@
     pull(
-        versionStoreId: string
+        versionStoreId: string,
+        localVersionStoreRoot?: Link
     ): Promise<
@@ -93,2 +95,3 @@ | { versionStore: VersionStore; graphStore: GraphStore; graph: Graph }
         restoreRootIndex,
+        restoreRandomBlocks,
     } = graphPackerFactory(linkCodec)
@@ -165,3 +168,3 @@
                 const remoteRootIndexBytes = await plumbing.indexPull(
-                    remoteVersionRoot.toString()
+                    linkCodec.encodeString(remoteVersionRoot)
                 )
@@ -253,3 +256,4 @@ const { blocks: remoteRootIndexBlocks } =
     const pull = async (
-        versionStoreId: string
+        versionStoreId: string,
+        localVersionStoreRoot?: Link
     ): Promise<
@@ -259,49 +263,262 @@ | { versionStore: VersionStore; graphStore: GraphStore; graph: Graph }
     > => {
-        const storeBytes = await plumbing.storePull(chunkSize, versionStoreId)
-        const memoryStore: MemoryBlockStore = memoryBlockStoreFactory()
-        const {
-            root: versionStoreRoot,
-            index: versionStoreIndex,
-            blocks: versionStoreBlocks,
-        } = await restoreVersionStore(storeBytes, memoryStore)
-        const versionStoreTransient: VersionStore = await versionStoreFactory({
-            storeRoot: versionStoreRoot,
-            chunk,
-            linkCodec,
-            valueCodec,
-            blockStore: memoryStore,
-        })
-        const versions: Version[] = versionStoreTransient.log()
-        for (const version of versions) {
+        let remoteVersionStoreBytes: Uint8Array | undefined
+        if (incremental && localVersionStoreRoot !== undefined) {
             try {
-                await blockStore.get(version.root)
-            } catch (e) {
-                const graphVersionBytes = await plumbing.graphPull(
-                    version.root.toString()
+                remoteVersionStoreBytes = await plumbing.storePull(
+                    chunkSize,
+                    versionStoreId
                 )
-                if (graphVersionBytes !== undefined) {
-                    await restoreGraphVersion(graphVersionBytes, memoryStore)
+            } catch (error) {
+                if (axios.isAxiosError(error)) {
+                    const axiosError: AxiosError = error
+                    if (axiosError.response?.status !== 404) {
+                        throw error
+                    }
                 }
             }
+            if (remoteVersionStoreBytes !== undefined) {
+                const diffStore: MemoryBlockStore = memoryBlockStoreFactory()
+                const { root: remoteVersionStoreRoot } =
+                    await restoreVersionStore(
+                        remoteVersionStoreBytes,
+                        diffStore
+                    )
+                const remoteVersionStore: VersionStore =
+                    await versionStoreFactory({
+                        storeRoot: remoteVersionStoreRoot,
+                        chunk,
+                        linkCodec,
+                        valueCodec,
+                        blockStore: diffStore,
+                    })
+                const remoteVersionRoot: Link = remoteVersionStore.currentRoot()
+                const localVersionStore: VersionStore =
+                    await versionStoreFactory({
+                        storeRoot: localVersionStoreRoot,
+                        chunk,
+                        linkCodec,
+                        valueCodec,
+                        blockStore,
+                    })
+                const localVersionRoot = localVersionStore.currentRoot()
+                if (
+                    linkCodec.encodeString(localVersionRoot) !==
+                    linkCodec.encodeString(remoteVersionRoot)
+                ) {
+                    const remoteRootIndexBytes = await plumbing.indexPull(
+                        linkCodec.encodeString(remoteVersionRoot)
+                    )
+                    const { blocks: remoteRootIndexBlocks } =
+                        await restoreRootIndex(remoteRootIndexBytes, diffStore)
+                    const localRootIndexBundle: Block = await packRootIndex(
+                        localVersionRoot,
+                        blockStore
+                    )
+                    const { blocks: localRootIndexBlocks } =
+                        await restoreRootIndex(
+                            localRootIndexBundle.bytes,
+                            diffStore
+                        )
+                    const requiredBlockIdentifiers: string[] = []
+                    for (const block of remoteRootIndexBlocks) {
+                        const linkString = linkCodec.encodeString(block.cid)
+                        if (
+                            !localRootIndexBlocks
+                                .map((block) =>
+                                    linkCodec.encodeString(block.cid)
+                                )
+                                .includes(linkString)
+                        ) {
+                            requiredBlockIdentifiers.push(linkString)
+                        }
+                    }
+                    const blockIndexBuilder = blockIndexFactory({
+                        linkCodec,
+                        blockStore: diffStore,
+                    })
+
+                    const contentDiff: ContentDiff =
+                        await blockIndexBuilder.diffRootIndex({
+                            currentRoot: localVersionRoot,
+                            otherRoot: remoteVersionRoot,
+                        })
+                    for (const link of contentDiff.added) {
+                        requiredBlockIdentifiers.push(
+                            linkCodec.encodeString(link)
+                        )
+                    }
+                    const randomBlocksBundle: Uint8Array | undefined =
+                        await plumbing.blocksPull(requiredBlockIdentifiers)
+
+                    if (randomBlocksBundle !== undefined) {
+                        const selectedBlocks = await restoreRandomBlocks(
+                            randomBlocksBundle,
+                            diffStore
+                        )
+                        const localVersionStoreBundle: Block =
+                            await packVersionStore(
+                                localVersionStoreRoot,
+                                blockStore,
+                                chunk,
+                                valueCodec
+                            )
+                        const { root: storeRootExisting } =
+                            await restoreVersionStore(
+                                localVersionStoreBundle.bytes,
+                                diffStore
+                            )
+                        const graphStoreBundleExisting: Block =
+                            await packGraphVersion(localVersionRoot, blockStore)
+                        const { root: versionRootExisting } =
+                            await restoreGraphVersion(
+                                graphStoreBundleExisting.bytes,
+                                diffStore
+                            )
+                        const versionStoreExisting: VersionStore =
+                            await versionStoreFactory({
+                                storeRoot: localVersionStoreRoot,
+                                versionRoot: localVersionRoot,
+                                chunk,
+                                linkCodec,
+                                valueCodec,
+                                blockStore: diffStore,
+                            })
+                        const {
+                            root: mergedRoot,
+                            index: mergedIndex,
+                            blocks: mergedBlocks,
+                        } = await versionStoreExisting.mergeVersions(
+                            remoteVersionStore
+                        )
+                        await diffStore.push(blockStore)
+                        const mergedVersionRoot =
+                            versionStoreExisting.currentRoot()
+                        const mergedVersionStoreRoot =
+                            versionStoreExisting.versionStoreRoot()
+                        const versionStore: VersionStore =
+                            await versionStoreFactory({
+                                storeRoot: mergedVersionStoreRoot,
+                                versionRoot: mergedVersionRoot,
+                                chunk,
+                                linkCodec,
+                                valueCodec,
+                                blockStore,
+                            })
+                        const graphStore = graphStoreFactory({
+                            chunk,
+                            linkCodec,
+                            valueCodec,
+                            blockStore,
+                        })
+                        const graph = new Graph(versionStore, graphStore)
+                        return {
+                            versionStore,
+                            graphStore,
+                            graph,
+                        }
+                    } else {
+                        throw new Error(
+                            `Failed to pull selected blocks: ${JSON.stringify(
+                                requiredBlockIdentifiers
+                            )}`
+                        )
+                    }
+                } else {
+                    const versionStore = await versionStoreFactory({
+                        storeRoot: localVersionStoreRoot,
+                        versionRoot: localVersionRoot,
+                        chunk,
+                        linkCodec,
+                        valueCodec,
+                        blockStore,
+                    })
+                    const graphStore = graphStoreFactory({
+                        chunk,
+                        linkCodec,
+                        valueCodec,
+                        blockStore,
+                    })
+                    const graph = new Graph(versionStore, graphStore)
+                    return {
+                        versionStore,
+                        graphStore,
+                        graph,
+                    }
+                }
+            } else {
+                return undefined
+            }
+        } else {
+            try {
+                remoteVersionStoreBytes = await plumbing.storePull(
+                    chunkSize,
+                    versionStoreId
+                )
+            } catch (error) {
+                if (axios.isAxiosError(error)) {
+                    const axiosError: AxiosError = error
+                    if (axiosError.response?.status !== 404) {
+                        throw error
+                    }
+                }
+            }
+            if (remoteVersionStoreBytes !== undefined) {
+                const transientStore: MemoryBlockStore =
+                    memoryBlockStoreFactory()
+                const {
+                    root: versionStoreRoot,
+                    index: versionStoreIndex,
+                    blocks: versionStoreBlocks,
+                } = await restoreVersionStore(
+                    remoteVersionStoreBytes,
+                    transientStore
+                )
+                const versionStoreTransient: VersionStore =
+                    await versionStoreFactory({
+                        storeRoot: versionStoreRoot,
+                        chunk,
+                        linkCodec,
+                        valueCodec,
+                        blockStore: transientStore,
+                    })
+                const versions: Version[] = versionStoreTransient.log()
+                for (const version of versions) {
+                    try {
+                        await blockStore.get(version.root)
+                    } catch (e) {
+                        const graphVersionBytes = await plumbing.graphPull(
+                            version.root.toString()
+                        )
+                        if (graphVersionBytes !== undefined) {
+                            await restoreGraphVersion(
+                                graphVersionBytes,
+                                transientStore
+                            )
+                        }
+                    }
+                }
+                await transientStore.push(blockStore)
+                const versionStore: VersionStore = await versionStoreFactory({
+                    storeRoot: versionStoreRoot,
+                    chunk,
+                    linkCodec,
+                    valueCodec,
+                    blockStore,
+                })
+                const graphStore = graphStoreFactory({
+                    chunk,
+                    linkCodec,
+                    valueCodec,
+                    blockStore,
+                })
+                const graph = new Graph(versionStore, graphStore)
+                return {
+                    versionStore,
+                    graphStore,
+                    graph,
+                }
+            } else {
+                return undefined
+            }
         }
-        await memoryStore.push(blockStore)
-        const versionStore: VersionStore = await versionStoreFactory({
-            storeRoot: versionStoreRoot,
-            chunk,
-            linkCodec,
-            valueCodec,
-            blockStore,
-        })
-        const graphStore = graphStoreFactory({
-            chunk,
-            linkCodec,
-            valueCodec,
-            blockStore,
-        })
-        const graph = new Graph(versionStore, graphStore)
-        return {
-            versionStore,
-            graphStore,
-            graph,
-        }
     }
@@ -424,2 +641,21 @@
 
+    const blocksPull = async (
+        links: string[]
+    ): Promise<Uint8Array | undefined> => {
+        const response: AxiosResponse<ArrayBuffer> = await httpClient.put(
+            '/blocks/pull',
+            { links },
+            {
+                responseType: 'arraybuffer',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+            }
+        )
+        if (response.data) {
+            const bytes = new Uint8Array(response.data)
+            return bytes
+        } else return undefined
+    }
+
     return {
@@ -433,2 +669,3 @@ storePush,
         blocksPush,
+        blocksPull,
     }
@@ -435,0 +672,0 @@ }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

1980198

increased by1.19%

Lines of code

26023

increased by1.55%

No dependency changes