Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More
Socket
Sign inDemoInstall
Socket

@blocksuite/store

Package Overview
Dependencies
Maintainers
5
Versions
1266
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@blocksuite/store - npm Package Compare versions

Comparing version 0.4.0-20230119023850-6877b78 to 0.4.0-20230119151920-9bca215

2

dist/__tests__/test-utils-dom.d.ts

@@ -16,3 +16,3 @@ import type { Workspace } from '../workspace/workspace.js';

export declare function runOnce(): Promise<void>;
export { assertExists } from '@blocksuite/global/utils';
export declare function assertExists<T>(val: T | null | undefined): asserts val is T;
export declare function nextFrame(): Promise<unknown>;

@@ -19,0 +19,0 @@ export declare function loadTestImageBlob(name: string): Promise<Blob>;

@@ -39,3 +39,8 @@ const testResult = {

}
export { assertExists } from '@blocksuite/global/utils';
// XXX: workaround typing issue in blobs/__tests__/test-entry.ts
export function assertExists(val) {
if (val === null || val === undefined) {
throw new Error('val does not exist');
}
}
export async function nextFrame() {

@@ -42,0 +47,0 @@ return new Promise(resolve => requestAnimationFrame(resolve));

@@ -0,4 +1,6 @@

import type { BlobOptionsGetter, BlobOptions } from './providers.js';
import { BlobStorage } from './storage.js';
export declare const getBlobStorage: (workspace?: string, cloudApi?: string) => Promise<BlobStorage | null>;
export declare function getBlobStorage(workspace?: string, optionsGetter?: BlobOptionsGetter): Promise<BlobStorage | null>;
export { BlobStorage } from './storage.js';
export type { BlobOptionsGetter, BlobOptions };
//# sourceMappingURL=index.d.ts.map
import { IndexedDBBlobProvider } from './providers.js';
import { BlobStorage } from './storage.js';
const CLOUD_API = '/api/workspace';
export const getBlobStorage = async (
const CLOUD_ENDPOINT_GETTER = (k) => ({ api: '/api/workspace' }[k]);
export async function getBlobStorage(
// Note: In the current backend design, the workspace id is a randomly generated int64 number
// so if you need to test or enable blob synchronization, the provided workspace needs to be a number
workspace, cloudApi = CLOUD_API) => {
workspace, optionsGetter = CLOUD_ENDPOINT_GETTER) {
if (workspace) {
const storage = new BlobStorage();
const provider = await IndexedDBBlobProvider.init(workspace, cloudApi);
const provider = await IndexedDBBlobProvider.init(workspace, optionsGetter);
storage.addProvider(provider);

@@ -15,4 +15,4 @@ return storage;

return null;
};
}
export { BlobStorage } from './storage.js';
//# sourceMappingURL=index.js.map
import { Signal } from '@blocksuite/global/utils';
import type { BlobId, BlobProvider, BlobURL, IDBInstance } from './types.js';
export type BlobOptions = Record<'api' | 'token', string>;
export type BlobOptionsGetter = (key: keyof BlobOptions) => string | undefined;
export declare class IndexedDBBlobProvider implements BlobProvider {
private readonly _database;
private readonly _cloud?;
private _uploading;
uploading: boolean;
readonly blobs: Set<string>;

@@ -14,5 +16,3 @@ readonly signals: {

};
onUploadStateChange(callback: (uploading: boolean) => void, sync?: boolean): void;
onUploadFinished(callback: (id: BlobId) => void): void;
static init(workspace: string, cloudApi?: string): Promise<IndexedDBBlobProvider>;
static init(workspace: string, optionsGetter?: BlobOptionsGetter): Promise<IndexedDBBlobProvider>;
private _initBlobs;

@@ -37,3 +37,3 @@ private constructor();

private _onUploadStateChanged;
constructor(workspace: string, prefixUrl: string, db: IDBInstance, onUploadStateChanged: Signal<boolean>, onUploadFinished: Signal<BlobId>);
constructor(workspace: string, blobOptionsGetter: BlobOptionsGetter, db: IDBInstance, onUploadStateChanged: Signal<boolean>, onUploadFinished: Signal<BlobId>);
private _handleTaskRetry;

@@ -40,0 +40,0 @@ private _taskRunner;

@@ -9,5 +9,4 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {

import ky from 'ky';
import { Signal } from '@blocksuite/global/utils';
import { assertExists, Signal, sleep } from '@blocksuite/global/utils';
import { getDatabase, sha } from './utils.js';
import { sleep } from '@blocksuite/global/utils';
const RETRY_TIMEOUT = 500;

@@ -18,12 +17,4 @@ function staticImplements() {

let IndexedDBBlobProvider = IndexedDBBlobProvider_1 = class IndexedDBBlobProvider {
onUploadStateChange(callback, sync = true) {
if (sync)
callback(this._uploading);
this.signals.uploadStateChanged.on(callback);
}
onUploadFinished(callback) {
this.signals.uploadFinished.on(callback);
}
static async init(workspace, cloudApi) {
const provider = new IndexedDBBlobProvider_1(workspace, cloudApi);
static async init(workspace, optionsGetter) {
const provider = new IndexedDBBlobProvider_1(workspace, optionsGetter);
await provider._initBlobs();

@@ -40,4 +31,4 @@ return provider;

}
constructor(workspace, cloudApi) {
this._uploading = false;
constructor(workspace, optionsGetter) {
this.uploading = false;
this.blobs = new Set();

@@ -51,7 +42,9 @@ this.signals = {

this._database = getDatabase('blob', workspace);
if (cloudApi) {
const endpoint = optionsGetter?.('api');
if (endpoint) {
assertExists(optionsGetter);
this.signals.uploadStateChanged.on(uploading => {
this._uploading = uploading;
this.uploading = uploading;
});
this._cloud = new CloudSyncManager(workspace, cloudApi, this._database, this.signals.uploadStateChanged, this.signals.uploadFinished);
this._cloud = new CloudSyncManager(workspace, optionsGetter, this._database, this.signals.uploadStateChanged, this.signals.uploadFinished);
}

@@ -100,3 +93,3 @@ }

export class CloudSyncManager {
constructor(workspace, prefixUrl, db, onUploadStateChanged, onUploadFinished) {
constructor(workspace, blobOptionsGetter, db, onUploadStateChanged, onUploadFinished) {
this._abortController = new AbortController();

@@ -110,5 +103,15 @@ this._pendingPipeline = [];

this._fetcher = ky.create({
prefixUrl,
prefixUrl: blobOptionsGetter('api'),
signal: this._abortController.signal,
throwHttpErrors: false,
hooks: {
beforeRequest: [
async (request) => {
const token = blobOptionsGetter('token');
if (token) {
request.headers.set('Authorization', token);
}
},
],
},
});

@@ -122,6 +125,7 @@ this._database = db;

const blob = await db.get(id);
if ((blob || type === 'delete') && type) {
if ((blob || type === 'delete') && type !== 'upload') {
return { id, blob, retry, type };
}
if (blob && type === 'upload') {
console.log('try resume uploading blob:', id);
this.addTask(id, 'add');

@@ -137,5 +141,11 @@ }

}
async _handleTaskRetry(task, status) {
async _handleTaskRetry(task, response) {
this._removeUploadId(task.id);
if (status?.exists) {
if (response?.status === 413 ||
response?.status === 200 ||
task.retry >= 10) {
// if blob is too large, may try to upload it forever
if (response?.status === 413) {
console.log('blob too large:', task.id);
}
await this._pending.delete(task.id);

@@ -167,6 +177,7 @@ this._onUploadFinished.emit(task.id);

this._addUploadId(task.id);
const status = await this._fetcher
.put(`${this._workspace}/blob`, { body: task.blob, retry: 3 })
.json();
await this._handleTaskRetry(task, status);
const response = await this._fetcher.put(`${this._workspace}/blob`, {
body: task.blob,
retry: task.retry,
});
await this._handleTaskRetry(task, response);
}

@@ -176,3 +187,5 @@ }

console.warn('Error while syncing blob', e);
await this._handleTaskRetry(task);
if (e)
await this._handleTaskRetry(task);
this._taskRunner();
}

@@ -186,13 +199,13 @@ }

this._uploadingIds.add(id);
this._onUploadStateChanged.emit(!!this._uploadingIds.size);
this._onUploadStateChanged.emit(Boolean(this._uploadingIds.size));
}
_removeUploadId(id) {
this._uploadingIds.delete(id);
this._onUploadStateChanged.emit(!!this._uploadingIds.size);
this._onUploadStateChanged.emit(Boolean(this._uploadingIds.size));
}
async get(id) {
const api = `${this._workspace}/blob/${id}`;
const endpoint = `${this._workspace}/blob/${id}`;
try {
const blob = await this._fetcher
.get(api, { throwHttpErrors: true })
.get(endpoint, { throwHttpErrors: true })
.blob();

@@ -199,0 +212,0 @@ await this._database.set(id, await blob.arrayBuffer());

@@ -7,3 +7,5 @@ import { Signal } from '@blocksuite/global/utils';

blobAdded: Signal<string>;
uploadStateChanged: Signal<boolean>;
};
get uploading(): boolean;
get providers(): Readonly<BlobProvider[]>;

@@ -10,0 +12,0 @@ get blobs(): Set<BlobId>;

@@ -7,4 +7,8 @@ import { Signal } from '@blocksuite/global/utils';

blobAdded: new Signal(),
uploadStateChanged: new Signal(),
};
}
get uploading() {
return this._providers.some(p => p.uploading);
}
get providers() {

@@ -28,2 +32,5 @@ return this._providers;

});
provider.signals.uploadStateChanged?.on(() => {
this.signals.uploadStateChanged.emit(this.uploading);
});
}

@@ -30,0 +37,0 @@ removeProvider(provider) {

@@ -6,4 +6,6 @@ import type { Signal } from '@blocksuite/global/utils';

blobs: Set<BlobId>;
uploading: boolean;
signals: {
blobAdded: Signal<BlobId>;
uploadStateChanged?: Signal<boolean>;
};

@@ -10,0 +12,0 @@ get(id: BlobId): Promise<BlobURL | null>;

@@ -8,2 +8,3 @@ /// <reference types="@blocksuite/global" />

import { AwarenessStore, RawAwarenessState } from './awareness.js';
import type { BlobOptionsGetter } from './blob/index.js';
export interface SerializedStore {

@@ -48,2 +49,3 @@ [key: string]: {

defaultFlags?: Partial<Flags>;
blobOptionsGetter?: BlobOptionsGetter;
}

@@ -50,0 +52,0 @@ export declare class Store {

@@ -9,3 +9,3 @@ /// <reference types="@blocksuite/global" />

import type { BaseBlockModel } from '../base.js';
import { BlobStorage } from '../blob/index.js';
import { BlobStorage, BlobOptionsGetter } from '../blob/index.js';
import type { BlockSuiteDoc } from '../yjs/index.js';

@@ -60,2 +60,3 @@ import type { AwarenessStore } from '../awareness.js';

private _blobStorage;
private _blobOptionsGetter?;
meta: WorkspaceMeta;

@@ -83,2 +84,3 @@ signals: {

search(query: QueryContent): Map<string, string>;
setGettingBlobOptions(blobOptionsGetter: BlobOptionsGetter): void;
/**

@@ -85,0 +87,0 @@ * @internal Only for testing

@@ -7,3 +7,3 @@ import * as Y from 'yjs';

import { Indexer } from './search.js';
import { getBlobStorage } from '../blob/index.js';
import { getBlobStorage, } from '../blob/index.js';
class WorkspaceMeta extends Space {

@@ -159,7 +159,13 @@ constructor(id, doc, awarenessStore) {

constructor(options) {
this._blobOptionsGetter = (k) => ({ api: '/api/workspace' }[k]);
this.flavourMap = new Map();
this._store = new Store(options);
this._indexer = new Indexer(this.doc);
if (options.blobOptionsGetter) {
this._blobOptionsGetter = options.blobOptionsGetter;
}
if (!options.isSSR) {
this._blobStorage = getBlobStorage(options.room);
this._blobStorage = getBlobStorage(options.room, k => {
return this._blobOptionsGetter ? this._blobOptionsGetter(k) : '';
});
}

@@ -245,2 +251,5 @@ else {

}
setGettingBlobOptions(blobOptionsGetter) {
this._blobOptionsGetter = blobOptionsGetter;
}
/**

@@ -247,0 +256,0 @@ * @internal Only for testing

{
"name": "@blocksuite/store",
"version": "0.4.0-20230119023850-6877b78",
"version": "0.4.0-20230119151920-9bca215",
"description": "BlockSuite data store built for general purpose state management.",

@@ -11,3 +11,3 @@ "main": "dist/index.js",

"dependencies": {
"@blocksuite/global": "0.4.0-20230119023850-6877b78",
"@blocksuite/global": "0.4.0-20230119151920-9bca215",
"@types/flexsearch": "^0.7.3",

@@ -14,0 +14,0 @@ "buffer": "^6.0.3",

@@ -71,3 +71,8 @@ import type { Workspace } from '../workspace/workspace.js';

export { assertExists } from '@blocksuite/global/utils';
// XXX: workaround typing issue in blobs/__tests__/test-entry.ts
export function assertExists<T>(val: T | null | undefined): asserts val is T {
if (val === null || val === undefined) {
throw new Error('val does not exist');
}
}

@@ -74,0 +79,0 @@ export async function nextFrame() {

@@ -13,4 +13,9 @@ // Test page entry located in playground/examples/blob/index.html

const optionsGetters = {
noop: () => void 0,
cloudEndpoint: (k: string) => ({ api: '/api/workspace' }[k]),
};
async function testBasic() {
const storage = await getBlobStorage('test', '');
const storage = await getBlobStorage('test', optionsGetters.noop);
assertExists(storage);

@@ -88,3 +93,3 @@

const storage = await getBlobStorage('test', '');
const storage = await getBlobStorage('test', optionsGetters.noop);
assertExists(storage);

@@ -102,3 +107,3 @@

async function testRefreshAfter() {
const storage = await getBlobStorage('test', '');
const storage = await getBlobStorage('test', optionsGetters.noop);
assertExists(storage);

@@ -148,3 +153,3 @@

await clearIndexedDB('114514');
const storage = await getBlobStorage('114514', '/api/workspace');
const storage = await getBlobStorage('114514', optionsGetters.cloudEndpoint);
assertExists(storage);

@@ -166,3 +171,3 @@

async function testCloudSyncAfter() {
const storage = await getBlobStorage('114514', '/api/workspace');
const storage = await getBlobStorage('114514', optionsGetters.cloudEndpoint);
assertExists(storage);

@@ -169,0 +174,0 @@

import { IndexedDBBlobProvider } from './providers.js';
import type { BlobOptionsGetter, BlobOptions } from './providers.js';
import { BlobStorage } from './storage.js';
const CLOUD_API = '/api/workspace';
const CLOUD_ENDPOINT_GETTER = (k: string) => ({ api: '/api/workspace' }[k]);
export const getBlobStorage = async (
export async function getBlobStorage(
// Note: In the current backend design, the workspace id is a randomly generated int64 number
// so if you need to test or enable blob synchronization, the provided workspace needs to be a number
workspace?: string,
cloudApi: string = CLOUD_API
) => {
optionsGetter: BlobOptionsGetter = CLOUD_ENDPOINT_GETTER
) {
if (workspace) {
const storage = new BlobStorage();
const provider = await IndexedDBBlobProvider.init(workspace, cloudApi);
const provider = await IndexedDBBlobProvider.init(workspace, optionsGetter);
storage.addProvider(provider);

@@ -20,4 +21,5 @@

return null;
};
}
export { BlobStorage } from './storage.js';
export type { BlobOptionsGetter, BlobOptions };
import ky from 'ky';
import { Signal } from '@blocksuite/global/utils';
import { assertExists, Signal, sleep } from '@blocksuite/global/utils';
import type { BlobId, BlobProvider, BlobURL, IDBInstance } from './types.js';
import { getDatabase, sha } from './utils.js';
import { sleep } from '@blocksuite/global/utils';
const RETRY_TIMEOUT = 500;
export type BlobOptions = Record<'api' | 'token', string>;
export type BlobOptionsGetter = (key: keyof BlobOptions) => string | undefined;
interface BlobProviderStatic {
init(workspace: string, cloudApi?: string): Promise<BlobProvider>;
init(
workspace: string,
optionsGetter?: BlobOptionsGetter
): Promise<BlobProvider>;
}

@@ -22,3 +28,3 @@

private readonly _cloud?: CloudSyncManager;
private _uploading = false;
public uploading = false;

@@ -33,16 +39,7 @@ readonly blobs: Set<string> = new Set();

onUploadStateChange(callback: (uploading: boolean) => void, sync = true) {
if (sync) callback(this._uploading);
this.signals.uploadStateChanged.on(callback);
}
onUploadFinished(callback: (id: BlobId) => void) {
this.signals.uploadFinished.on(callback);
}
static async init(
workspace: string,
cloudApi?: string
optionsGetter?: BlobOptionsGetter
): Promise<IndexedDBBlobProvider> {
const provider = new IndexedDBBlobProvider(workspace, cloudApi);
const provider = new IndexedDBBlobProvider(workspace, optionsGetter);
await provider._initBlobs();

@@ -61,11 +58,14 @@ return provider;

private constructor(workspace: string, cloudApi?: string) {
private constructor(workspace: string, optionsGetter?: BlobOptionsGetter) {
this._database = getDatabase('blob', workspace);
if (cloudApi) {
const endpoint = optionsGetter?.('api');
if (endpoint) {
assertExists(optionsGetter);
this.signals.uploadStateChanged.on(uploading => {
this._uploading = uploading;
this.uploading = uploading;
});
this._cloud = new CloudSyncManager(
workspace,
cloudApi,
optionsGetter,
this._database,

@@ -132,3 +132,3 @@ this.signals.uploadStateChanged,

type BlobStatus = {
exists: boolean;
status: number;
};

@@ -155,3 +155,3 @@

workspace: string,
prefixUrl: string,
blobOptionsGetter: BlobOptionsGetter,
db: IDBInstance,

@@ -164,5 +164,15 @@ onUploadStateChanged: Signal<boolean>,

this._fetcher = ky.create({
prefixUrl,
prefixUrl: blobOptionsGetter('api'),
signal: this._abortController.signal,
throwHttpErrors: false,
hooks: {
beforeRequest: [
async request => {
const token = blobOptionsGetter('token');
if (token) {
request.headers.set('Authorization', token);
}
},
],
},
});

@@ -180,6 +190,7 @@

const blob = await db.get(id);
if ((blob || type === 'delete') && type) {
if ((blob || type === 'delete') && type !== 'upload') {
return { id, blob, retry, type };
}
if (blob && type === 'upload') {
console.log('try resume uploading blob:', id);
this.addTask(id, 'add');

@@ -200,5 +211,13 @@ }

private async _handleTaskRetry(task: SyncTask, status?: BlobStatus) {
private async _handleTaskRetry(task: SyncTask, response?: BlobStatus) {
this._removeUploadId(task.id);
if (status?.exists) {
if (
response?.status === 413 ||
response?.status === 200 ||
task.retry >= 10
) {
// if blob is too large, may try to upload it forever
if (response?.status === 413) {
console.log('blob too large:', task.id);
}
await this._pending.delete(task.id);

@@ -235,10 +254,15 @@ this._onUploadFinished.emit(task.id);

this._addUploadId(task.id);
const status = await this._fetcher
.put(`${this._workspace}/blob`, { body: task.blob, retry: 3 })
.json<BlobStatus>();
await this._handleTaskRetry(task, status);
const response = await this._fetcher.put(
`${this._workspace}/blob`,
{
body: task.blob,
retry: task.retry,
}
);
await this._handleTaskRetry(task, response);
}
} catch (e) {
console.warn('Error while syncing blob', e);
await this._handleTaskRetry(task);
if (e) await this._handleTaskRetry(task);
this._taskRunner();
}

@@ -254,3 +278,3 @@ }

this._uploadingIds.add(id);
this._onUploadStateChanged.emit(!!this._uploadingIds.size);
this._onUploadStateChanged.emit(Boolean(this._uploadingIds.size));
}

@@ -260,10 +284,10 @@

this._uploadingIds.delete(id);
this._onUploadStateChanged.emit(!!this._uploadingIds.size);
this._onUploadStateChanged.emit(Boolean(this._uploadingIds.size));
}
async get(id: BlobId): Promise<BlobURL | null> {
const api = `${this._workspace}/blob/${id}`;
const endpoint = `${this._workspace}/blob/${id}`;
try {
const blob = await this._fetcher
.get(api, { throwHttpErrors: true })
.get(endpoint, { throwHttpErrors: true })
.blob();

@@ -270,0 +294,0 @@

@@ -6,6 +6,12 @@ import { Signal } from '@blocksuite/global/utils';

private _providers: BlobProvider[] = [];
signals = {
blobAdded: new Signal<BlobId>(),
uploadStateChanged: new Signal<boolean>(),
};
get uploading(): boolean {
return this._providers.some(p => p.uploading);
}
get providers(): Readonly<BlobProvider[]> {

@@ -31,2 +37,5 @@ return this._providers;

});
provider.signals.uploadStateChanged?.on(() => {
this.signals.uploadStateChanged.emit(this.uploading);
});
}

@@ -33,0 +42,0 @@

@@ -8,4 +8,6 @@ import type { Signal } from '@blocksuite/global/utils';

blobs: Set<BlobId>;
uploading: boolean;
signals: {
blobAdded: Signal<BlobId>;
uploadStateChanged?: Signal<boolean>;
};

@@ -12,0 +14,0 @@ get(id: BlobId): Promise<BlobURL | null>;

@@ -15,2 +15,3 @@ import type { Space } from './space.js';

import { AwarenessStore, RawAwarenessState } from './awareness.js';
import type { BlobOptionsGetter } from './blob/index.js';

@@ -62,2 +63,3 @@ export interface SerializedStore {

defaultFlags?: Partial<Flags>;
blobOptionsGetter?: BlobOptionsGetter;
}

@@ -64,0 +66,0 @@

@@ -8,3 +8,7 @@ import * as Y from 'yjs';

import type { BaseBlockModel } from '../base.js';
import { BlobStorage, getBlobStorage } from '../blob/index.js';
import {
BlobStorage,
BlobOptionsGetter,
getBlobStorage,
} from '../blob/index.js';
import type { BlockSuiteDoc } from '../yjs/index.js';

@@ -221,2 +225,4 @@ import type { AwarenessStore } from '../awareness.js';

private _blobStorage: Promise<BlobStorage | null>;
private _blobOptionsGetter?: BlobOptionsGetter = (k: string) =>
({ api: '/api/workspace' }[k]);

@@ -236,4 +242,9 @@ meta: WorkspaceMeta;

this._indexer = new Indexer(this.doc);
if (options.blobOptionsGetter) {
this._blobOptionsGetter = options.blobOptionsGetter;
}
if (!options.isSSR) {
this._blobStorage = getBlobStorage(options.room);
this._blobStorage = getBlobStorage(options.room, k => {
return this._blobOptionsGetter ? this._blobOptionsGetter(k) : '';
});
} else {

@@ -344,2 +355,6 @@ // blob storage is not reachable in server side

setGettingBlobOptions(blobOptionsGetter: BlobOptionsGetter) {
this._blobOptionsGetter = blobOptionsGetter;
}
/**

@@ -346,0 +361,0 @@ * @internal Only for testing

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

Packages

npm

Stay in touch

Get open source security insights delivered straight into your inbox.


  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc