ts-datastore-orm

ts-datastore-orm (Typescript Orm wrapper for Google Datastore)

ts-datastore-orm targets to provide a strong typed and structural ORM feature for Datastore (Firestore in Datastore mode).

Please check the examples below to check out all the amazing features!

This package has 0 dependencies. You have to install the @google-cloud/datastore manually.

npm install -s @google-cloud/datastore@latest

This package is compatible with @google-cloud/datastore v5.0.0 or above. I have tested the library specifically with the following versions: v5.0.6, v5.1.0, v6.1.1, v6.2.0, v6.3.0 v7.0.0

You can also compare the native performance of @google-cloud/datastore with ts-datastorem-orm. Basically there is no significant overhead compared with native nodejs-datastore package.

Project Setup

• npm install -s @google-cloud/datastore@latest
• npm install -s ts-datastore-orm@latest
• In tsconfig.json
  • set "experimentalDecorators" to true.
  • set "emitDecoratorMetadata" to true.
  • set "strictNullChecks" to true.
• Generate service-account.json from Goolge APIs. (Details won't be covered here)

Example: connection

async function connectionExamples() {
  const connection1 = await createConnection({keyFilename: "./datastoreServiceAccount.json"});
  const connection2 = await createConnection({clientEmail: "", privateKey: ""});
  const datastore1 = connection1.datastore;
  const datastore2 = connection2.datastore;
}

Example: define Entity

import { BaseEntity, CompositeIndex, CompositeIndexExporter, createConnection, Entity, Field, 
tsDatastoreOrm, TsDatastoreOrmError, BeforeDelete, BeforeInsert, BeforeUpdate, BeforeUpsert, AfterLoad} from "ts-datastore-orm";

@CompositeIndex({_id: "desc"})
@Entity({namespace: "testing", kind: "User", enumerable: true})
export class User extends BaseEntity {
    @Field({generateId: true})
    public _id: number = 0;

    @Field()
    public date: Date = new Date();

    @Field({index: true})
    public string: string = "";

    @Field()
    public number: number = 10;

    @Field()
    public buffer: Buffer = Buffer.alloc(1);

    @Field()
    public array: number[] = [];

    @Field({index: true, excludeFromIndexes: ["object.name"]})
    public object: any = {};

    @Field()
    public undefined: undefined = undefined;

    @Field()
    public null: null = null;
}

@CompositeIndex({number: "desc", name: "desc"})
@CompositeIndex({_id: "desc"})
@Entity() // namespace: default, kind: same as class name
export class TaskGroup extends BaseEntity {
    @Field({generateId: true})
    public _id: number = 0;

    @Field()
    public name: string = "";

    @Field()
    public number: number = 0;

    @AfterLoad()
    @BeforeInsert()
    @BeforeUpsert()
    @BeforeUpdate()
    @BeforeDelete()
    public hook(type: string) {
        // you can update the entity after certain events happened
    }
}

Example: general

async function keyExamples() {
    const connection = await createConnection({keyFilename: "./datastoreServiceAccount.json"});
    const userRepository = connection.getRepository(User);

    const user1 = new User();
    const userKey1 = user1.getKey();
    const userKey2 = userRepository.create().getKey();
    const user3 = userRepository.create({_id: 3});
    user3._ancestorKey = userKey1;
    const userKey3 = user3.getKey();

    const result1 = isSameKey(userKey1, userKey2); // true
    const result2 = isSameNamespace(userKey1, userKey2); // true
    const result3 = isSameKind(userKey1, userKey2); // true
    const result4 = isSamePath(userKey1, userKey2); // true
    const result5 = isSameKey(userKey1, userKey3); // false

    const [encoded1] = await userRepository.datastore.keyToLegacyUrlSafe(userKey1);
    const restoredKey = userRepository.datastore.keyFromLegacyUrlsafe(encoded1);
    const result6 = isSameKey(userKey1, restoredKey); // true
}

Example: general

async function generalExamples() {
    const connection = await createConnection({keyFilename: "./datastoreServiceAccount.json"});
    const repository = connection.getRepository(User, {namespace: "mynamespace", kind: "NewUser"});
    const taskGroupRepository = connection.getRepository(TaskGroup);
    const datastore = connection.datastore; // access to native datastore

    const user1 = repository.create();
    await repository.insert(user1);
    const key = user1.getKey(); // the native datastore key

    // the kind and namespace is attached to entity, but they are not enumerable by default
    // use @Entity({enumerable: true}) such that if you console.log(entity), _kind and _namespace will be displayed as well
    const {_kind, _namespace, _ancestorKey} = user1;

    // simple query
    const findUser1 = repository.findOne(user1._id);

    // find users
    const users = await repository
        .query()
        .filter("_id", x => x.gt(5))
        .limit(100)
        .findMany();

    // get id
    const ids = await repository.allocateIds(10);

    // remove all data
    await repository.truncate();
}

Example: multiple entities

async function multipleEntities() {
    const connection = await createConnection({keyFilename: "./datastoreServiceAccount.json"});
    const repository = connection.getRepository(User, {namespace: "mynamespace", kind: "NewUser"});

    const users = Array(10).map((_, i) => repository.create({number: i}));
    await repository.insert(users);
    await repository.update(users);
    await repository.upsert(users);
    await repository.delete(users);
}

Example: ancestors

async function ancestorExamples() {
    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const userRepository = connection.getRepository(User);
    const taskGroupRepository = connection.getRepository(TaskGroup);

    const user1 = await userRepository.insert(userRepository.create({_id: 1}));
    const taskGroup = taskGroupRepository.create({_id: 1, name: "group 1", _ancestorKey: user1.getKey()});
    await taskGroupRepository.insert(taskGroup);

    // ignore the strong type on method call
    const findTaskGroup1 = await taskGroupRepository.query()
        .filterKey(taskGroup.getKey())
        .findOne();

    // get back the user
    if (findTaskGroup1?._ancestorKey) {
        const findUser1 = await userRepository.findOne(findTaskGroup1._ancestorKey);
    }

    // another way to query the entities
    const findTaskGroup2 = await taskGroupRepository.query()
        .setAncestorKey(user1.getKey())
        .filter("_id", 1)
        .findOne();
}

Example: admin

async function adminExamples() {
    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const myAdmin = connection.getAdmin();
    const namespaces = await myAdmin.getNamespaces();
    const kinds = await myAdmin.getKinds();
}

Example: query

async function queryExamples() {
    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const userRepository = connection.getRepository(User);
    const user = userRepository.create({_id: 1});

    const findUser1 = await userRepository.query().findOne();
    const findUsers2 = await userRepository.findMany([1, 2, 3, 4, 5]);
    const findUsers3 = await userRepository.query().filter("_id", x => x.ge(1).lt(6)).findMany();
    const findUsers4 = await userRepository.query().limit(10).offset(3).order("number", {descending: true}).findMany();

    // complex query with strong type
    const query1 = userRepository.query()
        .filter("number", 10)
        .setAncestorKey(user.getKey())
        .groupBy("number")
        .order("number", {descending: true})
        .offset(5)
        .limit(10);

    // complex query with strong type
    const query = userRepository.query({weakType: true})
        .filter("object.name", 10)
        .setAncestorKey(user.getKey())
        .groupBy("object.name")
        .order("object.name", {descending: true})
        .offset(5)
        .limit(10);

    // iterator
    const batch = 500;
    const iterator = userRepository.query().limit(batch).getAsyncIterator();
    for await (const entities of iterator) {
        if (entities.length === batch) {
            // true
        }
    }

    // select key query
    // this can save some query cost and also return faster
    const keys = await userRepository.selectKeyQuery().findMany();
}

Example: transaction manager

async function transactionManagerExamples() {
    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const userRepository = connection.getRepository(User);

    const transactionManager1 = connection.getTransactionManager();

    // customize behavior of the transaction
    // for readonly transaction, please refer to datastore documentation
    const transactionManager2 = connection.getTransactionManager({maxRetry: 3, retryDelay: 200, readOnly: true});

    const result = await transactionManager1.start(async (session) => {
        const findEntity1 = await userRepository.findOneWithSession(1, session);
        const findEntity2 = await userRepository.queryWithSession( session).findOne();
        const ids = await userRepository.allocateIdsWithSession(10, session);

        if (findEntity2) {
            // only the last operation of the same entity will applies only
            userRepository.insertWithSession(findEntity2, session);
            userRepository.updateWithSession(findEntity2, session);
            userRepository.upsertWithSession(findEntity2, session);
            userRepository.deleteWithSession(findEntity2, session);
        } else {
            await session.rollback();
        }

        return 5;
    });

    // value === 5 in above case
    const {value, hasCommitted, totalRetry} = result;
}

Example: error

async function errorExamples() {
    // this help you to provide a better stack upon error
    tsDatastoreOrm.useFriendlyErrorStack = true;

    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const userRepository = connection.getRepository(User);
    const user = userRepository.create({_id: 1});

    try {
        await userRepository.delete(user);
    } catch (err) {
        if (err instanceof TsDatastoreOrmError) {
            // error from this library

        }
    }
}

Example: increment helper

async function incrementHelperExamples() {
    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const userRepository = connection.getRepository(User);
    const user = userRepository.create({_id: 1});
    const incrementHelper = userRepository.getIncrementHelper();

    // take all kind of parameters
    const latestValue1 = await incrementHelper.increment(user._id, "number", 10);
    const latestValue2 = await incrementHelper.increment(user, "number", 10);
    const latestValue3 = await incrementHelper.increment(user.getKey(), "number", 10);
}

Example: index resave helper

async function indexResaveHelperExamples() {
    // sometimes u added new index and need to resave the entities
    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const userRepository = connection.getRepository(User);
    const helper = userRepository.getIndexResaveHelper();

    await helper.resave("number");
    await helper.resave(["number", "string"]);
}

Example: lock manager

async function lockManagerExamples() {
    // this is a tool for atomic lock
    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const lockManager1 = connection.getLockManager({expiresIn: 1000});

    // this look will try to acquire the lock 2 more times if it failed, waiting 100ms for each retry
    // you can also customize which namespace and kind to save temporary data of the lock
    const lockManager2 = connection.getLockManager({expiresIn: 1000, maxRetry: 2, retryDelay: 100, namespace: "custom", kind: "Lock"});

    try {
        const lockKey = "anyKey";
        const result = await lockManager1.start(lockKey, async () => {
            return 5;
        });
        console.log(result.value);

    } catch (err) {
        // your own error or lock acquire error

    }
}

Example: composite index exporter

async function compositeIndexExamples() {
    const filename = "./index.yaml";
    const exporter = new CompositeIndexExporter();
    exporter.addEntity(User, {kind: "NewUser"});
    exporter.addEntity(TaskGroup, {kind: "NewTaskGroup"});
    // you can add multiple class, but you can't customize the kind name
    exporter.addEntity([User, TaskGroup]);

    const yaml = exporter.getYaml();
    exporter.exportTo(filename);

}

More Examples

Examples are in the tests/ directory.

Sample	Source Code
General	source code
Query	source code
Subclass	source code
Errors	source code
CompositeIndex	source code
Admin	source code
TransactionManager	source code
LockManager	source code
Hook	source code
IndexResaveHelper	source code
IncrementHelper	source code

Useful links

• https://googleapis.dev/nodejs/datastore/7.0.0/index.html
• https://googleapis.dev/nodejs/datastore/6.0.0/index.html
• https://cloud.google.com/datastore/docs/
• https://www.npmjs.com/package/@google-cloud/datastore
• https://www.npmjs.com/package/@google-cloud/firestore