@lbu/store - npm Package Compare versions

Comparing version 0.0.65 to 0.0.66

index.d.ts

@@ -60,2 +60,11 @@ import * as minioVendor from "minio";
 /**
+ * Copy all objects from sourceBucket to destination bucket
+ */
+export function copyAllObjects(
+  minio: minioVendor.Client,
+  sourceBucket: string,
+  destinationBucket: string,
+): Promise<void>;
+
+/**
  * Removes all objects and then deletes the bucket
@@ -79,13 +88,16 @@ */
 /**
- * Setup a test database once
- * Copies the current app database
- * Calls callback, so seeding is possible, then reuses this as a template.
- * Make sure to call this before any other call to createTestPostgresDatabase.
- * It is safe to call this multiple times, the callback will only be executed once.
+ * Set test database.
+ * New createTestPostgresConnection calls will use this as a template,
+ * so things like seeding only need to happen once
  */
-export function setupTestDatabase(
-  callback?: (sql: Postgres) => Promise<void>,
+export function setPostgresDatabaseTemplate(
+  databaseNameOrConnection: Postgres | string,
 ): Promise<void>;
 
 /**
+ * Cleanup the database, set with `setPostgresDatabaseTemplate`
+ */
+export function cleanupPostgresDatabaseTemplate(): Promise<void>;
+
+/**
  * Drops connections to 'normal' database and creates a new one based on the 'normal' database.
@@ -92,0 +104,0 @@ * It will truncate all tables and return a connection to the new database.

index.js

@@ -10,2 +10,3 @@ import { dirnameForModule } from "@lbu/stdlib";
   removeBucketAndObjectsInBucket,
+  copyAllObjects,
 } from "./src/minio.js";
@@ -18,3 +19,4 @@
   createTestPostgresDatabase,
-  setupTestDatabase,
+  setPostgresDatabaseTemplate,
+  cleanupPostgresDatabaseTemplate,
 } from "./src/testing.js";
@@ -21,0 +23,0 @@

package.json

 {
   "name": "@lbu/store",
-  "version": "0.0.65",
+  "version": "0.0.66",
   "description": "Postgres & S3-compatible wrappers for common things",
@@ -18,4 +18,4 @@ "main": "./index.js",
   "dependencies": {
-    "@lbu/insight": "0.0.65",
-    "@lbu/stdlib": "0.0.65",
+    "@lbu/insight": "0.0.66",
+    "@lbu/stdlib": "0.0.66",
     "@types/minio": "7.0.6",
@@ -47,3 +47,3 @@ "mime-types": "2.1.27",
   },
-  "gitHead": "843efcbe36702f8a7e17992d81fa24ff621fda18"
+  "gitHead": "828e6b5c7605a1f299cf549a34776ad2aa61db58"
 }

src/generated/queries.js

@@ -13,3 +13,5 @@ // Generated by @lbu/code-gen
   fileStoreSelect: (sql, where) => sql`
-SELECT fs."id", fs."bucketName", fs."contentLength", fs."contentType", fs."filename", fs."createdAt", fs."updatedAt" FROM "fileStore" fs 
+SELECT
+fs."id", fs."bucketName", fs."contentLength", fs."contentType", fs."filename", fs."createdAt", fs."updatedAt"
+FROM "fileStore" fs
 WHERE (COALESCE(${where?.id ?? null}, NULL) IS NULL OR fs."id" = ${
@@ -28,2 +30,3 @@ where?.id ?? null
   }, NULL) IS NULL OR fs."bucketName" LIKE ${`%${where?.bucketNameLike}%`})
+ORDER BY fs."createdAt", fs."updatedAt" , fs."id"
 `,
@@ -203,3 +206,6 @@
   fileStoreSelectHistory: (sql, where) => sql`
-SELECT fs."id", fs."bucketName", fs."contentLength", fs."contentType", fs."filename", fs."createdAt", fs."updatedAt", array_agg(to_jsonb(fsh.*)) AS history FROM "fileStore" fs LEFT JOIN "fileStoreHistory" fsh ON fs.id = fsh."fileStoreId" 
+SELECT
+fs."id", fs."bucketName", fs."contentLength", fs."contentType", fs."filename", fs."createdAt", fs."updatedAt", array_agg(to_jsonb(fsh.*)) AS history
+FROM "fileStore" fs
+LEFT JOIN "fileStoreHistory" fsh ON fs.id = fsh."fileStoreId"
 WHERE (COALESCE(${where?.id ?? null}, NULL) IS NULL OR fs."id" = ${
@@ -219,2 +225,3 @@ where?.id ?? null
 GROUP BY fs.id
+ORDER BY fs."createdAt", fs."updatedAt" , fs."id"
 `,
@@ -228,3 +235,5 @@
   sessionStoreSelect: (sql, where) => sql`
-SELECT ss."id", ss."expires", ss."data", ss."createdAt", ss."updatedAt" FROM "sessionStore" ss 
+SELECT
+ss."id", ss."expires", ss."data", ss."createdAt", ss."updatedAt"
+FROM "sessionStore" ss
 WHERE (COALESCE(${where?.id ?? null}, NULL) IS NULL OR ss."id" = ${
@@ -245,2 +254,3 @@ where?.id ?? null
   }, NULL) IS NULL OR ss."expires" < ${where?.expiresLowerThan ?? null})
+ORDER BY ss."createdAt", ss."updatedAt" , ss."id"
 `,
@@ -443,3 +453,5 @@
   jobQueueSelect: (sql, where) => sql`
-SELECT jq."id", jq."isComplete", jq."priority", jq."scheduledAt", jq."name", jq."data", jq."createdAt", jq."updatedAt" FROM "jobQueue" jq 
+SELECT
+jq."id", jq."isComplete", jq."priority", jq."scheduledAt", jq."name", jq."data", jq."createdAt", jq."updatedAt"
+FROM "jobQueue" jq
 WHERE (COALESCE(${where?.id ?? null}, NULL) IS NULL OR jq."id" = ${
@@ -458,2 +470,3 @@ where?.id ?? null
   }, NULL) IS NULL OR jq."name" LIKE ${`%${where?.nameLike}%`})
+ORDER BY jq."createdAt", jq."updatedAt" , jq."id"
 `,
@@ -460,0 +473,0 @@

src/generated/structure.js

 export const structureString =
-  '{"store":{"fileStore":{"type":"object","group":"store","name":"fileStore","docString":"","isOptional":false,"validator":{"strict":false},"enableQueries":true,"queryOptions":{"withHistory":true},"keys":{"id":{"type":"uuid","docString":"","isOptional":false,"sql":{"searchable":true,"primary":true}},"bucketName":{"type":"string","docString":"","isOptional":false,"validator":{"convert":false,"trim":false,"lowerCase":false,"upperCase":false},"sql":{"searchable":true}},"contentLength":{"type":"number","docString":"","isOptional":false,"validator":{"convert":false,"integer":true}},"contentType":{"type":"string","docString":"","isOptional":false,"validator":{"convert":false,"trim":false,"lowerCase":false,"upperCase":false}},"filename":{"type":"string","docString":"","isOptional":false,"validator":{"convert":false,"trim":false,"lowerCase":false,"upperCase":false}}},"uniqueName":"StoreFileStore"},"sessionStore":{"type":"object","group":"store","name":"sessionStore","docString":"","isOptional":false,"validator":{"strict":false},"enableQueries":true,"queryOptions":{"withDates":true},"keys":{"id":{"type":"uuid","docString":"","isOptional":false,"sql":{"searchable":true,"primary":true}},"expires":{"type":"date","docString":"","isOptional":false,"sql":{"searchable":true}},"data":{"type":"any","docString":"","isOptional":true,"defaultValue":"{}"}},"uniqueName":"StoreSessionStore"},"jobQueue":{"type":"object","group":"store","name":"jobQueue","docString":"","isOptional":false,"validator":{"strict":false},"enableQueries":true,"queryOptions":{"withDates":true},"keys":{"id":{"type":"number","docString":"","isOptional":false,"validator":{"convert":false,"integer":true},"sql":{"searchable":true,"primary":true}},"isComplete":{"type":"boolean","docString":"","isOptional":true,"defaultValue":"false","validator":{"convert":false},"sql":{"searchable":true}},"priority":{"type":"number","docString":"","isOptional":true,"defaultValue":"0","validator":{"convert":false,"integer":true}},"scheduledAt":{"type":"date","docString":"","isOptional":true,"defaultValue":"(new Date())","sql":{"searchable":true}},"name":{"type":"string","docString":"","isOptional":false,"validator":{"convert":false,"trim":false,"lowerCase":false,"upperCase":false},"sql":{"searchable":true}},"data":{"type":"any","docString":"","isOptional":true,"defaultValue":"{}"}},"uniqueName":"StoreJobQueue"}}}';
+  '{"store":{"fileStore":{"type":"object","group":"store","name":"fileStore","docString":"","isOptional":false,"validator":{"strict":false},"enableQueries":true,"queryOptions":{"withHistory":true},"keys":{"id":{"type":"uuid","docString":"","isOptional":false,"sql":{"searchable":true,"primary":true}},"bucketName":{"type":"string","docString":"","isOptional":false,"validator":{"convert":false,"trim":false,"lowerCase":false,"upperCase":false,"min":1},"sql":{"searchable":true}},"contentLength":{"type":"number","docString":"","isOptional":false,"validator":{"convert":false,"integer":true}},"contentType":{"type":"string","docString":"","isOptional":false,"validator":{"convert":false,"trim":false,"lowerCase":false,"upperCase":false,"min":1}},"filename":{"type":"string","docString":"","isOptional":false,"validator":{"convert":false,"trim":false,"lowerCase":false,"upperCase":false,"min":1}}},"uniqueName":"StoreFileStore"},"sessionStore":{"type":"object","group":"store","name":"sessionStore","docString":"","isOptional":false,"validator":{"strict":false},"enableQueries":true,"queryOptions":{"withDates":true},"keys":{"id":{"type":"uuid","docString":"","isOptional":false,"sql":{"searchable":true,"primary":true}},"expires":{"type":"date","docString":"","isOptional":false,"sql":{"searchable":true}},"data":{"type":"any","docString":"","isOptional":true,"defaultValue":"{}"}},"uniqueName":"StoreSessionStore"},"jobQueue":{"type":"object","group":"store","name":"jobQueue","docString":"","isOptional":false,"validator":{"strict":false},"enableQueries":true,"queryOptions":{"withDates":true},"keys":{"id":{"type":"number","docString":"","isOptional":false,"validator":{"convert":false,"integer":true},"sql":{"searchable":true,"primary":true}},"isComplete":{"type":"boolean","docString":"","isOptional":true,"defaultValue":"false","validator":{"convert":false},"sql":{"searchable":true}},"priority":{"type":"number","docString":"","isOptional":true,"defaultValue":"0","validator":{"convert":false,"integer":true}},"scheduledAt":{"type":"date","docString":"","isOptional":true,"defaultValue":"(new Date())","sql":{"searchable":true}},"name":{"type":"string","docString":"","isOptional":false,"validator":{"convert":false,"trim":false,"lowerCase":false,"upperCase":false,"min":1},"sql":{"searchable":true}},"data":{"type":"any","docString":"","isOptional":true,"defaultValue":"{}"}},"uniqueName":"StoreJobQueue"}}}';
 export const structure = JSON.parse(structureString);

src/minio.js

@@ -74,2 +74,26 @@ import { isProduction, merge } from "@lbu/stdlib";
 
+/**
+ * @param {minio.Client} minio
+ * @param {string} sourceBucket
+ * @param {string} destinationBucket
+ * @returns {Promise<void>}
+ */
+export async function copyAllObjects(minio, sourceBucket, destinationBucket) {
+  await ensureBucket(minio, destinationBucket);
+  const objects = await listObjects(minio, sourceBucket);
+
+  const pArr = [];
+  for (const object of objects) {
+    pArr.push(
+      minio.copyObject(
+        destinationBucket,
+        object.name,
+        `${sourceBucket}/${object.name}`,
+      ),
+    );
+  }
+
+  await Promise.all(pArr);
+}
+
 export { minio };

src/testing.js

@@ -9,72 +9,44 @@ import { log } from "@lbu/insight";
 /**
- * This database is reused in in setupTestDatabase
- * @type {string|undefined}
+ * If set, new databases are derived from this database
+ * @type {undefined}
  */
-let testDatabaseName = undefined;
+let testDatabase = undefined;
 
 /**
- * Setup a test database once
- * Copies the current app database
- * Calls callback, so seeding is possible, then reuses this as a template
- *
- * @param {function(Postgres): Promise<void>} callback
- * @returns {Promise<void>}
+ * Set test database.
+ * New createTestPostgresConnection calls will use this as a template,
+ * so things like seeding only need to happen once
+ * @param {Postgres|string} databaseNameOrConnection
  */
-export async function setupTestDatabase(callback) {
-  if (testDatabaseName !== undefined) {
-    return;
+export async function setPostgresDatabaseTemplate(databaseNameOrConnection) {
+  if (!isNil(testDatabase)) {
+    await cleanupPostgresDatabaseTemplate();
   }
 
-  testDatabaseName = process.env.APP_NAME + uuid().substring(0, 7);
-
-  // Open connection to default database
-  const creationSql = await createDatabaseIfNotExists(
-    undefined,
-    process.env.APP_NAME,
-  );
-
-  // Drop all connections to the database, this is required before it is usable as a
-  // template
-  await creationSql`
-SELECT pg_terminate_backend(pg_stat_activity.pid)
-FROM pg_stat_activity
-WHERE pg_stat_activity.datname = ${process.env.APP_NAME}
-  AND pid <> pg_backend_pid()
-  `;
-
-  // Create testDatabase based on the default app database
-  await createDatabaseIfNotExists(
-    creationSql,
-    testDatabaseName,
-    process.env.APP_NAME,
-  );
-
-  const sql = await newPostgresConnection({
-    database: testDatabaseName,
-  });
-
-  // List all tables
-  const tables = await sql`SELECT table_name
-                             FROM information_schema.tables
-                             WHERE table_schema = 'public' AND table_name != 'migrations'`;
-
-  if (tables.length > 0) {
-    await sql.unsafe(
-      `TRUNCATE ${tables.map((it) => `"${it.table_name}"`).join(", ")} CASCADE`,
+  if (typeof databaseNameOrConnection === "string") {
+    testDatabase = databaseNameOrConnection;
+  } else if (typeof databaseNameOrConnection?.options?.database === "string") {
+    testDatabase = databaseNameOrConnection.options.database;
+  } else {
+    throw new Error(
+      `Expected string or sql connection. Found ${typeof databaseNameOrConnection}`,
     );
-  } else {
-    // Just a query to initialize the connection
-    await sql`SELECT 1 + 1 AS sum;`;
   }
+}
 
-  if (!isNil(callback) && typeof callback === "function") {
-    // Call user seeder
-    await callback(sql);
+/**
+ * Cleanup the test template
+ * @returns {Promise<void>}
+ */
+export async function cleanupPostgresDatabaseTemplate() {
+  if (!isNil(testDatabase)) {
+    // We mock a connection here, since cleanTestPostgresDatabase doesn't use the
+    // connection any way
+    await cleanupTestPostgresDatabase({
+      options: {
+        database: testDatabase,
+      },
+      end: () => Promise.resolve(),
+    });
   }
-
-  // Cleanup connections
-  await Promise.all([
-    creationSql.end({ timeout: 0.01 }),
-    sql.end({ timeout: 0.01 }),
-  ]);
 }
@@ -87,8 +59,58 @@
   const name = process.env.APP_NAME + uuid().substring(0, 7);
-  await setupTestDatabase(() => {});
 
+  // Setup a template to work from
+  if (isNil(testDatabase)) {
+    testDatabase = process.env.APP_NAME + uuid().substring(0, 7);
+
+    const creationSql = await createDatabaseIfNotExists(
+      undefined,
+      process.env.APP_NAME,
+    );
+
+    // Clean all connections
+    // They prevent from using this as a template
+    await creationSql`
+  SELECT pg_terminate_backend(pg_stat_activity.pid)
+  FROM pg_stat_activity
+  WHERE pg_stat_activity.datname = ${process.env.APP_NAME}
+    AND pid <> pg_backend_pid()
+    `;
+
+    // Use the current 'app' database as a base.
+    // We expect the user to have done all necessary migrations
+    await createDatabaseIfNotExists(
+      creationSql,
+      testDatabase,
+      process.env.APP_NAME,
+    );
+
+    const sql = await newPostgresConnection({
+      database: testDatabase,
+    });
+
+    // Cleanup all tables, except migrations
+    const tables = await sql`SELECT table_name
+                               FROM information_schema.tables
+                               WHERE table_schema = 'public'
+                                 AND table_name != 'migrations'`;
+    if (tables.length > 0) {
+      await sql.unsafe(
+        `TRUNCATE ${tables
+          .map((it) => `"${it.table_name}"`)
+          .join(", ")} CASCADE`,
+      );
+    }
+
+    // Cleanup all connections
+    await Promise.all([
+      creationSql.end({ timeout: 0.01 }),
+      sql.end({ timeout: 0.01 }),
+    ]);
+  }
+
+  // Real database creation
   const creationSql = await createDatabaseIfNotExists(
     undefined,
     name,
-    testDatabaseName,
+    testDatabase,
   );
@@ -104,3 +126,3 @@
     creationSql.end({ timeout: 0.01 }),
-    sql`SELECT 1 + 1 as sum`,
+    sql`SELECT 1 + 1 AS sum`,
   ]);
@@ -107,0 +129,0 @@

No alert changes

Improved metrics

Total package byte prevSize

79962

increased by2.31%

Lines of code

2301

increased by3.05%

Dependency changes

• + Added@lbu/insight@0.0.66(transitive)

• + Added@lbu/stdlib@0.0.66(transitive)

• - Removed@lbu/insight@0.0.65(transitive)

• - Removed@lbu/stdlib@0.0.65(transitive)

• Updated@lbu/insight@0.0.66

• Updated@lbu/stdlib@0.0.66