@nuclia/sync-agent - npm Package Compare versions

Comparing version 1.0.11 to 1.1.0

build/logic/connector/domain/connector.js

@@ -18,4 +18,5 @@ "use strict";
     modifiedGMT: zod_1.z.string().optional(),
+    mimeType: zod_1.z.string().optional(),
     isFolder: zod_1.z.boolean().optional(),
     parents: zod_1.z.array(zod_1.z.string()).optional(),
 });

build/logic/connector/infrastructure/connectors/folder.connector.js

@@ -35,2 +35,3 @@ "use strict";
 const connector_1 = require("../../domain/connector");
+const mime_types_1 = require("mime-types");
 const FILES_TO_IGNORE = ['.DS_Store', 'Thumbs.db'];
@@ -154,2 +155,3 @@ exports.FolderConnector = {
             originalId: file,
+            mimeType: (0, mime_types_1.lookup)(file) || 'application/octet-stream',
             metadata: {
@@ -156,0 +158,0 @@ path: file.split('/').slice(0, -1).join('/'),

build/logic/connector/infrastructure/connectors/gdrive.connector.js

@@ -213,5 +213,5 @@ "use strict";
             parents: item.parents,
+            mimeType: needsPdfConversion ? 'application/pdf' : item.mimeType,
             metadata: {
                 needsPdfConversion: needsPdfConversion ? 'yes' : 'no',
-                mimeType: needsPdfConversion ? 'application/pdf' : item.mimeType,
             },
@@ -218,0 +218,0 @@ status: connector_1.FileStatus.PENDING,

build/logic/connector/infrastructure/connectors/onedrive.connector.js

@@ -176,4 +176,4 @@ "use strict";
             modifiedGMT: item.lastModifiedDateTime,
+            mimeType: item.file.mimeType,
             metadata: {
-                mimeType: item.file.mimeType,
                 downloadLink: item['@microsoft.graph.downloadUrl'],
@@ -180,0 +180,0 @@ path: item.parentReference.path,

build/logic/connector/infrastructure/connectors/sitemap.connector.js

@@ -104,2 +104,3 @@ "use strict";
                 originalId: parsedUrl.loc,
+                mimeType: 'text/html',
                 metadata: {
@@ -106,0 +107,0 @@ uri: parsedUrl.loc,

build/logic/sync/domain/dto/create-sync.dto.js

@@ -18,2 +18,3 @@ "use strict";
             id: this.options.id,
+            filters: this.options.filters,
         };
@@ -48,2 +49,10 @@ if (this.options.labels)
         }
+        if (props.filters) {
+            try {
+                (0, validate_1.validateZodSchema)(sync_entity_1.FiltersValidator, props.filters);
+            }
+            catch (error) {
+                return [`Invalid format for filters: ${error}`];
+            }
+        }
         return [undefined, new CreateSyncDto(Object.assign(Object.assign({}, props), { id }))];
@@ -50,0 +59,0 @@ }

build/logic/sync/domain/dto/update-sync.dto.js

@@ -6,2 +6,3 @@ "use strict";
 const factory_1 = require("../../../connector/infrastructure/factory");
+const sync_entity_1 = require("../sync.entity");
 const validate_1 = require("./validate");
@@ -28,6 +29,8 @@ class UpdateSyncDto {
             returnObj.foldersToSync = this.options.foldersToSync;
+        if (this.options.filters)
+            returnObj.filters = this.options.filters;
         return returnObj;
     }
     static create(props) {
-        const { id, connector, kb, foldersToSync } = props;
+        const { id, connector, kb, foldersToSync, filters } = props;
         if (!id) {
@@ -56,2 +59,10 @@ return ['id is mandatory'];
         }
+        if (filters) {
+            try {
+                (0, validate_1.validateZodSchema)(sync_entity_1.FiltersValidator, filters);
+            }
+            catch (error) {
+                return [`Invalid format for filters: ${error}`];
+            }
+        }
         if (foldersToSync && foldersToSync.length > 0) {
@@ -58,0 +69,0 @@ let errorMsg = '';

build/logic/sync/domain/nuclia-cloud.js

@@ -68,3 +68,3 @@ "use strict";
                             .upload('file', buffer, false, {
-                            contentType: data.metadata.mimeType || (0, mime_types_1.lookup)(filename) || 'application/octet-stream',
+                            contentType: data.mimeType || (0, mime_types_1.lookup)(filename) || 'application/octet-stream',
                             filename,
@@ -71,0 +71,0 @@ })

build/logic/sync/domain/sync.entity.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SyncEntity = exports.NucliaOptionsValidator = void 0;
+exports.SyncEntity = exports.FiltersValidator = exports.NucliaOptionsValidator = void 0;
 const rxjs_1 = require("rxjs");
@@ -34,6 +34,20 @@ const zod_1 = require("zod");
 });
+exports.FiltersValidator = zod_1.z.object({
+    fileExtensions: zod_1.z
+        .object({
+        extensions: zod_1.z.string(),
+        exclude: zod_1.z.boolean().optional(),
+    })
+        .optional(),
+    modified: zod_1.z
+        .object({
+        from: zod_1.z.string().optional(),
+        to: zod_1.z.string().optional(),
+    })
+        .optional(),
+});
 class SyncEntity {
     constructor(options) {
         this.foldersToSync = [];
-        const { connector, kb, labels, title, id, lastSyncGMT, foldersToSync } = options;
+        const { connector, kb, labels, title, id, lastSyncGMT, foldersToSync, filters } = options;
         this.connector = connector;
@@ -46,2 +60,3 @@ this.kb = kb;
         this.foldersToSync = foldersToSync;
+        this.filters = filters;
         this.setConnectorDefinition();
@@ -48,0 +63,0 @@ }

build/logic/sync/domain/use-cases/sync-all-folders-data.use-case.js

@@ -23,2 +23,3 @@ "use strict";
 const update_sync_use_case_1 = require("./update-sync.use-case");
+const mime_types_1 = require("mime-types");
 require('localstorage-polyfill');
@@ -69,3 +70,30 @@ class SyncAllFolders {
     processItems(syncEntity, items) {
-        return (0, rxjs_1.of)(...items).pipe((0, rxjs_1.concatMap)((item) => new sync_single_file_use_case_1.SyncSingleFile(syncEntity, item).execute().pipe((0, rxjs_1.map)((res) => ({ id: item.originalId, success: res.success })), 
+        var _a, _b;
+        const filteredMimetypes = (((_b = (_a = syncEntity.filters) === null || _a === void 0 ? void 0 : _a.fileExtensions) === null || _b === void 0 ? void 0 : _b.extensions) || '')
+            .split(',')
+            .map((ext) => (0, mime_types_1.lookup)(ext.trim()) || '')
+            .filter((ext) => ext);
+        return (0, rxjs_1.of)(...items).pipe((0, rxjs_1.filter)((item) => {
+            var _a, _b, _c;
+            let isExtensionOk = true;
+            if (!item.mimeType) {
+                isExtensionOk = false;
+            }
+            else {
+                if (filteredMimetypes.length > 0) {
+                    const isFiltered = filteredMimetypes.includes(item.mimeType);
+                    isExtensionOk = ((_b = (_a = syncEntity.filters) === null || _a === void 0 ? void 0 : _a.fileExtensions) === null || _b === void 0 ? void 0 : _b.exclude) ? !isFiltered : isFiltered;
+                }
+            }
+            let isDateOk = true;
+            if (item.modifiedGMT && ((_c = syncEntity.filters) === null || _c === void 0 ? void 0 : _c.modified)) {
+                if (syncEntity.filters.modified.from) {
+                    isDateOk = item.modifiedGMT >= syncEntity.filters.modified.from;
+                }
+                if (syncEntity.filters.modified.to) {
+                    isDateOk = isDateOk && item.modifiedGMT <= syncEntity.filters.modified.to;
+                }
+            }
+            return isDateOk && isExtensionOk;
+        }), (0, rxjs_1.concatMap)((item) => new sync_single_file_use_case_1.SyncSingleFile(syncEntity, item).execute().pipe((0, rxjs_1.map)((res) => ({ id: item.originalId, success: res.success })), 
         // do not overwhelm the source
@@ -72,0 +100,0 @@ (0, rxjs_1.delay)(500))), (0, rxjs_1.toArray)());

build/logic/sync/domain/use-cases/sync-single-file.use-case.js

@@ -40,2 +40,3 @@ "use strict";
                             metadata: Object.assign(Object.assign({}, item.metadata), { labels: sync.labels }),
+                            mimeType: item.mimeType,
                         });
@@ -42,0 +43,0 @@ }));

build/types/logic/connector/domain/connector.d.ts

@@ -24,2 +24,3 @@ import { Observable } from 'rxjs';
     modifiedGMT: z.ZodOptional<z.ZodString>;
+    mimeType: z.ZodOptional<z.ZodString>;
     isFolder: z.ZodOptional<z.ZodBoolean>;
@@ -34,2 +35,3 @@ parents: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
     modifiedGMT?: string | undefined;
+    mimeType?: string | undefined;
     isFolder?: boolean | undefined;
@@ -44,2 +46,3 @@ parents?: string[] | undefined;
     modifiedGMT?: string | undefined;
+    mimeType?: string | undefined;
     isFolder?: boolean | undefined;
@@ -46,0 +49,0 @@ parents?: string[] | undefined;

build/types/logic/sync/domain/nuclia-cloud.d.ts

@@ -12,2 +12,3 @@ import { Observable } from 'rxjs';
         metadata?: any;
+        mimeType?: string;
     }): Observable<{
@@ -14,0 +15,0 @@ success: boolean;

build/types/logic/sync/domain/sync.entity.d.ts

@@ -48,2 +48,43 @@ import { Observable } from 'rxjs';
 export type Classification = {};
+export declare const FiltersValidator: z.ZodObject<{
+    fileExtensions: z.ZodOptional<z.ZodObject<{
+        extensions: z.ZodString;
+        exclude: z.ZodOptional<z.ZodBoolean>;
+    }, "strip", z.ZodTypeAny, {
+        extensions: string;
+        exclude?: boolean | undefined;
+    }, {
+        extensions: string;
+        exclude?: boolean | undefined;
+    }>>;
+    modified: z.ZodOptional<z.ZodObject<{
+        from: z.ZodOptional<z.ZodString>;
+        to: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        from?: string | undefined;
+        to?: string | undefined;
+    }, {
+        from?: string | undefined;
+        to?: string | undefined;
+    }>>;
+}, "strip", z.ZodTypeAny, {
+    fileExtensions?: {
+        extensions: string;
+        exclude?: boolean | undefined;
+    } | undefined;
+    modified?: {
+        from?: string | undefined;
+        to?: string | undefined;
+    } | undefined;
+}, {
+    fileExtensions?: {
+        extensions: string;
+        exclude?: boolean | undefined;
+    } | undefined;
+    modified?: {
+        from?: string | undefined;
+        to?: string | undefined;
+    } | undefined;
+}>;
+export type Filters = z.infer<typeof FiltersValidator>;
 export interface ISyncEntity {
@@ -57,2 +98,3 @@ connector: Connector;
     foldersToSync?: SyncItem[];
+    filters?: Filters;
 }
@@ -68,2 +110,3 @@ export declare class SyncEntity {
     foldersToSync?: SyncItem[];
+    filters?: Filters;
     constructor(options: ISyncEntity);
@@ -70,0 +113,0 @@ private setConnectorDefinition;

package.json

 {
   "name": "@nuclia/sync-agent",
-  "version": "1.0.11",
+  "version": "1.1.0",
   "description": "This is a sync agent to synchronize user files from diferent sources to nuclia",
@@ -5,0 +5,0 @@ "main": "build/index.js",

No alert changes

Improved metrics

Total package byte prevSize

173840

increased by2.52%

Lines of code

4129

increased by2.86%

No dependency changes