optimade - npm Package Compare versions

Comparing version 1.2.1 to 2.0.0

CHANGELOG.md

 # optimade changelog
 
+## 2.0
+
+- Add new `npm run prefetch` logic in prefetch.js to check provider avialability,
+  cache provider pagination limits in `provider.attributes.query_limits`,
+  introduce the sorted and structured providers.json (as source) and prefetched.json (as cache)
+- Add new arguments `getStructures(providerId, filter, page, limit)` for pagination
+  and pagination limits
+- Add new logic for catching errors like
+  `Error: messageFromProvider { response: { errors, meta } }`
+
 ## 1.2.1
 
-* Fix providers prefetching
+- Fix providers prefetching
 
 ## 1.2.0
 
-* Make batch data aggregation is optional to support per-provider progressive data receiving
+- Make batch data aggregation is optional to support per-provider progressive data receiving
 
 ## 1.1.5
 
-* Add http request timeout
+- Add http request timeout
 
 ## 1.0.0
 
-* First release
+- First release

package.json

 {
   "name": "optimade",
-  "version": "1.2.1",
+  "version": "2.0.0",
   "description": "Aggregating Optimade client for the online materials databases",
@@ -5,0 +5,0 @@ "main": "dist/index.js",

prefetch.js

 #!/usr/bin/env node
 
+/* jshint esversion: 6 */
+
 const fs = require('fs');
@@ -12,17 +14,68 @@ const path = require('path');
 const optimade = new Optimade({
-    providersUrl: 'https://providers.optimade.org/providers.json'
+	providersUrl: 'https://providers.optimade.org/providers.json'
 });
 
-optimade.getProviders().then(() => {
+optimade.getProviders().then(async () => {
 
-    const data = {
-        providers: optimade.providers,
-        apis: optimade.apis,
-    };
+	const filteredApis = Object.entries(optimade.apis).filter(([k, v]) => v.length);
+	const apis = filteredApis.sort().reduce((acc, [k, v]) => {
+		return { ...acc, ...{ [k]: v } };
+	}, {});
 
-    fs.writeFile(path.join(__dirname, 'dist/prefetched.json'), JSON.stringify(data), (err) => {
-        if (err) throw err;
-        console.log('The cache file has been saved!');
-    });
+	const source = Object.keys(optimade.providers).sort().reduce(
+		(obj, key) => {
+			obj[key] = optimade.providers[key];
+			return obj;
+		}, {});
 
-});
+	async function getQueryLimits(providers, max = 1000) {
+
+		const fetchLimits = async (k, v) => {
+			const formula = `chemical_formula_anonymous="A2B"`;
+			const url = `${v.attributes.base_url}/v1/structures?filter=${formula}&page_limit=${max}`;
+			try {
+				const res = await fetch(url).then(res => res.json());
+				const api = res.meta && res.meta.api_version;
+				console.dir(res);
+				const detail = (e) => {
+					return e
+						? e.length
+							? e[0].detail
+							: e.detail
+						: '0';
+				};
+				const nums = detail(res.errors).match(/\d+/g).filter(n => +n < max).map(n => +n);
+				if (!nums.includes(0))
+					return {
+						[k]: { ...v, attributes: { ...v.attributes, api_version: api, ['query_limits']: nums } }
+					};
+			} catch (error) {
+				console.log(error);
+			}
+		};
+
+		providers = await Object.entries(providers).reduce(async (promise, [k, v]) => {
+			const provider = await fetchLimits(k, v);
+			const acc = await promise;
+			return { ...acc, ...provider };
+		}, Promise.resolve({}));
+
+		const log = { prefetched: Object.keys(providers).length, source: Object.keys(source).length };
+		console.log(log);
+
+		return providers;
+	}
+
+	getQueryLimits(source).then(providers => {
+		const data = { providers, apis };
+		fs.writeFile(path.join(__dirname, 'dist/prefetched.json'), JSON.stringify(data), (err) => {
+			if (err) throw err;
+			console.log('The prefetched.json file has been saved!');
+		});
+		fs.writeFile(path.join(__dirname, 'dist/providers.json'), JSON.stringify(source), (err) => {
+			if (err) throw err;
+			console.log('The providers.json file has been saved!');
+		});
+	});
+
+});

README.md

 # Aggregating Optimade client for the online materials databases
 
-[![NPM version](https://img.shields.io/npm/v/optimade.svg?style=flat)](https://www.npmjs.com/package/optimade) [![NPM downloads](https://img.shields.io/npm/dm/optimade.svg?style=flat)](https://www.npmjs.com/package/optimade) [![GitHub issues](https://img.shields.io/github/issues/tilde-lab/optimade-client?style=flat)](https://github.com/tilde-lab/optimade-client/issues)
+[![NPM version](https://img.shields.io/npm/v/optimade.svg?style=flat)](https://www.npmjs.com/package/optimade)
+[![NPM downloads](https://img.shields.io/npm/dm/optimade.svg?style=flat)](https://www.npmjs.com/package/optimade)
+[![GitHub issues](https://img.shields.io/github/issues/tilde-lab/optimade-client?style=flat)](https://github.com/tilde-lab/optimade-client/issues)
 
@@ -10,16 +12,20 @@ ## Features
 - queries them all, in the browser, at the server, everywhere
+- provides pagination, with the minimized number of pages
 
 ## Install
 
-```bash
+```sh
 npm i optimade --save
 ```
 
-```bash
+```sh
 yarn add optimade
 ```
 
-CDN: [UNPKG](https://unpkg.com/optimade/) | [jsDelivr](https://cdn.jsdelivr.net/npm/optimade/) (available as `window.optimade`)
+CDN: [UNPKG](https://unpkg.com/optimade/) |
+[jsDelivr](https://cdn.jsdelivr.net/npm/optimade/) (available as
+`window.optimade`)
 
-If you are **not** using es6 or CDN, add to your HTML just before closing the `body` tag:
+If you are **not** using ES6 or CDN, add to your HTML just before closing the
+`body` tag:
 
@@ -34,6 +40,7 @@ ```html
 
-```javascript
+### Discovery and querying
 
+```ts
 const optimadeClient = new Optimade({
-    providersUrl: '/path/to/optimade/providers.json'
+  providersUrl: "/path/to/optimade/providers.json",
 });
@@ -47,11 +54,44 @@
 
-const results = await optimadeClient.getStructuresAll(providerIds, YOUR_OPTIMADE_QUERY); // [Structures[], Provider][]
+const results = await optimadeClient.getStructuresAll(
+  providerIds,
+  YOUR_OPTIMADE_QUERY,
+); // [StructuresResponse[], Provider][]
 ```
 
-Importing depends on your environment. See also the `examples` folder. The `.html` examples are suited for the browser environment, the `.js` examples are suited for the server environment.
+Importing depends on your environment. See also the `examples` folder. The
+`.html` examples are suited for the browser environment, the `.js` examples are
+suited for the server environment.
 
-The code is generally isomorphic, however one should additionally take care of downloading the cache or setting the CORS policy for the browsers. Concerning the CORS, the `Optimade` class constructor accepts the `corsProxyUrl` parameter, pointing to _e.g._  a running `cors-anywhere` proxy instance. This will be valid until all the Optimade providers are supplying the header `Access-Control-Allow-Origin $http_origin` in their responses. For the server-side environment this is not required.
+### Pagination
 
+```ts
+import prefetched from 'optimade/dist/prefetched.json';
+
+const optimadeClient = new Optimade({
+  providersUrl: "/path/to/optimade/providers.json",
+});
+
+optimadeClient.providers = prefetched.providers;
+optimadeClient.apis = prefetched.apis;
+
+const results = await optimadeClient.getStructuresAll(
+  providerIds,
+  YOUR_OPTIMADE_QUERY,
+  page: number,
+  limit: number
+); // [StructuresResponse[], Provider][]
+```
+
+See also the `demo` folder.
+
+The code is generally isomorphic, however one should additionally take care of
+downloading the cache or setting the CORS policy for the browsers. Concerning
+the CORS, the `Optimade` class constructor accepts the `corsProxyUrl` parameter,
+pointing to a running `cors-anywhere` proxy instance. This will be valid
+until all the Optimade providers are supplying the header
+`Access-Control-Allow-Origin $http_origin` in their responses. For the
+server-side environment this is not required.
+
 ## License
 
-MIT © [PaulMaly](https://github.com/PaulMaly), Tilde Materials Informatics
+MIT © [Pavel Malyshev](https://github.com/PaulMaly) and [Alexander Volkov](https://github.com/valexr), Tilde Materials Informatics

src/index.ts

@@ -1,2 +0,1 @@
-
 import { allSettled, fetchWithTimeout } from './utils';
@@ -10,7 +9,7 @@
     private corsProxyUrl: string = '';
-    public providers: Types.ProvidersMap | null = null;
-    public apis: Types.ApisMap = {};
+    providers: Types.ProvidersMap | null = null;
+    apis: Types.ApisMap = {};
     private reqStack: string[] = [];
 
-    constructor({ providersUrl = '', corsProxyUrl = '' } : {providersUrl?: string; corsProxyUrl?: string} = {} ) {
+    constructor({ providersUrl = '', corsProxyUrl = '' }: { providersUrl?: string; corsProxyUrl?: string; } = {}) {
         this.corsProxyUrl = corsProxyUrl;
@@ -68,3 +67,3 @@ this.providersUrl = this.wrapUrl(providersUrl);
 
-    async getStructures(providerId: string, filter: string = ''): Promise<Types.Structure[] | null> {
+    async getStructures(providerId: string, filter: string = '', page: number = 1, limit: number): Promise<Types.StructuresResponse[] | Types.ResponseError> {
 
@@ -74,17 +73,34 @@ if (!this.apis[providerId]) return null;
         const apis = this.apis[providerId].filter(api => api.attributes.available_endpoints.includes('structures'));
+        const provider = this.providers[providerId];
 
-        const structures: Types.StructuresResponse[] = await allSettled(apis.map((api: Types.Api) => {
-            const url: string = this.wrapUrl(Optimade.apiVersionUrl(api), filter ? `/structures?filter=${filter}` : '/structures');
-            // TODO pagination e.g. url += (filter ? '&' : '?') + 'page_limit=100'
-            return Optimade.getJSON(url);
+        const structures: Types.StructuresResponse[] = await allSettled(apis.map(async (api: Types.Api) => {
+            if (page <= 0) page = 1;
+            const pageLimit = limit ? `&page_limit=${limit}` : '';
+            const pageNumber = page ? `&page_number=${page - 1}` : '';
+            const pageOffset = limit && page ? `&page_offset=${limit * (page - 1)}` : '';
+            const params = filter ? `${pageLimit + pageNumber + pageOffset}` : `?${pageLimit}`;
+            const url: string = this.wrapUrl(Optimade.apiVersionUrl(api), filter ? `/structures?filter=${filter + params}` : `/structures${params}`);
+
+            try {
+                return await Optimade.getJSON(url, {}, { Origin: 'https://cors.optimade.science', 'X-Requested-With': 'XMLHttpRequest' });
+            } catch (error) {
+                return error;
+            }
         }));
 
-        //console.log('Ready ' + providerId);
-        return structures.reduce((structures: Types.Structure[], structure: Types.StructuresResponse | null) => {
-            return structure ? structures.concat(structure.data) : structures;
+        return structures.reduce((structures: any[], structure: Types.StructuresResponse | Types.ResponseError): Types.StructuresResponse[] => {
+            console.dir(`optimade-client-${providerId}:`, structure);
+
+            if (structure instanceof Error || Object.keys(structure).includes('errors')) {
+                return structures.concat(structure);
+            } else {
+                structure.meta.pages = Math.ceil(structure.meta.data_returned / (limit || structure.data.length));
+                structure.meta.limits = provider.attributes.query_limits;
+                return structures.concat(structure);
+            }
         }, []);
     }
 
-    getStructuresAll(providerIds: string[], filter: string = '', batch: boolean = true): Promise<Promise<Types.StructuresResult>[]> | Promise<Types.StructuresResult>[] {
-        
+    getStructuresAll(providerIds: string[], filter: string = '', page: number = 0, limit: number, batch: boolean = true): Promise<Promise<Types.StructuresResult>[]> | Promise<Types.StructuresResult>[] {
+
         const results = providerIds.reduce((structures: Promise<any>[], providerId: string) => {
@@ -94,3 +110,3 @@ const provider = this.providers[providerId];
                 structures.push(allSettled([
-                    this.getStructures(providerId, filter),
+                    this.getStructures(providerId, filter, page, limit),
                     Promise.resolve(provider)
@@ -134,5 +150,6 @@ ]));
         if (!res.ok) {
-            const err: Types.ResponseError = new Error(res.statusText);
-            err.response = res;
-            throw err;
+            const err: Types.ErrorResponse = await res.json();
+            const error: Types.ResponseError = new Error(err.errors[0].detail);
+            error.response = err;
+            throw error;
         }
@@ -139,0 +156,0 @@

src/types.ts

@@ -21,3 +21,5 @@ export interface Meta {
         prefix: string;
-    };
+    },
+    pages?: number;
+    limits?: number[];
 }
@@ -42,4 +44,4 @@
 export interface Api {
+    id: string;
     type: string;
-    id: string;
     attributes: {
@@ -63,2 +65,3 @@ api_version: string;
         link_type?: string;
+        query_limits?: number[];
     };
@@ -93,3 +96,3 @@ }
 export interface StructuresResponse {
-    data: Structure[];
+    data?: Structure[];
     links?: Links;
@@ -109,5 +112,16 @@ meta?: Meta;
 
-export type ProvidersMap = { [key: string]: Provider }
-export type ApisMap = { [key: string]: Api[] }
+export interface ErrorObject {
+    status: string;
+    title: string;
+    detail?: string;
+    length?: string;
+}
+export interface ErrorResponse {
+    errors: ErrorObject;
+    meta: any;
+}
 
-export type StructuresResult = [Promise<Structure>, Promise<Provider>][];
+export type ProvidersMap = { [key: string]: Provider; };
+export type ApisMap = { [key: string]: Api[]; };
+
+export type StructuresResult = [Promise<StructuresResponse[]>, Promise<Provider>][];

src/utils.ts

@@ -17,3 +17,3 @@
         return await fetch(url, { ...options, signal });
-    } catch(err) {
+    } catch (err) {
         if (err.name === 'AbortError') {
@@ -20,0 +20,0 @@ throw new Error('Request timed out');

No alert changes

Improved metrics

Number of lines in readme file

95

72.73%

Worsened metrics

Total package byte prevSize

21300

-58.31%

Number of package files

13

-27.78%

Lines of code

454

-43.53%

No dependency changes