@google-cloud/bigquery - npm Package Compare versions

Comparing version 7.5.2 to 7.6.0

build/src/bigquery.d.ts

@@ -24,2 +24,3 @@ /*!
 import bigquery from './types';
+import { setLogFunction } from './logger';
 export { common };
@@ -90,2 +91,4 @@ export interface RequestCallback<T> {
 export type GetJobsCallback = PagedCallback<Job, GetJobsOptions, bigquery.IJobList>;
+export type JobsQueryResponse = [Job, bigquery.IQueryResponse];
+export type JobsQueryCallback = ResourceCallback<Job, bigquery.IQueryResponse>;
 export interface BigQueryTimeOptions {
@@ -116,2 +119,3 @@ hours?: number | string;
 export type QueryParameter = bigquery.IQueryParameter;
+export type ParameterMode = bigquery.IJobConfigurationQuery['parameterMode'];
 export interface BigQueryOptions extends GoogleAuthOptions {
@@ -197,2 +201,8 @@ /**
  *
+ * This client supports enabling query-related preview features via environmental
+ * variables.  By setting the environment variable QUERY_PREVIEW_ENABLED to the string
+ * "TRUE", the client will enable preview features, though behavior may still be
+ * controlled via the bigquery service as well.  Currently, the feature(s) in scope
+ * include: stateless queries (query execution without corresponding job metadata).
+ *
  * @class
@@ -234,2 +244,3 @@ *
     private _universeDomain;
+    private _enableQueryPreview;
     createQueryStream(options?: Query | string): ResourceStream<RowMetadata>;
@@ -239,2 +250,3 @@ getDatasetsStream(options?: GetDatasetsOptions): ResourceStream<Dataset>;
     constructor(options?: BigQueryOptions);
+    private trace_;
     get universeDomain(): string;
@@ -716,2 +728,3 @@ private static sanitizeEndpoint;
     createQueryJob(options: Query | string, callback: JobCallback): void;
+    private buildQueryParams_;
     /**
@@ -1058,2 +1071,13 @@ * Creates a job. Typically when creating a job you'll have a very specific
     /**
+     * Check if the given Query can run using the `jobs.query` endpoint.
+     * Returns a bigquery.IQueryRequest that can be used to call `jobs.query`.
+     * Return undefined if is not possible to convert to a bigquery.IQueryRequest.
+     *
+     * @param query string | Query
+     * @param options QueryOptions
+     * @returns bigquery.IQueryRequest | undefined
+     */
+    private buildQueryRequest_;
+    private runJobsQuery;
+    /**
      * This method will be called by `createQueryStream()`. It is required to
@@ -1065,2 +1089,3 @@ * properly set the `autoPaginate` option value.
     queryAsStream_(query: Query, callback?: SimpleQueryRowsCallback): void;
+    static setLogFunction: typeof setLogFunction;
 }
@@ -1067,0 +1092,0 @@ /**

build/src/bigquery.js

@@ -33,2 +33,3 @@ "use strict";
 const table_1 = require("./table");
+const logger_1 = require("./logger");
 exports.PROTOCOL_REGEX = /^(\w*):\/\//;
@@ -69,2 +70,8 @@ /**
  *
+ * This client supports enabling query-related preview features via environmental
+ * variables.  By setting the environment variable QUERY_PREVIEW_ENABLED to the string
+ * "TRUE", the client will enable preview features, though behavior may still be
+ * controlled via the bigquery service as well.  Currently, the feature(s) in scope
+ * include: stateless queries (query execution without corresponding job metadata).
+ *
  * @class
@@ -146,2 +153,9 @@ *
         super(config, options);
+        const QUERY_PREVIEW_ENABLED = process.env.QUERY_PREVIEW_ENABLED;
+        this._enableQueryPreview = false;
+        if (typeof QUERY_PREVIEW_ENABLED === 'string') {
+            if (QUERY_PREVIEW_ENABLED.toUpperCase() === 'TRUE') {
+                this._enableQueryPreview = true;
+            }
+        }
         this._universeDomain = universeDomain;
@@ -256,2 +270,6 @@ this.location = options.location;
     }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    trace_(msg, ...otherArgs) {
+        (0, logger_1.logger)('[bigquery]', msg, ...otherArgs);
+    }
     get universeDomain() {
@@ -378,3 +396,4 @@ return this._universeDomain;
                 case 'TIMESTAMP': {
-                    const pd = new precise_date_1.PreciseDate(BigInt(value) * BigInt(1000));
+                    const pd = new precise_date_1.PreciseDate();
+                    pd.setFullTime(precise_date_1.PreciseDate.parseFull(BigInt(value) * BigInt(1000)));
                     value = BigQuery.timestamp(pd);
@@ -968,9 +987,10 @@ break;
         const options = typeof opts === 'object' ? opts : { query: opts };
+        this.trace_('[createQueryJob]', options, callback);
         if ((!options || !options.query) && !options.pageToken) {
             throw new Error('A SQL query string is required.');
         }
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
         const query = extend(true, {
             useLegacySql: false,
         }, options);
+        this.trace_('[createQueryJob]', query);
         if (options.destination) {
@@ -987,59 +1007,12 @@ if (!(options.destination instanceof table_1.Table)) {
         }
-        if (query.params) {
-            query.parameterMode = is.array(query.params) ? 'positional' : 'named';
-            if (query.parameterMode === 'named') {
-                query.queryParameters = [];
-                // tslint:disable-next-line forin
-                for (const namedParameter in query.params) {
-                    const value = query.params[namedParameter];
-                    let queryParameter;
-                    if (query.types) {
-                        if (!is.object(query.types)) {
-                            throw new Error('Provided types must match the value type passed to `params`');
-                        }
-                        if (query.types[namedParameter]) {
-                            queryParameter = BigQuery.valueToQueryParameter_(value, query.types[namedParameter]);
-                        }
-                        else {
-                            queryParameter = BigQuery.valueToQueryParameter_(value);
-                        }
-                    }
-                    else {
-                        queryParameter = BigQuery.valueToQueryParameter_(value);
-                    }
-                    queryParameter.name = namedParameter;
-                    query.queryParameters.push(queryParameter);
-                }
-            }
-            else {
-                query.queryParameters = [];
-                if (query.types) {
-                    if (!is.array(query.types)) {
-                        throw new Error('Provided types must match the value type passed to `params`');
-                    }
-                    if (query.params.length !== query.types.length) {
-                        throw new Error('Incorrect number of parameter types provided.');
-                    }
-                    query.params.forEach((value, i) => {
-                        const queryParameter = BigQuery.valueToQueryParameter_(value, query.types[i]);
-                        query.queryParameters.push(queryParameter);
-                    });
-                }
-                else {
-                    query.params.forEach((value) => {
-                        const queryParameter = BigQuery.valueToQueryParameter_(value);
-                        query.queryParameters.push(queryParameter);
-                    });
-                }
-            }
-            delete query.params;
-        }
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const reqOpts = {
-            configuration: {
-                query,
-            },
+        const { parameterMode, params } = this.buildQueryParams_(query.params, query.types);
+        query.parameterMode = parameterMode;
+        query.queryParameters = params;
+        delete query.params;
+        const reqOpts = {};
+        reqOpts.configuration = {
+            query,
         };
         if (typeof query.jobTimeoutMs === 'number') {
-            reqOpts.configuration.jobTimeoutMs = query.jobTimeoutMs;
+            reqOpts.configuration.jobTimeoutMs = query.jobTimeoutMs.toString();
             delete query.jobTimeoutMs;
@@ -1069,2 +1042,61 @@ }
     }
+    buildQueryParams_(params, types) {
+        if (!params) {
+            return {
+                parameterMode: undefined,
+                params: undefined,
+            };
+        }
+        const parameterMode = is.array(params) ? 'positional' : 'named';
+        const queryParameters = [];
+        if (parameterMode === 'named') {
+            const namedParams = params;
+            for (const namedParameter of Object.getOwnPropertyNames(namedParams)) {
+                const value = namedParams[namedParameter];
+                let queryParameter;
+                if (types) {
+                    if (!is.object(types)) {
+                        throw new Error('Provided types must match the value type passed to `params`');
+                    }
+                    const namedTypes = types;
+                    if (namedTypes[namedParameter]) {
+                        queryParameter = BigQuery.valueToQueryParameter_(value, namedTypes[namedParameter]);
+                    }
+                    else {
+                        queryParameter = BigQuery.valueToQueryParameter_(value);
+                    }
+                }
+                else {
+                    queryParameter = BigQuery.valueToQueryParameter_(value);
+                }
+                queryParameter.name = namedParameter;
+                queryParameters.push(queryParameter);
+            }
+        }
+        else {
+            if (types) {
+                if (!is.array(types)) {
+                    throw new Error('Provided types must match the value type passed to `params`');
+                }
+                const positionalTypes = types;
+                if (params.length !== types.length) {
+                    throw new Error('Incorrect number of parameter types provided.');
+                }
+                params.forEach((value, i) => {
+                    const queryParameter = BigQuery.valueToQueryParameter_(value, positionalTypes[i]);
+                    queryParameters.push(queryParameter);
+                });
+            }
+            else {
+                params.forEach((value) => {
+                    const queryParameter = BigQuery.valueToQueryParameter_(value);
+                    queryParameters.push(queryParameter);
+                });
+            }
+        }
+        return {
+            parameterMode,
+            params: queryParameters,
+        };
+    }
     createJob(options, callback) {
@@ -1252,14 +1284,51 @@ var _a;
         const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
-        this.createQueryJob(query, (err, job, resp) => {
+        this.trace_('[query]', query, options);
+        const queryReq = this.buildQueryRequest_(query, options);
+        this.trace_('[query] queryReq', queryReq);
+        if (!queryReq) {
+            this.createQueryJob(query, (err, job, resp) => {
+                if (err) {
+                    callback(err, null, resp);
+                    return;
+                }
+                if (typeof query === 'object' && query.dryRun) {
+                    callback(null, [], resp);
+                    return;
+                }
+                // The Job is important for the `queryAsStream_` method, so a new query
+                // isn't created each time results are polled for.
+                options = extend({ job }, queryOpts, options);
+                job.getQueryResults(options, callback);
+            });
+            return;
+        }
+        this.runJobsQuery(queryReq, (err, job, res) => {
+            this.trace_('[runJobsQuery callback]: ', query, err, job, res);
             if (err) {
-                callback(err, null, resp);
+                callback(err, null, res);
                 return;
             }
-            if (typeof query === 'object' && query.dryRun) {
-                callback(null, [], resp);
+            options = extend({ job }, queryOpts, options);
+            if (res && res.jobComplete) {
+                let rows = [];
+                if (res.schema && res.rows) {
+                    rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, {
+                        wrapIntegers: options.wrapIntegers || false,
+                        parseJSON: options.parseJSON,
+                    });
+                }
+                this.trace_('[runJobsQuery] job complete');
+                options._cachedRows = rows;
+                if (res.pageToken) {
+                    this.trace_('[runJobsQuery] has more pages');
+                    options.pageToken = res.pageToken;
+                }
+                else {
+                    this.trace_('[runJobsQuery] no more pages');
+                }
+                job.getQueryResults(options, callback);
                 return;
             }
-            // The Job is important for the `queryAsStream_` method, so a new query
-            // isn't created each time results are polled for.
-            options = extend({ job }, queryOpts, options);
+            delete options.timeoutMs;
+            this.trace_('[runJobsQuery] job not complete');
             job.getQueryResults(options, callback);
@@ -1269,2 +1338,101 @@ });
     /**
+     * Check if the given Query can run using the `jobs.query` endpoint.
+     * Returns a bigquery.IQueryRequest that can be used to call `jobs.query`.
+     * Return undefined if is not possible to convert to a bigquery.IQueryRequest.
+     *
+     * @param query string | Query
+     * @param options QueryOptions
+     * @returns bigquery.IQueryRequest | undefined
+     */
+    buildQueryRequest_(query, options) {
+        if (process.env.FAST_QUERY_PATH === 'DISABLED') {
+            return undefined;
+        }
+        const queryObj = typeof query === 'string'
+            ? {
+                query: query,
+            }
+            : query;
+        this.trace_('[buildQueryRequest]', query, options, queryObj);
+        // This is a denylist of settings which prevent us from composing an equivalent
+        // bq.QueryRequest due to differences between configuration parameters accepted
+        // by jobs.insert vs jobs.query.
+        if (!!queryObj.destination ||
+            !!queryObj.tableDefinitions ||
+            !!queryObj.createDisposition ||
+            !!queryObj.writeDisposition ||
+            (!!queryObj.priority && queryObj.priority !== 'INTERACTIVE') ||
+            queryObj.useLegacySql ||
+            !!queryObj.maximumBillingTier ||
+            !!queryObj.timePartitioning ||
+            !!queryObj.rangePartitioning ||
+            !!queryObj.clustering ||
+            !!queryObj.destinationEncryptionConfiguration ||
+            !!queryObj.schemaUpdateOptions ||
+            !!queryObj.jobTimeoutMs ||
+            // User has defined the jobID generation behavior
+            !!queryObj.jobId) {
+            return undefined;
+        }
+        if (queryObj.dryRun) {
+            return undefined;
+        }
+        if (options.job) {
+            return undefined;
+        }
+        const req = {
+            useQueryCache: queryObj.useQueryCache,
+            labels: queryObj.labels,
+            defaultDataset: queryObj.defaultDataset,
+            createSession: queryObj.createSession,
+            maximumBytesBilled: queryObj.maximumBytesBilled,
+            timeoutMs: options.timeoutMs,
+            location: queryObj.location || options.location,
+            formatOptions: {
+                useInt64Timestamp: true,
+            },
+            maxResults: queryObj.maxResults || options.maxResults,
+            query: queryObj.query,
+            useLegacySql: false,
+            requestId: uuid.v4(),
+            jobCreationMode: 'JOB_CREATION_OPTIONAL',
+        };
+        if (!this._enableQueryPreview) {
+            delete req.jobCreationMode;
+        }
+        const { parameterMode, params } = this.buildQueryParams_(queryObj.params, queryObj.types);
+        if (params) {
+            req.queryParameters = params;
+        }
+        if (parameterMode) {
+            req.parameterMode = parameterMode;
+        }
+        return req;
+    }
+    runJobsQuery(req, callback) {
+        this.trace_('[runJobsQuery]', req, callback);
+        this.request({
+            method: 'POST',
+            uri: '/queries',
+            json: req,
+        }, async (err, res) => {
+            this.trace_('jobs.query res:', res, err);
+            if (err) {
+                callback(err, null, res);
+                return;
+            }
+            let job = null;
+            if (res.jobReference) {
+                const jobRef = res.jobReference;
+                job = this.job(jobRef.jobId, {
+                    location: jobRef.location,
+                });
+            }
+            else if (res.queryId) {
+                job = this.job(res.queryId); // stateless query
+            }
+            callback(null, job, res);
+        });
+    }
+    /**
      * This method will be called by `createQueryStream()`. It is required to
@@ -1298,2 +1466,3 @@ * properly set the `autoPaginate` option value.
 exports.BigQuery = BigQuery;
+BigQuery.setLogFunction = logger_1.setLogFunction;
 /*! Developer Documentation
@@ -1300,0 +1469,0 @@ *

build/src/job.d.ts

@@ -32,3 +32,8 @@ /*!
     parseJSON?: boolean;
-} & PagedRequest<bigquery.jobs.IGetQueryResultsParams>;
+} & PagedRequest<bigquery.jobs.IGetQueryResultsParams> & {
+    /**
+     * internal properties
+     */
+    _cachedRows?: any[];
+};
 /**
@@ -109,2 +114,3 @@ * @callback QueryResultsCallback
     constructor(bigQuery: BigQuery, id: string, options?: JobOptions);
+    private trace_;
     /**
@@ -111,0 +117,0 @@ * @callback CancelCallback

build/src/job.js

@@ -27,2 +27,3 @@ "use strict";
 const bigquery_1 = require("./bigquery");
+const logger_1 = require("./logger");
 /**
@@ -334,2 +335,6 @@ * @callback QueryResultsCallback
     }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    trace_(msg, ...otherArgs) {
+        (0, logger_1.logger)(`[job][${this.id}]`, msg, ...otherArgs);
+    }
     cancel(callback) {
@@ -353,2 +358,3 @@ let qs;
         }, options);
+        this.trace_('[getQueryResults]', this.id, options.pageToken, options.startIndex);
         const wrapIntegers = qs.wrapIntegers ? qs.wrapIntegers : false;
@@ -360,2 +366,13 @@ delete qs.wrapIntegers;
         const timeoutOverride = typeof qs.timeoutMs === 'number' ? qs.timeoutMs : false;
+        if (options._cachedRows) {
+            let nextQuery = null;
+            if (options.pageToken) {
+                nextQuery = Object.assign({}, options, {
+                    pageToken: options.pageToken,
+                });
+                delete nextQuery._cachedRows;
+            }
+            callback(null, options._cachedRows, nextQuery);
+            return;
+        }
         this.bigQuery.request({
@@ -389,2 +406,3 @@ uri: '/queries/' + this.id,
             else if (resp.pageToken) {
+                this.trace_('[getQueryResults] has more pages', resp.pageToken);
                 // More results exist.
@@ -391,0 +409,0 @@ nextQuery = Object.assign({}, options, {

package.json

 {
   "name": "@google-cloud/bigquery",
   "description": "Google BigQuery Client Library for Node.js",
-  "version": "7.5.2",
+  "version": "7.6.0",
   "license": "Apache-2.0",
@@ -6,0 +6,0 @@ "author": "Google LLC",

New alerts

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 2 instances in 1 package

Improved metrics

Total package byte prevSize

716937

increased by1.72%

Number of package files

25

increased by8.7%

Lines of code

14750

increased by1.99%

Worsened metrics

Number of low supply chain risk alerts

4

increased by100%

No dependency changes