@wmfs/tymly-solr-plugin - npm Package Compare versions

Comparing version 1.9.0 to 1.9.1

CHANGELOG.md

@@ -0,1 +1,48 @@
+## [1.9.1](https://github.com/wmfs/tymly-solr-plugin/compare/v1.9.0...v1.9.1) (2019-01-26)
+
+
+### 🐛 Bug Fixes
+
+* Build a storage service search query that makes some kind of sense. ([a8716ff](https://github.com/wmfs/tymly-solr-plugin/commit/a8716ff))
+* Drop solr view before recreating it ([09fd130](https://github.com/wmfs/tymly-solr-plugin/commit/09fd130))
+* Exclude some fields from storage search query ([4d66a54](https://github.com/wmfs/tymly-solr-plugin/commit/4d66a54))
+* Prevent local searches (ie not using solr) from trying to load the entire search table ([03ba1e5](https://github.com/wmfs/tymly-solr-plugin/commit/03ba1e5))
+
+
+### 🛠 Builds
+
+* **deps-dev:** update [@semantic-release](https://github.com/semantic-release)/git requirement ([d9fab0c](https://github.com/wmfs/tymly-solr-plugin/commit/d9fab0c))
+* **deps-dev:** update [@wmfs](https://github.com/wmfs)/tymly-pg-plugin requirement ([d8222be](https://github.com/wmfs/tymly-solr-plugin/commit/d8222be))
+* **deps-dev:** update semantic-release requirement ([5609746](https://github.com/wmfs/tymly-solr-plugin/commit/5609746))
+
+
+### 📦 Code Refactoring
+
+* Change example format ([4edef8f](https://github.com/wmfs/tymly-solr-plugin/commit/4edef8f))
+* Factor out listUserRoles method ([3b833f9](https://github.com/wmfs/tymly-solr-plugin/commit/3b833f9))
+* Rework storage search so all filters are applied in the SQL rather than post-hoc. ([cc3b04c](https://github.com/wmfs/tymly-solr-plugin/commit/cc3b04c))
+
+
+### 🚨 Tests
+
+* Remove console.log output ([59ba05b](https://github.com/wmfs/tymly-solr-plugin/commit/59ba05b))
+* Storage service search now respects default limit, so pass in higher limit to get all results ([eed1ad5](https://github.com/wmfs/tymly-solr-plugin/commit/eed1ad5))
+* Switch tests to use await ([7d45b37](https://github.com/wmfs/tymly-solr-plugin/commit/7d45b37))
+
+
+### ⚙️ Continuous Integrations
+
+* **circle:** add circle ci config ([ee75064](https://github.com/wmfs/tymly-solr-plugin/commit/ee75064))
+* **travis:** update travis config ([9c4a1b4](https://github.com/wmfs/tymly-solr-plugin/commit/9c4a1b4))
+
+
+### ♻️ Chores
+
+* Add example files ([3a297f5](https://github.com/wmfs/tymly-solr-plugin/commit/3a297f5))
+
+
+### 💎 Styles
+
+* Lint fixes ([ffed1c4](https://github.com/wmfs/tymly-solr-plugin/commit/ffed1c4))
+
 # [1.9.0](https://github.com/wmfs/tymly-solr-plugin/compare/v1.8.2...v1.9.0) (2018-12-21)
@@ -2,0 +49,0 @@

lib/components/services/solr/index.js

@@ -11,3 +11,3 @@ 'use strict'
 class SolrService {
-  boot (options, callback) {
+  async boot (options, callback) {
     this.solrConnection = SolrService._solrConnection(options.config)
@@ -38,10 +38,13 @@ this.solrUrl = this.solrConnection ? `http://${this.solrConnection.host}:${this.solrConnection.port}${this.solrConnection.path}` : null
           SolrService.constructSearchDocsArray(this.searchDocs))
-        if (this.createViewSQL) {
-          storageClient.query(this.createViewSQL, [], (err) => {
-            debug('Database view created with SQL: ', this.createViewSQL)
-            callback(err)
-          })
-        } else {
+        if (!this.createViewSQL) {
           callback(boom.notFound('failed to construct create view SQL'))
         }
+
+        try {
+          await storageClient.query('DROP VIEW IF EXISTS tymly.solr_data', [])
+          await storageClient.query(this.createViewSQL, [])
+          callback()
+        } catch (err) {
+          callback(err)
+        }
       }
@@ -48,0 +51,0 @@ }

lib/components/state-resources/add-docs/doc/example.json

@@ -1,1 +0,29 @@
-{}
+{
+  "AddDocs": {
+    "Type": "Task",
+    "Resource": "module:addDocs",
+    "InputPath": "$.incidents.incidentsInProgress",
+    "ResourceConfig": {
+      "mapping": {
+        "id": "incident#||incidentNumber",
+        "docId": "incidentNumber",
+        "domain": "search",
+        "docType": "incident",
+        "title": "Incident ||incidentNumber||/||callTimeYear",
+        "description": "incidentClassificationLabel",
+        "category": "iip",
+        "point": "locationLatitude||,||locationLongitude",
+        "activeEvent": true,
+        "author": "incident",
+        "roles": "$authenticated::text[]",
+        "language": "ENG",
+        "sortString": "incidentNumber",
+        "launches": "{\"launches\":[{\"input\": {\"boardKeys\":{\"incidentYear\": ||callTimeYear||, \"incidentNumber\": ||incidentNumber||}}, \"stateMachineName\": \"wmfs_getIncidentSummary_1_0\"}]}",
+        "created": "$NOW",
+        "modified": "$NOW"
+      }
+    },
+    "ResultPath": "$.incidents",
+    "Next": "AwaitingHumanInput"
+  }
+}

lib/components/state-resources/delta-reindex/doc/example.json

@@ -1,1 +0,10 @@
-{}
+{
+  "DeltaReindex": {
+    "Type": "Task",
+    "Resource": "module:deltaReindex",
+    "ResourceConfig": {
+      "core": "tymly"
+    },
+    "End": true
+  }
+}

lib/components/state-resources/full-reindex/doc/example.json

@@ -1,1 +0,10 @@
-{}
+{
+  "FullReindex": {
+    "Type": "Task",
+    "Resource": "module:fullReindex",
+    "ResourceConfig": {
+      "core": "tymly"
+    },
+    "End": true
+  }
+}

lib/components/state-resources/remove-docs/doc/index.js

 'use strict'
 
 module.exports = {
-  description: 'Remove SOLR Docs'
+  description: 'Remove SOLR Docs',
+  example: require('./example.json')
 }

lib/components/state-resources/search/index.js

@@ -16,2 +16,5 @@ 'use strict'
 
+  get rbac () { return this.services.rbac }
+  get solr () { return this.services.solr }
+
   get solrClient () {
@@ -22,3 +25,3 @@ if (this.solrClient_) {
 
-    const solrConnection = this.services.solr.solrConnection
+    const solrConnection = this.solr.solrConnection
     this.solrClient_ = solr.createClient({
@@ -35,5 +38,2 @@ host: solrConnection.host,
   async run (event, context) {
-    const rbacService = this.services.rbac
-    const solrService = this.services.solr
-
     if (!context.userId) {
@@ -47,24 +47,12 @@ return context.sendTaskFailure({
     try {
-      const userRoles = await rbacService.listUserRoles(context.userId)
+      const userRoles = await this.listUserRoles(context)
 
-      if (!userRoles.includes('$authenticated')) userRoles.push('$authenticated')
+      this.buildSearchFields()
 
-      if (solrService.searchDocs) {
-        const searchDocs = this.services.solr.searchDocs
-        this.searchFields = new Set()
-        Object.keys(searchDocs).map(s => {
-          Object.keys(searchDocs[s].attributeMapping).map(a => {
-            this.searchFields.add(_.snakeCase(a))
-          })
-        })
-      } else {
-        this.searchFields = defaultSolrSchemaFields
-      }
-
       const filters = this.processFilters(event)
 
-      if (solrService.solrUrl) {
+      if (this.solr.solrUrl) {
         this.runSolrSearch(event, context, filters, userRoles)
       } else {
-        this.runStorageSearch(context, filters, userRoles)
+        this.runStorageSearch(event, context, filters, userRoles)
       }
@@ -76,2 +64,42 @@ } catch (err) {
 
+  async listUserRoles (context) {
+    const userRoles = await this.rbac.listUserRoles(context.userId)
+
+    if (!userRoles.includes('$authenticated')) userRoles.push('$authenticated')
+
+    return userRoles
+  } // listUserRoles
+
+  buildSearchFields () {
+    this.allSearchFields = this.findSearchFields()
+    const wantedFields = this.allSearchFields
+      .filter(f =>
+        f !== 'modified' &&
+        f !== 'created' &&
+        f !== 'eventTimestamp' &&
+        f !== 'point' &&
+        f !== 'activeEvent' &&
+        f !== 'category'
+      )
+    const snaked = wantedFields.map(f => _.snakeCase(f))
+    this.searchFields = snaked
+  } // buildSearchFields
+
+  findSearchFields () {
+    if (!this.solr.searchDocs) {
+      return defaultSolrSchemaFields
+    }
+
+    const searchFields = new Set()
+    const searchDocs = this.solr.searchDocs
+
+    Object.keys(searchDocs).map(s => {
+      Object.keys(searchDocs[s].attributeMapping).map(a => {
+        searchFields.add(a)
+      })
+    })
+
+    return [...searchFields]
+  } // buildSearchFields
+
   runSolrSearch (event, context, filters, userRoles) {
@@ -90,15 +118,3 @@ const searchTerm = event.query
 
-    const filterQuery = []
-    this.searchFields.forEach(s => {
-      if (
-        s !== 'modified' &&
-        s !== 'created' &&
-        s !== 'event_timestamp' &&
-        s !== 'point' &&
-        s !== 'active_event' &&
-        s !== 'category'
-      ) {
-        filterQuery.push(`${_.camelCase(s)}:${searchTerm}`)
-      }
-    })
+    const filterQuery = this.searchFields.map(s => `${_.camelCase(s)}:${searchTerm}`)
     const fq = searchTerm ? `&fq=(${filterQuery.join('%20OR%20')})` : ''
@@ -119,22 +135,86 @@ const categoryQuery = event.categoryRestriction && event.categoryRestriction.length > 0 ? `%20AND%20category:(${event.categoryRestriction.join('%20OR%20')})` : ''
 
-  runStorageSearch (context, filters, userRoles) {
-    const where = userRoles.map(role => `'${role}' = any(roles)`)
-    const query = `select * from tymly.solr_data` + (where.length > 0 ? ` where ${where.join(' or ')}` : ``)
+  async runStorageSearch (event, context, filters, userRoles) {
+    const searchClause = this.storageSearchQuery(filters.query)
+    const roleWhereClause = userRoles.map(role => `'${role}' = any(roles)`).join(' or ')
+    const domainClause = this.storageSearchDomain(filters.domain)
+    const activeEventClause = this.storageSearchActiveEvent(filters.showActiveEventsOnly)
+    const categoryClause = this.storageSearchCategory(filters.categoryRestriction)
+    const limitClause = `limit ${filters.limit} offset ${filters.offset}`
 
-    this.storageClient.query(query, (err, results) => {
-      if (err) {
-        return context.sendTaskFailure({ error: 'searchFail', cause: err })
-      }
-      const matchingDocs = this.filterDocs(results.rows, filters)
-      this.processResults(context, matchingDocs, filters, matchingDocs.length)
-    })
+    const filterClauses = [
+      searchClause,
+      roleWhereClause,
+      domainClause,
+      activeEventClause,
+      categoryClause
+    ].filter(f => f)
+      .map(f => `(${f})`)
+
+    const whereClause = filterClauses.length ? `where ${filterClauses.join(' and ')}` : ''
+
+    const query = `select * from tymly.solr_data ${whereClause} ${limitClause}`
+
+    try {
+      const results = await this.storageClient.query(query)
+
+      this.processResults(context, results.rows, filters, results.rows.length)
+    } catch (err) {
+      return context.sendTaskFailure({ error: 'searchFail', cause: err })
+    }
   } // runStorageSearch
 
-  async processResults (context, matchingDocs, filters, totalHits) {
+  storageSearchQuery (searchTerm = '') {
+    const terms = searchTerm
+      .trim()
+      .replace(emojiRegex, '') // remove emojis
+      .replace(/([-]|[_]|[.]|[!]|[~]|[*]|[']|[(]|[)])/g, '') // remove unescaped
+      .split(' ')
+      .filter(x => x)
+
+    if (terms.length === 0) {
+      return null
+    }
+
+    const queries = this.searchFields
+      .filter(f => ['id', 'roles', 'domain', 'launches'].indexOf(f) === -1)
+      .map(f =>
+        terms.map(t => `cast(${f} as text) ilike '%${t}%'`).join(' and ')
+      )
+
+    const whereClause = queries
+      .map(q => `(${q})`)
+      .join(' or ')
+    return whereClause
+  }
+
+  storageSearchDomain (domain) {
+    if (!domain || !this.allSearchFields.domain) {
+      return null
+    }
+    return `domain = ${domain}`
+  } // storageSearchDomain
+
+  storageSearchActiveEvent (activeEventOnly) {
+    if (!activeEventOnly || !this.allSearchFields.activeEvent) {
+      return null
+    }
+    return `active_event = true`
+  }
+
+  storageSearchCategory (categoryRestriction) {
+    if (categoryRestriction.length === 0 || !this.allSearchFields.category) {
+      return null
+    }
+
+    return categoryRestriction.map(cat => `'${cat}' = any(category)`).join(' or ')
+  }
+
+  async processResults (context, results, filters, totalHits) {
     const searchResults = {
       input: filters,
-      totalHits: totalHits
+      totalHits: totalHits,
+      results: this.jsonifyLaunches(results),
+      categoryCounts: this.countCategories(results)
     }
 
-    this.constructSearchResults(searchResults, filters, matchingDocs)
     try {
@@ -148,3 +228,3 @@ await this.updateSearchHistory(searchResults.results, context.userId)
 
-  constructSearchResults (searchResults, filters, results) {
+  constructSearchResults (searchResults, results) {
     searchResults.results = this.jsonifyLaunches(results)
@@ -195,54 +275,2 @@ searchResults.categoryCounts = this.countCategories(results)
 
-  filterDocs (docs, filters) {
-    const matchingDocs = []
-    docs.map(candidate => {
-      if (
-        this.domainMatch(filters.domain, candidate) &&
-        this.categoryMatch(filters.categoryRestriction, candidate) &&
-        this.activeEventMatch(filters.showActiveEventsOnly, candidate) &&
-        this.queryMatch(filters.query, candidate)) {
-        matchingDocs.push(candidate)
-      }
-    })
-    return matchingDocs
-  }
-
-  queryMatch (query, doc) {
-    let match = false
-    if (_.isUndefined(query)) {
-      match = true
-    } else {
-      this.searchFields.forEach(s => {
-        if (s !== 'created' && s !== 'modified') {
-          if (doc[s] && doc[s].toString().toUpperCase().includes(query.toUpperCase())) match = true
-        }
-      })
-    }
-    return match
-  }
-
-  categoryMatch (categoryRestriction, doc) {
-    if (categoryRestriction.length === 0) {
-      return true
-    } else {
-      return categoryRestriction.indexOf(doc.category) !== -1
-    }
-  }
-
-  domainMatch (domain, doc) {
-    if (!domain) {
-      return true
-    } else {
-      return domain === doc.domain
-    }
-  }
-
-  activeEventMatch (showActiveEventsOnly, doc) {
-    if (!showActiveEventsOnly) {
-      return true
-    } else {
-      return doc.activeEvent
-    }
-  }
-
   processFilters (event) {
@@ -277,3 +305,3 @@ const searchDefaults = {
 
-    if (_.isInteger(event.offset)) {
+    if (_.isInteger(event.limit)) {
       filters.limit = event.limit
@@ -280,0 +308,0 @@ } else {

package.json

 {
   "name": "@wmfs/tymly-solr-plugin",
-  "version": "1.9.0",
+  "version": "1.9.1",
   "description": "Plugin which handles interaction with Apache Solr for Tymly framework",
@@ -34,5 +34,5 @@ "homepage": "https://github.com/wmfs/tymly-solr-plugin#readme",
     "@semantic-release/changelog": "3.0.1",
-    "@semantic-release/git": "7.0.5",
+    "@semantic-release/git": "7.0.6",
     "@wmfs/tymly": "1.75.0",
-    "@wmfs/tymly-pg-plugin": "1.112.0",
+    "@wmfs/tymly-pg-plugin": "1.113.0",
     "@wmfs/tymly-rbac-plugin": "1.12.1",
@@ -45,3 +45,3 @@ "chai": "4.2.0",
     "nyc": "13.1.0",
-    "semantic-release": "15.12.2",
+    "semantic-release": "15.13.1",
     "standard": "12.0.1"
@@ -48,0 +48,0 @@ },

test/search-state-tests.js

@@ -91,3 +91,5 @@ /* eslint-env mocha */
     const executionDescription = await statebox.startExecution(
-      {}, // input
+      {
+        limit: 100
+      }, // input
       STATE_MACHINE_NAME, // state machine name
@@ -100,3 +102,2 @@ {
 
-    console.log(JSON.stringify(executionDescription, null, 2))
     expect(executionDescription.currentStateName).to.eql('Search')
@@ -124,3 +125,2 @@ expect(executionDescription.currentResource).to.eql('module:search')
 
-    console.log(JSON.stringify(executionDescription, null, 2))
     expect(executionDescription.currentStateName).to.eql('Search')
@@ -134,4 +134,4 @@ expect(executionDescription.currentResource).to.eql('module:search')
 
-  it('should search with a query input as a user without any roles', function (done) {
-    statebox.startExecution(
+  it('should search with a query input as a user without any roles', async () => {
+    const executionDescription = await statebox.startExecution(
       {}, // input
@@ -142,14 +142,9 @@ STATE_MACHINE_NAME, // state machine name
         userId: 'jim.smith'
-      }, // options
-      function (err, executionDescription) {
-        expect(err).to.eql(null)
-        expect(executionDescription.ctx.searchResults.totalHits).to.eql(0)
-        expect(executionDescription.ctx.searchResults.results.length).to.eql(0)
-        done()
-      }
-    )
+      })
+    expect(executionDescription.ctx.searchResults.totalHits).to.eql(0)
+    expect(executionDescription.ctx.searchResults.results.length).to.eql(0)
   })
 
-  it('should fail to search when user role is a minor', function (done) {
-    statebox.startExecution(
+  it('should fail to search when user role is a minor', async () => {
+    const executionDescription = await statebox.startExecution(
       {}, // input
@@ -160,10 +155,7 @@ STATE_MACHINE_NAME, // state machine name
         userId: 'jane.smith'
-      }, // options
-      function (err, executionDescription) {
-        expect(err).to.eql(null)
-        expect(executionDescription.ctx.searchResults.totalHits).to.eql(0)
-        expect(executionDescription.ctx.searchResults.results.length).to.eql(0)
-        done()
       }
     )
+
+    expect(executionDescription.ctx.searchResults.totalHits).to.eql(0)
+    expect(executionDescription.ctx.searchResults.results.length).to.eql(0)
   })
@@ -170,0 +162,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

115108

increased by5.2%

Number of package files

66

increased by3.13%

Lines of code

2004

increased by3.51%

No dependency changes