ghcrawler - npm Package Compare versions

Comparing version 0.2.5 to 0.2.6

lib/githubProcessor.js

@@ -16,3 +16,3 @@ // Copyright (c) Microsoft Corporation. All rights reserved.
     this.store = store;
-    this.version = 11;
+    this.version = 12;
   }
@@ -188,3 +188,3 @@
     // this._addRelation(request, 'collaborators', 'user', document.collaborators_url.replace('{/collaborator}', ''));
-    this._addRelation(request, 'collaborators', 'user', document.collaborators_url.replace('{/collaborator}', '?affiliation=outside'));
+    this._addRelation(request, 'collaborators', 'user', document.collaborators_url.replace('{/collaborator}', '?affiliation=direct'));
     this._addRelation(request, 'contributors', 'user');
@@ -207,4 +207,14 @@ if (document.subscribers_count) {
     const context = request.context;
-    request.addSelfLink('sha');
-    request.linkSiblings(`${context.qualifier}:commits`);
+    let repoUrn = null;
+    if (context.qualifier.includes('pull_request')) {
+      // if this is a PR commit, put it in a central spot for the repo and setup the qualifier so comments go there too
+      repoUrn = `urn:repo:${context.qualifier.split(':')[2]}`;
+      context.qualifier = `${repoUrn}:pull_request_commit`;
+      request.linkResource('self', `${context.qualifier}:${document.sha}`);
+    } else {
+      request.addSelfLink('sha');
+      repoUrn = context.qualifier;
+      request.linkSiblings(`${context.qualifier}:commits`);
+    }
+    this._addRoot(request, 'repo', 'repo', document.url.replace(/\/commits\/.*/, ''), repoUrn);
 
@@ -221,3 +231,3 @@ // Most often there actually are no comments. Get the comments if we think there will be some and this resource is being processed (vs. traversed).
     }
-    this._addRoot(request, 'repo', 'repo', document.url.replace(/\/commits\/.*/, ''), `${context.qualifier}`);
+
     // TODO some commits have author and committer properties, others have email info in a "commit" property
@@ -276,7 +286,3 @@ // For the former, this code works.  For the latter, consider queuing an email lookup and storing a
     if (document._links.commits && document.commits) {
-      // PR commits are not necessarily in this repo.  Unfortunately, they do not include explicit repo info we can use
-      // to form a PR nor is the origin repo in our filter set. So, keep the commit as PR-specific.  The commit itself
-      // may or may not be fetched depending on filtering.  This may duplicate the commits in the output but fetching from
-      // GitHub will be deduplicated as the commit URLs are the same.
-      this._addCollection(request, 'pull_request_commits', 'commit', document._links.commits.href);
+      this._addRelation(request, 'commits', 'commit', document._links.commits.href);
     }
@@ -627,3 +633,3 @@
     const collections = new Set([
-      'collaborators', 'commit_comments', 'commits', 'contributors', 'events', 'issues', 'issue_comments', 'members', 'orgs', 'repos', 'reviews', 'review_comments', 'pull_request_commits', 'subscribers', 'stargazers', 'statuses', 'teams'
+      'collaborators', 'commit_comments', 'commits', 'contributors', 'events', 'issues', 'issue_comments', 'members', 'orgs', 'repos', 'reviews', 'review_comments', 'subscribers', 'stargazers', 'statuses', 'teams'
     ]);
@@ -774,3 +780,5 @@ return collections.has(request.type);
     request.linkCollection('unique', `${relation.qualifier}:pages:${relation.guid}`);
-    const urns = document.elements.map(element => `urn:${relation.type}:${element.id}`);
+    const urnPrefix = relation.type === 'commit' && qualifier.includes('pull_request') ? `repo:${qualifier.split(':')[2]}:pull_request_commit` : relation.type;
+    const id = relation.type === 'commit' ? 'sha' : 'id';
+    const urns = document.elements.map(element => `urn:${urnPrefix}:${element[id]}`);
     request.linkResource('resources', urns);
@@ -777,0 +785,0 @@ return document;

lib/traversalPolicy.js

@@ -131,2 +131,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved.
 
+  static reprocessAlways(map) {
+    return new TraversalPolicy('storageOnly', 'always', TraversalPolicy._resolveMapSpec(map));
+  }
+
   static clone(policy) {
@@ -133,0 +137,0 @@ return new TraversalPolicy(policy.fetch, policy.freshness, policy.map);

lib/visitorMap.js

@@ -193,3 +193,3 @@ // Copyright (c) Microsoft Corporation. All rights reserved.
   statuses: collection(status),
-  pull_request_commits: collection(commit),
+  commits: collection(commit),
   issue: issue,
@@ -196,0 +196,0 @@ issue_comments: collection(issue_comment)

package.json

 {
   "name": "ghcrawler",
-  "version": "0.2.5",
+  "version": "0.2.6",
   "description": "A robust GitHub API crawler that walks a queue of GitHub entities retrieving and storing their contents.",
@@ -5,0 +5,0 @@ "main": "./index.js",

test/gitHubProcessorTests.js

@@ -461,3 +461,3 @@ // Copyright (c) Microsoft Corporation. All rights reserved.
       review_comments: { href: 'urn:repo:12:pull_request:13:review_comments', type: 'collection' },
-      pull_request_commits: { href: 'urn:repo:12:pull_request:13:pull_request_commits', type: 'collection' },
+      commits: { href: 'urn:repo:12:pull_request:13:commits:pages:*', type: 'relation' },
       statuses: { href: 'urn:repo:12:commit:funkySHA:statuses', type: 'collection' },
@@ -478,3 +478,3 @@ issue: { href: 'urn:repo:12:issue:13', type: 'resource' },
       { type: 'statuses', url: 'http://statuses/funkySHA', qualifier: 'urn:repo:12:pull_request:13', path: '/statuses' },
-      { type: 'pull_request_commits', url: 'http://commits', qualifier: 'urn:repo:12:pull_request:13', path: '/pull_request_commits' }
+      { type: 'commits', url: 'http://commits', qualifier: 'urn:repo:12:pull_request:13', path: '/commits' }
     ];
@@ -481,0 +481,0 @@ expectQueued(queue, expected);

No alert changes

Improved metrics

Total package byte prevSize

228350

increased by0.12%

Lines of code

5151

increased by0.19%

No dependency changes