Sign In

cryptoserve

Package Overview
Dependencies
Maintainers
1
Versions
13
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

cryptoserve - npm Package Compare versions

Comparing version
0.4.0
 to
0.5.0
+15
lib/census/format.mjs
/**
* Formatting shared by the census report renderers.
*
* This is all that survives of the CLI's own aggregator. The census is
* aggregated once, where it is collected, and published as a dated snapshot;
* this package renders that snapshot and computes nothing from it.
*/
/** Format a large number with a suffix (B, M, K). */
export function formatNumber(n) {
if (n >= 1_000_000_000) return (n / 1_000_000_000).toFixed(1) + 'B';
if (n >= 1_000_000) return (n / 1_000_000).toFixed(1) + 'M';
if (n >= 1_000) return (n / 1_000).toFixed(1) + 'K';
return String(n);
}
+122
-163
lib/census/index.mjs
/**
* Census orchestrator: run collectors, aggregate, cache results.
* Census client: fetch the published census snapshot and cache it locally.
*
* Supports 11 ecosystems: npm, PyPI, Go, Maven, crates.io, Packagist, NuGet,
* RubyGems, Hex (Elixir), pub.dev (Dart), and CocoaPods (Swift/ObjC).
* This used to collect the census itself -- its own copy of thirteen collectors
* over its own copy of the catalog. That was a second definition of a published
* measurement, and the two disagreed: the CLI catalogued 357 packages while the
* census catalogued 355, and every collector in this package still records a
* failed request as `downloads: 0`, so anyone running `cryptoserve census` was
* shown collection failures as measurements of zero. `cryptography` alone really
* has over a billion downloads a month and was reported as none.
*
* The census now publishes from committed dated snapshots, so there is one
* number and one collection date. This module fetches that and renders it. It
* does not measure anything, which is why nothing here can disagree with what
* census.cryptoserve.dev shows.
*/

@@ -11,56 +21,25 @@

import {
NPM_PACKAGES, PYPI_PACKAGES, GO_PACKAGES,
MAVEN_PACKAGES, CRATES_PACKAGES, PACKAGIST_PACKAGES, NUGET_PACKAGES,
RUBYGEMS_PACKAGES, HEX_PACKAGES, PUB_PACKAGES, COCOAPODS_PACKAGES,
} from './package-catalog.mjs';
import { collectNpmDownloads } from './collectors/npm-downloads.mjs';
import { collectPypiDownloads } from './collectors/pypi-downloads.mjs';
import { collectGoDownloads } from './collectors/go-downloads.mjs';
import { collectMavenDownloads } from './collectors/maven-downloads.mjs';
import { collectCratesDownloads } from './collectors/crates-downloads.mjs';
import { collectPackagistDownloads } from './collectors/packagist-downloads.mjs';
import { collectNugetDownloads } from './collectors/nuget-downloads.mjs';
import { collectRubygemsDownloads } from './collectors/rubygems-downloads.mjs';
import { collectHexDownloads } from './collectors/hex-downloads.mjs';
import { collectPubDownloads } from './collectors/pub-downloads.mjs';
import { collectCocoapodsDownloads } from './collectors/cocoapods-downloads.mjs';
import { collectNvdCves } from './collectors/nvd-cves.mjs';
import { collectGithubAdvisories } from './collectors/github-advisories.mjs';
import { aggregate } from './aggregator.mjs';
/** Where the published snapshot lives. Overridable for tests and local checks. */
export const CENSUS_URL =
process.env.CRYPTOSERVE_CENSUS_URL || 'https://census.cryptoserve.dev/api/census';
const cacheFile = () => configPath('census-cache.json');
const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
// A census run fans out to thirteen third-party services. Without a per-request
// deadline one unresponsive registry blocks the whole command indefinitely,
// which is exactly what `cryptoserve census` used to do: no output, no timeout,
// no way to tell a slow run from a hung one.
export const DEFAULT_REQUEST_TIMEOUT_MS = 15_000;
/**
* fetch with a hard per-request deadline. Returned as a non-ok response shape
* on timeout so collectors take their existing "registry did not answer" path
* instead of aborting the run.
* The snapshot is a dated measurement that changes when someone publishes a new
* one, not a live feed, so the cache is a day rather than an hour. npm and
* pypistats both report a rolling 30-day window: re-fetching it four times a day
* advances that window by under a percent and returns the same number.
*/
export function fetchWithTimeout(timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, baseFetch = globalThis.fetch) {
return async (url, options = {}) => {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
return await baseFetch(url, { ...options, signal: controller.signal });
} catch (err) {
if (err?.name === 'AbortError') {
return { ok: false, status: 408, statusText: `timeout after ${timeoutMs}ms`, json: async () => ({}), text: async () => '' };
}
throw err;
} finally {
clearTimeout(timer);
}
};
}
export const CACHE_TTL_MS = 24 * 60 * 60 * 1000;
export const DEFAULT_REQUEST_TIMEOUT_MS = 15_000;
/**
* Load cached data if valid.
* @returns {Object|null}
* Read the cache. Returns `{ payload, fetchedAt, ageMs }` or null.
*
* Age is measured from when this machine fetched the snapshot, never from the
* snapshot's own `collectedAt` -- that is the date of the measurement, which is
* legitimately weeks old and would expire every cache entry the moment it was
* written.
*/

@@ -71,7 +50,7 @@ function loadCache() {

if (!existsSync(file)) return null;
const raw = readFileSync(file, 'utf-8');
const cached = JSON.parse(raw);
const age = Date.now() - new Date(cached.collectedAt).getTime();
if (age < CACHE_TTL_MS) return cached;
return null;
const cached = JSON.parse(readFileSync(file, 'utf-8'));
if (!cached?.payload || !cached?.fetchedAt) return null;
const ageMs = Date.now() - new Date(cached.fetchedAt).getTime();
if (!Number.isFinite(ageMs) || ageMs < 0) return null;
return { payload: cached.payload, fetchedAt: cached.fetchedAt, ageMs };
} catch {

@@ -82,134 +61,114 @@ return null;

/**
* Save data to cache.
*/
function saveCache(data) {
function saveCache(payload) {
try {
ensureConfigDir();
writeFileSync(cacheFile(), JSON.stringify(data, null, 2));
writeFileSync(cacheFile(), JSON.stringify({
fetchedAt: new Date().toISOString(),
source: CENSUS_URL,
payload,
}, null, 2));
} catch {
// Cache write failure is non-fatal
// A cache we could not write is not a reason to fail a read-only command.
}
}
/** A payload that is missing its headline is a fetch that went somewhere else. */
function looksLikeCensus(payload) {
return Boolean(
payload &&
typeof payload === 'object' &&
typeof payload.totalDownloads === 'number' &&
typeof payload.collectedAt === 'string' &&
payload.npm && typeof payload.npm === 'object'
);
}
export function describeAge(ms) {
const days = Math.floor(ms / 86_400_000);
if (days >= 1) return `${days} day${days === 1 ? '' : 's'} old`;
const hours = Math.floor(ms / 3_600_000);
if (hours >= 1) return `${hours} hour${hours === 1 ? '' : 's'} old`;
return 'under an hour old';
}
/**
* Run the full census: collect data from all sources and aggregate.
* Fetch the published census snapshot.
*
* @param {Object} [options]
* @param {boolean} [options.verbose] - Log progress to stderr
* @param {boolean} [options.noCache] - Skip cache
* @param {string[]} [options.sources] - Which sources to query (default: all)
* @param {Function} [options.fetchFn] - Injected fetch for testing
* @returns {Promise<Object>} Aggregated census data
* @param {Object} [options]
* @param {boolean} [options.noCache] Ignore any cached copy
* @param {boolean} [options.verbose] Narrate to stderr
* @param {Function} [options.fetchFn] Injected fetch, for tests
* @param {number} [options.timeoutMs]
* @returns {Promise<{data: Object, source: 'network'|'cache'|'stale-cache', fetchedAt: string}>}
*/
export async function runCensus(options = {}) {
export async function fetchCensus(options = {}) {
const {
noCache = false,
verbose = false,
noCache = false,
sources,
fetchFn,
fetchFn = globalThis.fetch,
timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS,
onProgress,
} = options;
// Check cache first
const note = (msg) => { if (verbose) process.stderr.write(`${msg}\n`); };
if (!noCache) {
const cached = loadCache();
if (cached) {
if (verbose) process.stderr.write('Using cached census data (< 1 hour old)\n');
return cached;
if (cached && cached.ageMs < CACHE_TTL_MS) {
note(`Using cached snapshot (${describeAge(cached.ageMs)})`);
return { data: cached.payload, source: 'cache', fetchedAt: cached.fetchedAt };
}
}
const enabledSources = sources || [
'npm', 'pypi', 'go', 'maven', 'crates', 'packagist', 'nuget',
'rubygems', 'hex', 'pub', 'cocoapods',
'nvd', 'github',
];
// An explicitly injected fetch (tests) is used as-is; otherwise every
// collector gets the deadline-bounded fetch rather than a bare global.
const collectorOpts = { verbose, fetchFn: fetchFn || fetchWithTimeout(timeoutMs) };
const empty = { packages: [], period: 'last_month', collectedAt: new Date().toISOString() };
note(`Fetching ${CENSUS_URL}`);
// Progress is reported unconditionally, not only under --verbose. A command
// that reaches out to thirteen services must never look like it has hung.
const report = onProgress || (msg => process.stderr.write(msg + '\n'));
report(`Collecting census data from ${enabledSources.length} sources (timeout ${Math.round(timeoutMs / 1000)}s per request)...`);
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
let payload = null;
let failure = null;
try {
const res = await fetchFn(CENSUS_URL, {
signal: controller.signal,
headers: { accept: 'application/json' },
});
if (!res.ok) {
failure = `${CENSUS_URL} returned HTTP ${res.status}`;
} else {
const body = await res.json();
if (!looksLikeCensus(body)) {
failure = `${CENSUS_URL} returned something that is not a census snapshot`;
} else {
payload = body;
}
}
} catch (err) {
const cause = err?.cause?.code || err?.cause?.message;
const reason = err?.name === 'AbortError'
? `no response within ${Math.round(timeoutMs / 1000)}s`
: `${err?.message || err}${cause ? ` (${cause})` : ''}`;
failure = `could not reach ${CENSUS_URL}: ${reason}`;
} finally {
clearTimeout(timer);
}
// Phase 1: Package downloads (all 11 ecosystems in parallel)
const downloadPromises = [
enabledSources.includes('npm')
? (report(' Fetching npm download counts...'), collectNpmDownloads(NPM_PACKAGES, collectorOpts))
: Promise.resolve(empty),
enabledSources.includes('pypi')
? (report(' Fetching PyPI download counts...'), collectPypiDownloads(PYPI_PACKAGES, collectorOpts))
: Promise.resolve(empty),
enabledSources.includes('go')
? (report(' Fetching Go module stats...'), collectGoDownloads(GO_PACKAGES, collectorOpts))
: Promise.resolve(empty),
enabledSources.includes('maven')
? (report(' Fetching Maven Central stats...'), collectMavenDownloads(MAVEN_PACKAGES, collectorOpts))
: Promise.resolve(empty),
enabledSources.includes('crates')
? (report(' Fetching crates.io download counts...'), collectCratesDownloads(CRATES_PACKAGES, collectorOpts))
: Promise.resolve(empty),
enabledSources.includes('packagist')
? (report(' Fetching Packagist download counts...'), collectPackagistDownloads(PACKAGIST_PACKAGES, collectorOpts))
: Promise.resolve(empty),
enabledSources.includes('nuget')
? (report(' Fetching NuGet download counts...'), collectNugetDownloads(NUGET_PACKAGES, collectorOpts))
: Promise.resolve(empty),
enabledSources.includes('rubygems')
? (report(' Fetching RubyGems download counts...'), collectRubygemsDownloads(RUBYGEMS_PACKAGES, collectorOpts))
: Promise.resolve(empty),
enabledSources.includes('hex')
? (report(' Fetching Hex.pm download counts...'), collectHexDownloads(HEX_PACKAGES, collectorOpts))
: Promise.resolve(empty),
enabledSources.includes('pub')
? (report(' Fetching pub.dev download counts...'), collectPubDownloads(PUB_PACKAGES, collectorOpts))
: Promise.resolve(empty),
enabledSources.includes('cocoapods')
? (report(' Fetching CocoaPods pod counts...'), collectCocoapodsDownloads(COCOAPODS_PACKAGES, collectorOpts))
: Promise.resolve(empty),
];
if (payload) {
saveCache(payload);
return { data: payload, source: 'network', fetchedAt: new Date().toISOString() };
}
const [npmData, pypiData, goData, mavenData, cratesData, packagistData, nugetData,
rubygemsData, hexData, pubData, cocoapodsData] =
await Promise.all(downloadPromises);
// Phase 2: Vulnerability data (NVD + GitHub in parallel)
const vulnPromises = [
enabledSources.includes('nvd')
? (report(' Fetching NVD CVE data...'), collectNvdCves(collectorOpts))
: Promise.resolve({ cves: [], collectedAt: new Date().toISOString() }),
enabledSources.includes('github')
? (report(' Fetching GitHub advisories...'), collectGithubAdvisories(collectorOpts))
: Promise.resolve({ advisories: [], collectedAt: new Date().toISOString() }),
];
const [nvdData, githubData] = await Promise.all(vulnPromises);
// Aggregate
const result = aggregate({
npm: npmData,
pypi: pypiData,
go: goData,
maven: mavenData,
crates: cratesData,
packagist: packagistData,
nuget: nugetData,
rubygems: rubygemsData,
hex: hexData,
pub: pubData,
cocoapods: cocoapodsData,
nvd: nvdData,
github: githubData,
});
// Cache the result
if (!noCache) {
saveCache(result);
// A stale local copy beats no answer, but it is announced as stale rather
// than passed off as current. Serving it silently is how a snapshot from
// months ago gets read as today's number.
const cached = loadCache();
if (cached) {
return {
data: cached.payload,
source: 'stale-cache',
fetchedAt: cached.fetchedAt,
failure,
};
}
return result;
const error = new Error(failure || 'census snapshot unavailable');
error.censusUnavailable = true;
throw error;
}
+1
-1
lib/census/report-html.mjs

@@ -6,3 +6,3 @@ /**

import { formatNumber } from './aggregator.mjs';
import { formatNumber } from './format.mjs';

@@ -9,0 +9,0 @@ /**

+37
-1
lib/census/report-terminal.mjs

@@ -5,5 +5,17 @@ /**

import { formatNumber } from './aggregator.mjs';
import { formatNumber } from './format.mjs';
/**
* A date-only rendering in UTC.
*
* Not `toLocaleDateString()`: a date-only value formatted in local time shows
* the previous day everywhere west of Greenwich, which the site hit on this
* same field.
*/
function formatDate(iso) {
const d = new Date(iso);
return Number.isNaN(d.getTime()) ? String(iso) : d.toISOString().slice(0, 10);
}
/**
* Render census results to the terminal.

@@ -26,3 +38,10 @@ *

// --- Headline ---
//
// The collection date leads, because this is a dated snapshot and not a live
// reading. Without it, a figure collected in March reads as today's, which is
// how "refresh in progress" sat on a public page for four months.
lines.push(section('Headline'));
if (data.collectedAt) {
lines.push(labelValue('Snapshot collected', formatDate(data.collectedAt), 26));
}
lines.push(labelValue('Weak crypto downloads', `${formatNumber(data.totalWeakDownloads)}/month (${data.weakPercentage.toFixed(1)}%)`, 26));

@@ -126,4 +145,21 @@ lines.push(labelValue('Modern crypto downloads', `${formatNumber(data.totalModernDownloads)}/month (${data.modernPercentage.toFixed(1)}%)`, 26));

lines.push(dim(' Advisories: GitHub Advisory Database (reviewed, crypto-CWE filtered)'));
// Not every registry publishes a monthly download count. NuGet and RubyGems
// divide a lifetime total by an assumed number of months; CocoaPods publishes
// nothing. Presenting the combined figure without saying so reports a proxy
// times a constant as a measurement.
if (typeof data.measuredDownloads === 'number' && typeof data.modelledDownloads === 'number') {
lines.push('');
lines.push(dim(` Measured: ${formatNumber(data.measuredDownloads)}/month from registries that publish a count` +
(typeof data.measuredShareOfTotal === 'number' ? ` (${data.measuredShareOfTotal.toFixed(1)}% of the total)` : '')));
lines.push(dim(` Modelled: ${formatNumber(data.modelledDownloads)}/month derived from lifetime totals or version counts`));
lines.push(dim(' The percentages above divide by the combined figure. Quote the measured one.'));
}
lines.push('');
lines.push(dim(' Download counts reflect package installs (CI/CD + transitive deps), not direct usage'));
lines.push(dim(' NIST 2030/2035 deadlines target public-key crypto only (AES, SHA-2, SHA-3 unaffected)'));
if (data.collectedAt) {
lines.push(dim(` Published snapshot collected ${formatDate(data.collectedAt)}; not a live reading`));
}
lines.push('');

@@ -130,0 +166,0 @@ 

+37
-6
lib/client.mjs

@@ -98,3 +98,15 @@ /**

export async function login(serverUrl = DEFAULT_SERVER) {
/**
* Browser login against an explicit server.
*
* No default. It used to fall back to https://localhost:8003, so a new user
* running `cryptoserve login` was sent to an address where nothing runs on
* their machine, and the flow could only ever time out. The CLI cannot know
* the operator's server, and a guess that is wrong for everyone is worse than
* an error that names what to pass.
*/
export async function login(serverUrl) {
if (!serverUrl) {
throw new Error('login requires a server URL. Pass --server <url>.');
}
const server = validateServerUrl(serverUrl);

@@ -104,2 +116,12 @@

return new Promise((resolve, reject) => {
let settled = false;
let timer = null;
const finish = (fn, arg) => {
if (settled) return;
settled = true;
if (timer) clearTimeout(timer);
try { httpServer.close(); } catch { /* already closed */ }
fn(arg);
};
const httpServer = createServer((req, res) => {

@@ -113,4 +135,3 @@ const url = new URL(req.url, `http://localhost:${CALLBACK_PORT}`);

res.end('<html><body><h2>Login successful</h2><p>You can close this tab.</p></body></html>');
httpServer.close();
resolve({ success: true, server });
finish(resolve, { success: true, server });
} else {

@@ -122,2 +143,13 @@ res.writeHead(400);

// Without this the http server emits an unhandled 'error' and the process
// dies with a raw Node stack trace. EADDRINUSE is the ordinary case: a
// previous login left the port held.
httpServer.on('error', (err) => {
const message = err.code === 'EADDRINUSE'
? `The login callback port ${CALLBACK_PORT} is already in use. `
+ 'Another cryptoserve login is probably still running; close it and try again.'
: `Could not start the login callback listener on port ${CALLBACK_PORT}: ${err.message}`;
finish(reject, new Error(message));
});
httpServer.listen(CALLBACK_PORT, () => {

@@ -137,7 +169,6 @@ const authUrl = `${server}/auth/cli?redirect=http://localhost:${CALLBACK_PORT}/callback`;

// Timeout after 120 seconds
setTimeout(() => {
httpServer.close();
reject(new Error('Login timed out after 120 seconds'));
timer = setTimeout(() => {
finish(reject, new Error('Login timed out after 120 seconds'));
}, 120000);
});
}
+14
-5
lib/init.mjs

@@ -205,3 +205,6 @@ /**

// Encrypted file with password
const pw = await promptPassword('Set vault password (for encrypted key storage): ');
const pw = await promptPassword('Set vault password (for encrypted key storage): ', {
hint: 'Run "cryptoserve init --insecure-storage" to store the key without a password, '
+ 'or run init in an interactive terminal.',
});
result.keyStorage = await storeMasterKey(keyBase64, {

@@ -228,6 +231,12 @@ useKeychain: false,

// 4. Create project config
const configPath = join(projectDir, '.cryptoserve.json');
if (!existsSync(configPath)) {
writeFileSync(configPath, JSON.stringify({
// 4. Create project config.
// NOT named `configPath`: that shadowed the `configPath` import above for the
// whole function body, so the `configPath('master.key')` call in the
// --insecure-storage branch read a `const` from its temporal dead zone and
// threw "Cannot access 'configPath' before initialization". That branch is
// the exact recovery the no-keychain error recommends, so `init` dead-ended
// for every user without a keychain.
const projectConfigPath = join(projectDir, '.cryptoserve.json');
if (!existsSync(projectConfigPath)) {
writeFileSync(projectConfigPath, JSON.stringify({
version: 1,

@@ -234,0 +243,0 @@ project: projectDir.split('/').pop(),

+57
-4
lib/keychain.mjs

@@ -158,3 +158,38 @@ /**

export function promptPassword(prompt = 'Password: ') {
/**
* Raised when a password is needed and there is no terminal to ask on.
*
* The bin turns this into a plain message and exit 2. It carries a code rather
* than being matched on its text so an intermediate catch can re-throw it
* without having to recognise the wording.
*/
export class NonInteractiveError extends Error {
constructor(message) {
super(message);
this.name = 'NonInteractiveError';
this.code = 'ERR_NO_TTY';
}
}
/**
* Ask for a password on the terminal.
*
* With stdin redirected — `</dev/null`, a pipe, any CI runner — the old
* implementation registered a `data` listener that would never fire, so the
* promise never settled. Node eventually printed its own internals at the user
* ("Detected unsettled top-level await at .../bin/cryptoserve.mjs:1287", with
* installed file paths) and exited 13. That is a diagnostic about our code, not
* an answer to the operator's question, which is "what should I have passed?".
*
* `hint` names the non-interactive form for the calling command, because the
* answer differs: most commands take `--password`, but `init` takes
* `--insecure-storage` and `vault set` takes the value as an argument.
*/
export function promptPassword(prompt = 'Password: ', { hint = 'Pass --password <value>.' } = {}) {
if (!process.stdin.isTTY) {
return Promise.reject(new NonInteractiveError(
`Cannot prompt for a password: stdin is not a terminal.\n${hint}`
));
}
return new Promise((resolve) => {

@@ -197,3 +232,21 @@ const rl = createInterface({ input: process.stdin, output: process.stderr });

/**
* Whether this process is allowed to touch the OS keychain at all.
*
* `CRYPTOSERVE_NO_KEYCHAIN=1` turns every keychain path off. Containers and CI
* runners have no keychain service, and probing for one there costs a
* subprocess and can hang; declaring its absence is better than discovering it.
*
* It also makes the `--insecure-storage` path reachable on demand. That branch
* only executes when no master key is found, so on a developer machine where
* `cryptoserve init` has ever run, a regression test for it silently exercises
* nothing -- which is how a ReferenceError on that exact branch shipped.
*/
function keychainDisabled() {
const v = process.env.CRYPTOSERVE_NO_KEYCHAIN;
return v !== undefined && v !== '' && v !== '0' && v.toLowerCase() !== 'false';
}
export async function isKeychainAvailable() {
if (keychainDisabled()) return false;
const os = platform();

@@ -229,3 +282,3 @@ const backend = backends[os];

if (useKeychain) {
if (useKeychain && !keychainDisabled()) {
const backend = backends[platform()];

@@ -257,3 +310,3 @@ if (backend) {

// Try OS keychain first
const backend = backends[platform()];
const backend = keychainDisabled() ? null : backends[platform()];
if (backend) {

@@ -281,3 +334,3 @@ try {

export async function deleteMasterKey() {
const backend = backends[platform()];
const backend = keychainDisabled() ? null : backends[platform()];
if (backend) {

@@ -284,0 +337,0 @@ try { await backend.delete(); } catch { /* OK */ }

+4
-1
lib/local-crypto.mjs

@@ -26,3 +26,3 @@ /**

const ALGORITHMS = {
export const ALGORITHMS = {
'AES-256-GCM': { cipher: 'aes-256-gcm', keySize: 32, nonceSize: AES_GCM_NONCE_SIZE },

@@ -197,2 +197,5 @@ 'AES-128-GCM': { cipher: 'aes-128-gcm', keySize: 16, nonceSize: AES_GCM_NONCE_SIZE },

/** The hash algorithms `hashPassword` implements. The CLI validates against this. */
export const HASH_ALGORITHMS = ['scrypt', 'pbkdf2'];
export function hashPassword(password, algorithm = 'scrypt') {

@@ -199,0 +202,0 @@ const salt = randomBytes(SALT_SIZE);

+13
-0
lib/sarif.mjs

@@ -88,2 +88,15 @@ /**

// Private keys were acted on by `gate` and absent from SARIF, so a CI job
// uploading the report saw no alert for the finding that failed its build.
for (const keyFile of scanResults.privateKeyFiles || []) {
findings.push({
kind: 'private-key',
message: 'Private key committed to the repository',
severity: 'critical',
file: keyFile,
cwe: 'CWE-798',
fix: 'Remove it from the tree and rotate the key',
});
}
for (const t of scanResults.tlsFindings || []) {

@@ -90,0 +103,0 @@ findings.push({

+91
-22
lib/scanner.mjs

@@ -107,2 +107,66 @@ /**

/**
* Record every hardcoded secret in one file's contents.
*
* Extracted from the source-file loop so config files can be scanned with the
* identical rules: one definition, so the two surfaces cannot drift.
*/
/**
* Variables whose VALUE is a credential even though the value itself carries no
* recognisable prefix.
*
* Prefix patterns (AKIA, ghp_, sk-) catch an identifier and miss its secret
* half: `AWS_SECRET_ACCESS_KEY` is 40 characters of base64 alphabet with
* nothing to key on. The variable it is assigned to is the signal, so the
* length and alphabet of the value are what qualify it.
*/
const SECRET_ASSIGNMENTS = [
{ id: 'aws-secret', name: 'AWS Secret Access Key', envVar: 'AWS_SECRET_ACCESS_KEY',
key: /AWS_SECRET_ACCESS_KEY/i, value: /^[A-Za-z0-9/+=]{40}$/ },
{ id: 'generic-secret-key', name: 'Secret Key', envVar: 'SECRET_KEY',
key: /^(?:[A-Z0-9_]*_)?SECRET_KEY$/i, value: /^[A-Fa-f0-9]{32,}$|^[A-Za-z0-9/+=]{32,}$/ },
];
/** Values that are obviously stand-ins rather than credentials. */
const PLACEHOLDER_VALUE = /^(?:$|<.*>$|your[-_ ]|xxx+$|changeme$|placeholder$|todo$|example$|dummy$|test$|\.\.\.$)/i;
/**
* Record every hardcoded secret in one file's contents.
*
* Extracted from the source-file loop so config files can be scanned with the
* identical rules: one definition, so the two surfaces cannot drift.
*/
function collectSecrets(content, relPath, results) {
const lines = content.split('\n');
const seen = new Set();
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line.length > 4096) continue; // ReDoS protection
// An indirection is not a secret. `.env` files legitimately hold
// `KEY=${OTHER}` and source holds `process.env.KEY`.
if (/\$\{[A-Z_]+\}/.test(line) || /\$[A-Z_]{2,}/.test(line) || /process\.env\.[A-Z_]+/.test(line)) continue;
const record = (id, name, envVar) => {
const key = `${id}:${i + 1}`;
if (seen.has(key)) return;
seen.add(key);
results.secrets.push({ type: id, name, file: relPath, line: i + 1, envVar, severity: 'critical' });
};
for (const { id, regex, name, envVar } of SECRET_PATTERNS) {
regex.lastIndex = 0;
if (regex.test(line)) record(id, name, envVar);
}
// Assignment-shaped credentials, judged by the value rather than a prefix.
const assignment = /^\s*(?:export\s+)?([A-Za-z0-9_.]+)\s*[:=]\s*["']?([^"'\s#]*)["']?\s*(?:#.*)?$/.exec(line);
if (!assignment) continue;
const [, varName, rawValue] = assignment;
if (PLACEHOLDER_VALUE.test(rawValue)) continue;
for (const { id, name, envVar, key, value } of SECRET_ASSIGNMENTS) {
if (key.test(varName) && value.test(rawValue)) record(id, name, envVar);
}
}
}
export function scanProject(projectDir, options = {}) {

@@ -115,2 +179,4 @@ const results = {

filesScanned: 0, // source files matched to a language and analyzed
configFilesScanned: 0, // config/dotenv files read for secrets
privateKeyFiles: [], // cert files whose contents are a PRIVATE key
filesWalked: 0, // every file the walker examined, analyzed or not

@@ -197,5 +263,15 @@ // New in v0.2.0

// Cert files from walker
// Cert files from walker. A public certificate and the private key that
// signs it were reported in one undifferentiated list, so a committed
// `server.key` looked exactly like a published `server.crt`. Publishing a
// certificate is routine; publishing its key is the incident.
for (const certPath of walked.certFiles) {
results.certFiles.push(relative(projectDir, certPath));
const rel = relative(projectDir, certPath);
results.certFiles.push(rel);
let head = '';
try { head = readFileSync(certPath, 'utf-8').slice(0, 4096); }
catch { continue; }
if (/-----BEGIN (?:[A-Z0-9 ]+ )?PRIVATE KEY-----/.test(head)) {
results.privateKeyFiles.push(rel);
}
}

@@ -327,23 +403,16 @@

// Hardcoded secrets
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line.length > 4096) continue; // ReDoS protection
// Skip env var references
if (/\$\{[A-Z_]+\}/.test(line) || /process\.env\.[A-Z_]+/.test(line)) continue;
collectSecrets(content, relPath, results);
}
for (const { id, regex, name, envVar } of SECRET_PATTERNS) {
regex.lastIndex = 0;
if (regex.test(line)) {
results.secrets.push({
type: id,
name,
file: relPath,
line: i + 1,
envVar,
severity: 'critical',
});
}
}
}
// Config files are scanned for secrets too. They were walked and then used
// only for TLS settings, so a committed .env holding a live key reported
// "Secrets found: 0" while the SAME literal in a .js file was found. That is
// a false negative on the highest-value target the scanner has, on a
// capability both help surfaces advertise.
for (const filePath of walked.configFiles) {
let content;
try { content = readFileSync(filePath, 'utf-8'); }
catch { continue; }
results.configFilesScanned++;
collectSecrets(content, relative(projectDir, filePath), results);
}

@@ -350,0 +419,0 @@ 

+16
-2
lib/vault.mjs

@@ -140,4 +140,18 @@ /**

export function resetVault(path = defaultVaultPath()) {
if (existsSync(path)) unlinkSync(path);
/**
* Delete the vault, after proving the caller can open it.
*
* This used to unlink unconditionally, so `vault reset --password wrong` and
* `vault reset` with no password both printed "Vault deleted." and exited 0.
* Deleting ONE secret was authenticated and deleting ALL of them was not, which
* is the wrong way round: an attacker with filesystem access could already
* remove the file, but a CLI that answers a wrong password by destroying the
* data is a footgun pointed at its owner.
*/
export function resetVault(password, path = defaultVaultPath()) {
if (!existsSync(path)) return false;
// Throws on a wrong password, exactly as every other vault read does.
loadVault(password, path);
unlinkSync(path);
return true;
}

@@ -144,0 +158,0 @@ 

+64
-2
lib/walker.mjs

@@ -10,3 +10,3 @@ /**

import { readdirSync, statSync } from 'node:fs';
import { readdirSync, statSync, openSync, readSync, closeSync } from 'node:fs';
import { join, extname } from 'node:path';

@@ -46,2 +46,50 @@

/**
* Whether a filename is a dotenv file.
*
* `.env` alone was the only name recognised, so `.env.local` and
* `.env.production` -- the ones a developer is most likely to have on disk with
* live credentials in them -- were not collected at all.
*
* Templates (`.env.example` and friends) are INCLUDED. Excluding them by name
* was the wrong instinct: a template is the file that actually gets committed,
* while `.env` is usually gitignored, so a real key pasted into `.env.example`
* is the higher-risk case rather than the lower-risk one. Placeholders are
* filtered by their VALUE instead, where the evidence is.
*/
/**
* Whether a file begins with a PEM private-key header.
*
* Reads only the first line. A public `-----BEGIN CERTIFICATE-----` is
* deliberately not a match: publishing a certificate is routine, publishing the
* key that signs it is the incident.
*/
/** A PEM private key is small; anything large is not worth opening to check. */
function isSmallEnoughToSniff(filePath) {
try {
const { size } = statSync(filePath);
return size > 0 && size <= 64 * 1024;
} catch {
return false;
}
}
function looksLikePemPrivateKey(filePath) {
let fd;
try {
fd = openSync(filePath, 'r');
const buf = Buffer.alloc(64);
const read = readSync(fd, buf, 0, 64, 0);
return /-----BEGIN (?:[A-Z0-9 ]+ )?PRIVATE KEY-----/.test(buf.subarray(0, read).toString('latin1'));
} catch {
return false;
} finally {
if (fd !== undefined) { try { closeSync(fd); } catch { /* already closed */ } }
}
}
export function isDotenvFile(name) {
return name === '.env' || (name.startsWith('.env.') && name.length > '.env.'.length);
}
const BINARY_EXTENSIONS = new Set([

@@ -118,2 +166,16 @@ '.exe', '.dll', '.so', '.dylib', '.wasm',

// Classify cert files (no size check needed — just record path)
// A private key's evidence is its first line, not its filename. Gating
// on the extension found `server.key` and missed a byte-identical
// `id_rsa`, which is the most common name a committed key actually has.
// Only small unclassified files are sniffed, and only their first line:
// PEM keys are tiny, so this costs one short read per candidate.
if (!CERT_EXTENSIONS.has(ext) && !BINARY_EXTENSIONS.has(ext)
&& !extraSourceExts.has(ext) && !CONFIG_EXTENSIONS.has(ext)
&& !CONFIG_NAMES.has(name) && !isDotenvFile(name)
&& isSmallEnoughToSniff(filePath) && looksLikePemPrivateKey(filePath)) {
certFiles.push(filePath);
totalFiles++;
continue;
}
if (CERT_EXTENSIONS.has(ext)) {

@@ -157,3 +219,3 @@ certFiles.push(filePath);

// Classify config files
if (CONFIG_EXTENSIONS.has(ext) || CONFIG_NAMES.has(name)) {
if (CONFIG_EXTENSIONS.has(ext) || CONFIG_NAMES.has(name) || isDotenvFile(name)) {
configFiles.push(filePath);

@@ -160,0 +222,0 @@ continue;

+3
-2
package.json
{
"name": "cryptoserve",
"version": "0.4.0",
"version": "0.5.0",
"description": "CryptoServe CLI - Cryptographic scanning, PQC analysis, encryption, and local key management",

@@ -41,3 +41,4 @@ "type": "module",

"release-smoke": "node scripts/release-smoke.mjs"
}
},
"homepage": "https://cryptoserve.dev"
}
-276
lib/census/aggregator.mjs
/**
* Aggregate raw census data into headline metrics.
*
* Supports all 11 ecosystems: npm, PyPI, Go, Maven, crates.io, Packagist, NuGet,
* RubyGems, Hex (Elixir), pub.dev (Dart), and CocoaPods (Swift/ObjC).
* Includes project-level transparency stats when available.
*/
import { TIERS, CATEGORIES, getCatalogSize } from './package-catalog.mjs';
// NIST Post-Quantum Cryptography deadlines
const NIST_2030 = new Date('2030-01-01T00:00:00Z');
const NIST_2035 = new Date('2035-01-01T00:00:00Z');
const ECOSYSTEM_IDS = ['npm', 'pypi', 'go', 'maven', 'crates', 'packagist', 'nuget', 'rubygems', 'hex', 'pub', 'cocoapods'];
/**
* Sum downloads for a given tier from a packages array.
*/
function sumByTier(packages, tier) {
return packages
.filter(p => p.tier === tier)
.reduce((sum, p) => sum + p.downloads, 0);
}
/**
* Get top N packages sorted by downloads descending.
*/
function topPackages(packages, n = 10) {
return [...packages]
.sort((a, b) => b.downloads - a.downloads)
.slice(0, n);
}
/**
* Calculate days remaining until a target date.
*/
function daysUntil(target) {
const now = new Date();
const diff = target.getTime() - now.getTime();
return Math.max(0, Math.ceil(diff / (1000 * 60 * 60 * 24)));
}
/**
* Format days as "X yrs, Y days".
*/
function formatDaysRemaining(totalDays) {
const years = Math.floor(totalDays / 365);
const days = totalDays % 365;
if (years === 0) return `${days} days`;
return `${years} yr${years !== 1 ? 's' : ''}, ${days} days`;
}
/**
* Format a large number with suffix (M, K).
*/
export function formatNumber(n) {
if (n >= 1_000_000_000) return (n / 1_000_000_000).toFixed(1) + 'B';
if (n >= 1_000_000) return (n / 1_000_000).toFixed(1) + 'M';
if (n >= 1_000) return (n / 1_000).toFixed(1) + 'K';
return String(n);
}
/**
* Build a per-ecosystem breakdown from a packages array.
*/
function buildEcosystemBreakdown(pkgs, period) {
const weak = sumByTier(pkgs, TIERS.WEAK);
const modern = sumByTier(pkgs, TIERS.MODERN);
const pqc = sumByTier(pkgs, TIERS.PQC);
return {
weak,
modern,
pqc,
total: weak + modern + pqc,
topPackages: topPackages(pkgs, 15),
period,
};
}
/**
* Build per-category breakdown across all packages.
* Groups packages by category and computes weak/modern/pqc totals + weak percentage.
*/
function buildCategoryBreakdown(allPkgs) {
const categoryMap = {};
for (const cat of CATEGORIES) {
categoryMap[cat] = { category: cat, weak: 0, modern: 0, pqc: 0, total: 0, weakPercentage: 0, topPackages: [] };
}
for (const pkg of allPkgs) {
const cat = pkg.category || 'general';
const entry = categoryMap[cat];
if (!entry) continue;
const dl = pkg.downloads || 0;
if (pkg.tier === TIERS.WEAK) entry.weak += dl;
else if (pkg.tier === TIERS.PQC) entry.pqc += dl;
else entry.modern += dl;
entry.total += dl;
entry.topPackages.push(pkg);
}
// Compute weak percentage and sort top packages
const result = [];
for (const cat of CATEGORIES) {
const entry = categoryMap[cat];
entry.weakPercentage = entry.total > 0
? Math.round((entry.weak / entry.total) * 1000) / 10
: 0;
entry.topPackages = entry.topPackages
.sort((a, b) => b.downloads - a.downloads)
.slice(0, 10);
if (entry.total > 0) {
result.push(entry);
}
}
// Sort by total downloads descending
result.sort((a, b) => b.total - a.total);
return result;
}
/**
* Aggregate all census data into headline metrics.
*
* @param {Object} data
* @param {Object} data.npm - Result from collectNpmDownloads
* @param {Object} data.pypi - Result from collectPypiDownloads
* @param {Object} [data.go] - Result from collectGoDownloads
* @param {Object} [data.maven] - Result from collectMavenDownloads
* @param {Object} [data.crates] - Result from collectCratesDownloads
* @param {Object} [data.packagist] - Result from collectPackagistDownloads
* @param {Object} [data.nuget] - Result from collectNugetDownloads
* @param {Object} [data.rubygems] - Result from collectRubygemsDownloads
* @param {Object} [data.hex] - Result from collectHexDownloads
* @param {Object} [data.pub] - Result from collectPubDownloads
* @param {Object} [data.cocoapods] - Result from collectCocoapodsDownloads
* @param {Object} [data.nvd] - Result from collectNvdCves
* @param {Object} [data.github] - Result from collectGithubAdvisories
* @param {Object} [data.projectDeps] - Result from collectProjectDeps
* @returns {Object} Aggregated metrics matching CensusData type
*/
export function aggregate(data) {
// Gather all package arrays
const npmPkgs = data.npm?.packages || [];
const pypiPkgs = data.pypi?.packages || [];
const goPkgs = data.go?.packages || [];
const mavenPkgs = data.maven?.packages || [];
const cratesPkgs = data.crates?.packages || [];
const packagistPkgs = data.packagist?.packages || [];
const nugetPkgs = data.nuget?.packages || [];
const rubygemsPkgs = data.rubygems?.packages || [];
const hexPkgs = data.hex?.packages || [];
const pubPkgs = data.pub?.packages || [];
const cocoapodsPkgs = data.cocoapods?.packages || [];
const allPkgs = [...npmPkgs, ...pypiPkgs, ...goPkgs, ...mavenPkgs, ...cratesPkgs, ...packagistPkgs, ...nugetPkgs, ...rubygemsPkgs, ...hexPkgs, ...pubPkgs, ...cocoapodsPkgs];
// Download totals by tier
const totalWeakDownloads = sumByTier(allPkgs, TIERS.WEAK);
const totalModernDownloads = sumByTier(allPkgs, TIERS.MODERN);
const totalPqcDownloads = sumByTier(allPkgs, TIERS.PQC);
const totalDownloads = totalWeakDownloads + totalModernDownloads + totalPqcDownloads;
// Percentages
const weakPercentage = totalDownloads > 0
? Math.round((totalWeakDownloads / totalDownloads) * 1000) / 10
: 0;
const modernPercentage = totalDownloads > 0
? Math.round((totalModernDownloads / totalDownloads) * 1000) / 10
: 0;
const pqcPercentage = totalDownloads > 0
? Math.round((totalPqcDownloads / totalDownloads) * 1000) / 10
: 0;
// The headline ratio
const weakToPqcRatio = totalPqcDownloads > 0
? Math.round(totalWeakDownloads / totalPqcDownloads)
: null;
// Per-ecosystem breakdowns
const npm = buildEcosystemBreakdown(npmPkgs, data.npm?.period);
const pypi = buildEcosystemBreakdown(pypiPkgs, data.pypi?.period);
const go = buildEcosystemBreakdown(goPkgs, data.go?.period);
const maven = buildEcosystemBreakdown(mavenPkgs, data.maven?.period);
const crates = buildEcosystemBreakdown(cratesPkgs, data.crates?.period);
const packagist = buildEcosystemBreakdown(packagistPkgs, data.packagist?.period);
const nuget = buildEcosystemBreakdown(nugetPkgs, data.nuget?.period);
const rubygems = buildEcosystemBreakdown(rubygemsPkgs, data.rubygems?.period);
const hex = buildEcosystemBreakdown(hexPkgs, data.hex?.period);
const pub = buildEcosystemBreakdown(pubPkgs, data.pub?.period);
const cocoapods = buildEcosystemBreakdown(cocoapodsPkgs, data.cocoapods?.period);
// CVE totals
const nvdCves = data.nvd?.cves || [];
const totalCryptoCves = nvdCves.reduce((sum, c) => sum + c.totalCount, 0);
// GitHub advisories
const ghAdvisories = data.github?.advisories || [];
const totalAdvisories = ghAdvisories.reduce((sum, a) => sum + a.count, 0);
// Merge severity counts across CWEs
const advisorySeverity = { critical: 0, high: 0, medium: 0, low: 0, unknown: 0 };
for (const adv of ghAdvisories) {
for (const [sev, count] of Object.entries(adv.bySeverity || {})) {
advisorySeverity[sev] = (advisorySeverity[sev] || 0) + count;
}
}
// NIST deadlines
const nistDeadline2030Days = daysUntil(NIST_2030);
const nistDeadline2035Days = daysUntil(NIST_2035);
// Project-level stats (if collected)
const projectDeps = data.projectDeps;
const projectStats = projectDeps?.stats || null;
// Count active ecosystems
const activeEcosystems = ECOSYSTEM_IDS.filter(eco => {
const d = data[eco];
return d?.packages?.length > 0;
});
// Category breakdown
const categoryBreakdown = buildCategoryBreakdown(allPkgs);
return {
// Headline numbers
totalDownloads,
totalWeakDownloads,
totalModernDownloads,
totalPqcDownloads,
weakPercentage,
modernPercentage,
pqcPercentage,
weakToPqcRatio,
// Category breakdown
categoryBreakdown,
// Per-ecosystem
npm,
pypi,
go,
maven,
crates,
packagist,
nuget,
rubygems,
hex,
pub,
cocoapods,
// Project-level transparency
...(projectStats ? { projectStats } : {}),
// Vulnerabilities
totalCryptoCves,
cveBreakdown: nvdCves,
totalAdvisories,
advisorySeverity,
advisoryBreakdown: ghAdvisories,
// Deadlines
nistDeadline2030: formatDaysRemaining(nistDeadline2030Days),
nistDeadline2030Days,
nistDeadline2035: formatDaysRemaining(nistDeadline2035Days),
nistDeadline2035Days,
// Metadata
collectedAt: data.npm?.collectedAt || data.pypi?.collectedAt || new Date().toISOString(),
catalogSize: getCatalogSize(),
ecosystemCount: activeEcosystems.length || ECOSYSTEM_IDS.length,
};
}
-72
lib/census/collectors/cocoapods-downloads.mjs
/**
* Collect download counts from the CocoaPods trunk API.
*
* Endpoint: GET https://trunk.cocoapods.org/api/v1/pods/{name}
* - CocoaPods has no public download stats API
* - Use Libraries.io API as fallback for estimated downloads
* - Estimate based on GitHub stars/dependents if available
*/
const TRUNK_API = 'https://trunk.cocoapods.org/api/v1/pods';
const LIBRARIES_API = 'https://libraries.io/api/cocoapods';
const REQUEST_DELAY_MS = 500;
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Fetch CocoaPods pod metadata. Since CocoaPods has no download stats,
* we fetch from trunk API for verification and use conservative estimates
* based on pod popularity metrics (stars, dependents, rank).
*
* @param {import('../package-catalog.mjs').CatalogEntry[]} packages
* @param {Object} [options]
* @param {Function} [options.fetchFn]
* @param {boolean} [options.verbose]
* @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
*/
export async function collectCocoapodsDownloads(packages, options = {}) {
const fetchFn = options.fetchFn || globalThis.fetch;
const verbose = options.verbose || false;
const results = [];
for (let i = 0; i < packages.length; i++) {
const pkg = packages[i];
if (verbose) {
process.stderr.write(` cocoapods ${i + 1}/${packages.length}: ${pkg.name}\n`);
}
try {
const res = await fetchFn(`${TRUNK_API}/${pkg.name}`, {
headers: { 'Accept': 'application/json' },
});
if (!res.ok) {
if (verbose) process.stderr.write(` cocoapods ${pkg.name}: ${res.status}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
} else {
// Trunk API confirms pod exists but has no download stats.
// Use conservative estimate: CocoaPods ecosystem is smaller,
// most crypto pods get 1K-50K installs/month based on GitHub activity.
// We set 0 and rely on scanner data if available.
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
} catch (err) {
if (verbose) process.stderr.write(` cocoapods ${pkg.name} error: ${err.message}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
if (i < packages.length - 1) {
await sleep(REQUEST_DELAY_MS);
}
}
return {
packages: results.sort((a, b) => b.downloads - a.downloads),
period: 'estimated_monthly',
collectedAt: new Date().toISOString(),
};
}
-71
lib/census/collectors/crates-downloads.mjs
/**
* Collect download counts from the crates.io API.
*
* Endpoint: GET https://crates.io/api/v1/crates/{name}
* - Returns total downloads and recent_downloads (last 90 days)
* - Requires User-Agent header
* - No authentication required
* - Rate limit: 1 request per second recommended
*/
const CRATES_API = 'https://crates.io/api/v1/crates';
const REQUEST_DELAY_MS = 300;
const USER_AGENT = 'crypto-census/1.0 (https://census.cryptoserve.dev)';
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Fetch crates.io download counts for a list of packages.
*
* @param {import('../package-catalog.mjs').CatalogEntry[]} packages
* @param {Object} [options]
* @param {Function} [options.fetchFn] - Fetch implementation (defaults to globalThis.fetch)
* @param {boolean} [options.verbose] - Log progress
* @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
*/
export async function collectCratesDownloads(packages, options = {}) {
const fetchFn = options.fetchFn || globalThis.fetch;
const verbose = options.verbose || false;
const results = [];
for (let i = 0; i < packages.length; i++) {
const pkg = packages[i];
if (verbose) {
process.stderr.write(` crates ${i + 1}/${packages.length}: ${pkg.name}\n`);
}
try {
const res = await fetchFn(`${CRATES_API}/${pkg.name}`, {
headers: { 'User-Agent': USER_AGENT },
});
if (!res.ok) {
if (verbose) process.stderr.write(` crates ${pkg.name}: ${res.status}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
} else {
const data = await res.json();
// recent_downloads = last 90 days, divide by 3 for monthly estimate
const recentDownloads = data?.crate?.recent_downloads || 0;
const monthlyEstimate = Math.round(recentDownloads / 3);
results.push({ name: pkg.name, downloads: monthlyEstimate, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
} catch (err) {
if (verbose) process.stderr.write(` crates ${pkg.name} error: ${err.message}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
if (i < packages.length - 1) {
await sleep(REQUEST_DELAY_MS);
}
}
return {
packages: results.sort((a, b) => b.downloads - a.downloads),
period: 'last_month_estimated',
collectedAt: new Date().toISOString(),
};
}
-132
lib/census/collectors/github-advisories.mjs
/**
* Collect crypto-related security advisories from the GitHub Advisory Database.
*
* Endpoint: GET https://api.github.com/advisories?per_page=100
* - The REST API does NOT support CWE filtering -- we fetch and filter client-side
* - Free, no authentication required (60 req/hr unauthenticated)
* - We fetch up to MAX_PAGES pages and filter for crypto-related CWEs
*/
const GITHUB_API = 'https://api.github.com/advisories';
const REQUEST_DELAY_MS = 2000;
const MAX_PAGES = 5; // 500 advisories max to stay under rate limits
const CRYPTO_CWE_IDS = new Set(['CWE-327', 'CWE-326', 'CWE-328']);
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Check if an advisory is crypto-related based on its CWEs.
*/
function isCryptoRelated(advisory) {
const cwes = advisory.cwes || [];
return cwes.some(c => CRYPTO_CWE_IDS.has(c.cwe_id));
}
/**
* Get the crypto CWE IDs from an advisory.
*/
function getCryptoCweIds(advisory) {
return (advisory.cwes || [])
.filter(c => CRYPTO_CWE_IDS.has(c.cwe_id))
.map(c => c.cwe_id);
}
/**
* Fetch crypto-related advisory counts from GitHub Advisory Database.
*
* @param {Object} [options]
* @param {Function} [options.fetchFn] - Fetch implementation (defaults to globalThis.fetch)
* @param {boolean} [options.verbose] - Log progress
* @returns {Promise<{advisories: Array<{cweId: string, count: number, bySeverity: Object, byEcosystem: Object}>, collectedAt: string}>}
*/
export async function collectGithubAdvisories(options = {}) {
const fetchFn = options.fetchFn || globalThis.fetch;
const verbose = options.verbose || false;
// Accumulate crypto advisories across all pages
const byCwe = {};
for (const cweId of CRYPTO_CWE_IDS) {
byCwe[cweId] = {
count: 0,
bySeverity: { critical: 0, high: 0, medium: 0, low: 0, unknown: 0 },
byEcosystem: {},
};
}
let url = `${GITHUB_API}?per_page=100&type=reviewed`;
let page = 0;
let totalScanned = 0;
while (url && page < MAX_PAGES) {
page++;
if (verbose) process.stderr.write(` github page ${page}/${MAX_PAGES}\n`);
try {
const res = await fetchFn(url, {
headers: { 'Accept': 'application/vnd.github+json' },
});
if (!res.ok) {
if (verbose) process.stderr.write(` github page ${page}: HTTP ${res.status}\n`);
break;
}
const data = await res.json();
if (!Array.isArray(data) || data.length === 0) break;
totalScanned += data.length;
for (const adv of data) {
if (!isCryptoRelated(adv)) continue;
const cweIds = getCryptoCweIds(adv);
const sev = (adv.severity || 'unknown').toLowerCase();
for (const cweId of cweIds) {
const entry = byCwe[cweId];
entry.count++;
if (sev in entry.bySeverity) {
entry.bySeverity[sev]++;
} else {
entry.bySeverity.unknown++;
}
const vulnerabilities = adv.vulnerabilities || [];
for (const vuln of vulnerabilities) {
const eco = vuln?.package?.ecosystem || 'other';
entry.byEcosystem[eco] = (entry.byEcosystem[eco] || 0) + 1;
}
}
}
// Check for next page
const linkHeader = res.headers?.get?.('link') || '';
const nextMatch = linkHeader.match(/<([^>]+)>;\s*rel="next"/);
url = nextMatch ? nextMatch[1] : null;
if (url) await sleep(REQUEST_DELAY_MS);
} catch (err) {
if (verbose) process.stderr.write(` github page ${page} error: ${err.message}\n`);
break;
}
}
if (verbose) {
process.stderr.write(` github scanned ${totalScanned} advisories across ${page} pages\n`);
}
const results = [...CRYPTO_CWE_IDS].map(cweId => ({
cweId,
count: byCwe[cweId].count,
bySeverity: byCwe[cweId].bySeverity,
byEcosystem: byCwe[cweId].byEcosystem,
}));
return {
advisories: results,
collectedAt: new Date().toISOString(),
};
}
-160
lib/census/collectors/github-enrichment.mjs
/**
* Enrich top packages with GitHub repository metadata.
*
* Fetches stars, forks, last push date, and archived status from the GitHub API
* for the top packages by download count. Uses unauthenticated requests
* (60 req/hr limit), so we limit to 30 packages with 1s delays.
*/
const REQUEST_DELAY_MS = 1000;
const MAX_ENRICHMENTS = 30;
/**
* Static mapping of package name to GitHub owner/repo.
* Many registries don't expose repo URLs in download APIs,
* so we maintain this mapping for top packages.
*/
const KNOWN_REPOS = {
// npm
'crypto-js': 'brix/crypto-js',
'@noble/hashes': 'paulmillr/noble-hashes',
'@noble/curves': 'paulmillr/noble-curves',
'@noble/ciphers': 'paulmillr/noble-ciphers',
'@noble/post-quantum': 'paulmillr/noble-post-quantum',
'node-forge': 'digitalbazaar/forge',
'jose': 'panva/jose',
'elliptic': 'indutny/elliptic',
'hash.js': 'indutny/hash.js',
'tweetnacl': 'nicola/tweetnacl-js',
'bcryptjs': 'nicola/bcrypt.js',
'jsonwebtoken': 'auth0/node-jsonwebtoken',
'sodium-native': 'nicola/sodium-native',
'md5': 'pvorb/node-md5',
'scrypt-js': 'nicola/scrypt-js',
// PyPI
'cryptography': 'pyca/cryptography',
'pycryptodome': 'Legrandin/pycryptodome',
'bcrypt': 'pyca/bcrypt',
'pynacl': 'pyca/pynacl',
'argon2-cffi': 'hynek/argon2-cffi',
'PyJWT': 'jpadilla/pyjwt',
'liboqs-python': 'open-quantum-safe/liboqs-python',
// Rust crates
'ring': 'briansmith/ring',
'rustls': 'rustls/rustls',
'ed25519-dalek': 'dalek-cryptography/curve25519-dalek',
'sha2': 'RustCrypto/hashes',
'aes-gcm': 'RustCrypto/AEADs',
'chacha20poly1305': 'RustCrypto/AEADs',
'argon2': 'RustCrypto/password-hashes',
// Go
'github.com/cloudflare/circl': 'cloudflare/circl',
'golang-jwt/jwt/v5': 'golang-jwt/jwt',
// Maven
'org.bouncycastle:bcprov-jdk18on': 'bcgit/bc-java',
'com.google.crypto.tink:tink': 'google/tink',
'io.jsonwebtoken:jjwt-api': 'jwtk/jjwt',
// PHP
'phpseclib/phpseclib': 'phpseclib/phpseclib',
'defuse/php-encryption': 'defuse/php-encryption',
'firebase/php-jwt': 'firebase/php-jwt',
// Ruby
'rbnacl': 'crypto-rb/rbnacl',
'jwt': 'jwt/ruby-jwt',
// Dart
'pointycastle': 'nicola/pc-dart',
// Swift
'CryptoSwift': 'nicola/CryptoSwift',
};
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Enrich packages with GitHub metadata.
*
* @param {Array} allPackages - All package entries (with name + downloads)
* @param {Object} [options]
* @param {Function} [options.fetchFn] - Fetch implementation
* @param {boolean} [options.verbose] - Log progress
* @returns {Promise<Map<string, {stars: number, forks: number, lastPush: string, archived: boolean}>>}
*/
export async function collectGithubEnrichment(allPackages, options = {}) {
const fetchFn = options.fetchFn || globalThis.fetch;
const verbose = options.verbose || false;
// Sort by downloads descending and pick top packages that have known repos
const sorted = [...allPackages]
.sort((a, b) => b.downloads - a.downloads);
const toEnrich = [];
const seen = new Set();
for (const pkg of sorted) {
const repo = KNOWN_REPOS[pkg.name];
if (!repo || seen.has(repo)) continue;
seen.add(repo);
toEnrich.push({ name: pkg.name, repo });
if (toEnrich.length >= MAX_ENRICHMENTS) break;
}
if (verbose) {
process.stderr.write(` github enrichment: ${toEnrich.length} packages to enrich\n`);
}
const results = new Map();
for (const { name, repo } of toEnrich) {
try {
const res = await fetchFn(`https://api.github.com/repos/${repo}`, {
headers: { 'Accept': 'application/vnd.github+json' },
});
if (!res.ok) {
if (verbose) process.stderr.write(` github ${repo}: HTTP ${res.status}\n`);
// Check rate limit
const remaining = res.headers?.get?.('x-ratelimit-remaining');
if (remaining === '0') {
if (verbose) process.stderr.write(' github rate limit hit, stopping\n');
break;
}
await sleep(REQUEST_DELAY_MS);
continue;
}
const data = await res.json();
results.set(name, {
stars: data.stargazers_count || 0,
forks: data.forks_count || 0,
lastPush: data.pushed_at || null,
archived: data.archived || false,
});
if (verbose) {
process.stderr.write(` github ${repo}: ${data.stargazers_count} stars\n`);
}
await sleep(REQUEST_DELAY_MS);
} catch (err) {
if (verbose) process.stderr.write(` github ${repo} error: ${err.message}\n`);
await sleep(REQUEST_DELAY_MS);
}
}
if (verbose) {
process.stderr.write(` github enrichment: ${results.size} packages enriched\n`);
}
return {
enrichments: Object.fromEntries(results),
collectedAt: new Date().toISOString(),
};
}
-132
lib/census/collectors/go-downloads.mjs
/**
* Collect download estimates for Go cryptographic packages.
*
* The Go module proxy (proxy.golang.org) does not provide download
* statistics. This collector uses the Go module proxy to verify package
* existence and the GitHub API for star counts as a popularity proxy.
*
* For stdlib packages (crypto/*), download counts are estimated based on
* Go's total developer population (~3M monthly active) and usage survey
* data from the Go Developer Survey.
*
* Endpoints used:
* https://proxy.golang.org/{module}/@latest - verify module exists
* https://api.github.com/repos/{owner}/{repo} - star count (popularity proxy)
*/
const GO_PROXY = 'https://proxy.golang.org';
const REQUEST_DELAY_MS = 200;
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
// Estimated monthly "downloads" for Go stdlib crypto packages.
// Based on Go Developer Survey data: ~3M monthly active Go devs,
// and usage patterns from ecosystem surveys.
const STDLIB_ESTIMATES = {
'crypto/tls': 38_000_000,
'crypto/aes': 22_000_000,
'crypto/sha256': 18_000_000,
'crypto/ecdsa': 12_000_000,
'crypto/ed25519': 9_800_000,
'crypto/rsa': 8_900_000,
'crypto/rand': 25_000_000,
'crypto/hmac': 14_000_000,
'crypto/cipher': 16_000_000,
'crypto/x509': 15_000_000,
'crypto/sha512': 6_200_000,
'crypto/sha3': 2_100_000,
'crypto/ecdh': 4_500_000,
'crypto/hkdf': 1_200_000,
'crypto/mlkem': 180_000,
'crypto/md5': 5_200_000,
'crypto/sha1': 3_200_000,
'crypto/des': 20_000,
'crypto/rc4': 8_000,
'crypto/dsa': 15_000,
'crypto/elliptic': 800_000,
};
/**
* Fetch Go module download estimates.
*
* @param {import('../package-catalog.mjs').CatalogEntry[]} packages
* @param {Object} [options]
* @param {Function} [options.fetchFn] - Fetch implementation
* @param {boolean} [options.verbose] - Log progress
* @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
*/
export async function collectGoDownloads(packages, options = {}) {
const fetchFn = options.fetchFn || globalThis.fetch;
const verbose = options.verbose || false;
const results = [];
for (let i = 0; i < packages.length; i++) {
const pkg = packages[i];
if (verbose) {
process.stderr.write(` go ${i + 1}/${packages.length}: ${pkg.name}\n`);
}
// Stdlib packages: use hardcoded estimates
if (pkg.name.startsWith('crypto/')) {
const estimate = STDLIB_ESTIMATES[pkg.name] || 10_000;
results.push({ name: pkg.name, downloads: estimate, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
continue;
}
// Third-party modules: verify existence via proxy, estimate from GitHub stars
try {
const proxyUrl = `${GO_PROXY}/${pkg.name}/@latest`;
const res = await fetchFn(proxyUrl);
if (!res.ok) {
if (verbose) process.stderr.write(` go ${pkg.name}: proxy ${res.status}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
continue;
}
// Module exists; estimate downloads from GitHub stars if available
let downloads = 100_000; // Default for verified modules
// Extract GitHub owner/repo from module path
const ghMatch = pkg.name.match(/^github\.com\/([^/]+\/[^/]+)/);
if (ghMatch) {
try {
const ghRes = await fetchFn(`https://api.github.com/repos/${ghMatch[1]}`, {
headers: { Accept: 'application/vnd.github.v3+json' },
});
if (ghRes.ok) {
const ghData = await ghRes.json();
// Stars * 1000 as monthly usage estimate
downloads = (ghData.stargazers_count || 0) * 1000;
}
} catch {
// GitHub API failed, use default
}
}
// For x/crypto sub-packages, use umbrella module popularity
if (pkg.name.startsWith('golang.org/x/crypto')) {
downloads = pkg.name === 'golang.org/x/crypto'
? 45_000_000
: Math.max(downloads, 500_000);
}
results.push({ name: pkg.name, downloads, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
} catch (err) {
if (verbose) process.stderr.write(` go ${pkg.name} error: ${err.message}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
if (i < packages.length - 1) await sleep(REQUEST_DELAY_MS);
}
return {
packages: results.sort((a, b) => b.downloads - a.downloads),
period: 'estimated',
collectedAt: new Date().toISOString(),
};
}
-69
lib/census/collectors/hex-downloads.mjs
/**
* Collect download counts from the Hex.pm API.
*
* Endpoint: GET https://hex.pm/api/packages/{name}
* - Returns downloads with recent breakdown
* - No authentication required
*/
const HEX_API = 'https://hex.pm/api/packages';
const REQUEST_DELAY_MS = 300;
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Fetch Hex.pm download counts for a list of packages.
*
* @param {import('../package-catalog.mjs').CatalogEntry[]} packages
* @param {Object} [options]
* @param {Function} [options.fetchFn]
* @param {boolean} [options.verbose]
* @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
*/
export async function collectHexDownloads(packages, options = {}) {
const fetchFn = options.fetchFn || globalThis.fetch;
const verbose = options.verbose || false;
const results = [];
for (let i = 0; i < packages.length; i++) {
const pkg = packages[i];
if (verbose) {
process.stderr.write(` hex ${i + 1}/${packages.length}: ${pkg.name}\n`);
}
try {
const res = await fetchFn(`${HEX_API}/${pkg.name}`, {
headers: { 'Accept': 'application/json' },
});
if (!res.ok) {
if (verbose) process.stderr.write(` hex ${pkg.name}: ${res.status}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
} else {
const data = await res.json();
// Hex provides downloads.recent (last 90 days) and downloads.all
const recentDownloads = data?.downloads?.recent || 0;
// Estimate monthly from 90-day window
const monthlyEstimate = Math.round(recentDownloads / 3);
results.push({ name: pkg.name, downloads: monthlyEstimate, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
} catch (err) {
if (verbose) process.stderr.write(` hex ${pkg.name} error: ${err.message}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
if (i < packages.length - 1) {
await sleep(REQUEST_DELAY_MS);
}
}
return {
packages: results.sort((a, b) => b.downloads - a.downloads),
period: 'estimated_monthly',
collectedAt: new Date().toISOString(),
};
}
-96
lib/census/collectors/maven-downloads.mjs
/**
* Collect download estimates for Maven Central packages.
*
* Maven Central does not provide a public download count API.
* This collector uses the Sonatype Central search API to verify
* package existence and returns estimated download counts based on
* publicly available ecosystem data (Maven Central stats reports,
* Sonatype annual reports, and GitHub dependency graph data).
*
* Endpoint: GET https://search.maven.org/solrsearch/select
* - No authentication required
* - Used to verify package existence and get latest version
*/
const SEARCH_API = 'https://search.maven.org/solrsearch/select';
const REQUEST_DELAY_MS = 300;
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Parse Maven coordinate "groupId:artifactId" into parts.
*/
function parseCoord(name) {
const parts = name.split(':');
return { groupId: parts[0], artifactId: parts[1] || '' };
}
/**
* Fetch Maven Central package metadata and estimate downloads.
*
* @param {import('../package-catalog.mjs').CatalogEntry[]} packages
* @param {Object} [options]
* @param {Function} [options.fetchFn] - Fetch implementation
* @param {boolean} [options.verbose] - Log progress
* @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
*/
export async function collectMavenDownloads(packages, options = {}) {
const fetchFn = options.fetchFn || globalThis.fetch;
const verbose = options.verbose || false;
const results = [];
for (let i = 0; i < packages.length; i++) {
const pkg = packages[i];
const { groupId, artifactId } = parseCoord(pkg.name);
if (verbose) {
process.stderr.write(` maven ${i + 1}/${packages.length}: ${pkg.name}\n`);
}
try {
const q = `g:"${groupId}" AND a:"${artifactId}"`;
const url = `${SEARCH_API}?q=${encodeURIComponent(q)}&rows=1&wt=json`;
const res = await fetchFn(url);
if (!res.ok) {
if (verbose) process.stderr.write(` maven ${pkg.name}: ${res.status}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
} else {
const data = await res.json();
const doc = data?.response?.docs?.[0];
// Maven Central search returns versionCount which can proxy popularity.
// Estimate: versionCount * 50,000 as a rough monthly download proxy.
// This is crude but better than nothing since Maven has no download API.
const versionCount = doc?.versionCount || 0;
const estimatedDownloads = versionCount > 0 ? versionCount * 50_000 : 0;
results.push({
name: pkg.name,
downloads: estimatedDownloads,
tier: pkg.tier,
category: pkg.category,
replacedBy: pkg.replacedBy,
algorithms: pkg.algorithms,
note: pkg.note,
});
}
} catch (err) {
if (verbose) process.stderr.write(` maven ${pkg.name} error: ${err.message}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
if (i < packages.length - 1) {
await sleep(REQUEST_DELAY_MS);
}
}
return {
packages: results.sort((a, b) => b.downloads - a.downloads),
period: 'estimated',
collectedAt: new Date().toISOString(),
};
}
-132
lib/census/collectors/npm-downloads.mjs
/**
* Collect download counts from the npm registry API.
*
* Endpoint: GET https://api.npmjs.org/downloads/point/last-month/pkg1,pkg2,...
* - Bulk endpoint does NOT support scoped packages (@org/pkg)
* - Scoped packages must be fetched individually
* - No authentication required
*/
const NPM_API = 'https://api.npmjs.org/downloads/point/last-month';
const BATCH_SIZE = 50;
const BATCH_DELAY_MS = 1000;
const INDIVIDUAL_DELAY_MS = 200;
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Fetch a single package's download count.
*/
async function fetchSingle(pkg, fetchFn, period, verbose) {
try {
const res = await fetchFn(`${NPM_API}/${pkg.name}`);
if (res.ok) {
const data = await res.json();
if (!period.start && data.start) {
period.start = data.start;
period.end = data.end;
}
return { name: pkg.name, downloads: data.downloads || 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note };
}
if (verbose) process.stderr.write(` npm ${pkg.name}: HTTP ${res.status}\n`);
} catch (err) {
if (verbose) process.stderr.write(` npm ${pkg.name} error: ${err.message}\n`);
}
return { name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note };
}
/**
* Fetch npm download counts for a list of packages.
*
* @param {import('../package-catalog.mjs').CatalogEntry[]} packages
* @param {Object} [options]
* @param {Function} [options.fetchFn] - Fetch implementation (defaults to globalThis.fetch)
* @param {boolean} [options.verbose] - Log progress
* @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: {start: string, end: string}, collectedAt: string}>}
*/
export async function collectNpmDownloads(packages, options = {}) {
const fetchFn = options.fetchFn || globalThis.fetch;
const verbose = options.verbose || false;
const results = [];
const period = { start: '', end: '' };
// Separate scoped (@org/pkg) from unscoped packages
const scoped = packages.filter(p => p.name.startsWith('@'));
const unscoped = packages.filter(p => !p.name.startsWith('@'));
// Batch unscoped packages
const batches = [];
for (let i = 0; i < unscoped.length; i += BATCH_SIZE) {
batches.push(unscoped.slice(i, i + BATCH_SIZE));
}
for (let i = 0; i < batches.length; i++) {
const batch = batches[i];
const names = batch.map(p => p.name).join(',');
const url = `${NPM_API}/${names}`;
if (verbose) {
process.stderr.write(` npm batch ${i + 1}/${batches.length} (${batch.length} unscoped packages)\n`);
}
try {
const res = await fetchFn(url);
if (!res.ok) {
if (verbose) process.stderr.write(` npm batch ${i + 1} failed: ${res.status}, falling back to individual\n`);
for (const pkg of batch) {
results.push(await fetchSingle(pkg, fetchFn, period, verbose));
await sleep(INDIVIDUAL_DELAY_MS);
}
continue;
}
const data = await res.json();
if (batch.length === 1) {
if (!period.start && data.start) {
period.start = data.start;
period.end = data.end;
}
results.push({ name: batch[0].name, downloads: data.downloads || 0, tier: batch[0].tier, category: batch[0].category, replacedBy: batch[0].replacedBy, algorithms: batch[0].algorithms, note: batch[0].note });
} else {
for (const pkg of batch) {
const entry = data[pkg.name];
if (entry) {
if (!period.start && entry.start) {
period.start = entry.start;
period.end = entry.end;
}
results.push({ name: pkg.name, downloads: entry.downloads || 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
} else {
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
}
}
} catch (err) {
if (verbose) process.stderr.write(` npm batch ${i + 1} error: ${err.message}\n`);
for (const pkg of batch) {
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
}
if (i < batches.length - 1) await sleep(BATCH_DELAY_MS);
}
// Fetch scoped packages individually (bulk API doesn't support them)
if (scoped.length > 0 && verbose) {
process.stderr.write(` npm fetching ${scoped.length} scoped packages individually\n`);
}
for (let i = 0; i < scoped.length; i++) {
results.push(await fetchSingle(scoped[i], fetchFn, period, verbose));
if (i < scoped.length - 1) await sleep(INDIVIDUAL_DELAY_MS);
}
return {
packages: results.sort((a, b) => b.downloads - a.downloads),
period,
collectedAt: new Date().toISOString(),
};
}
-72
lib/census/collectors/nuget-downloads.mjs
/**
* Collect download counts from the NuGet API.
*
* Endpoint: GET https://api.nuget.org/v3/registration5-semver1/{id}/index.json
* - Returns per-version download counts
* - No authentication required
* - Rate limit: be polite, 300ms between requests
*
* Alternative: NuGet search API for total downloads
* GET https://azuresearch-usnc.nuget.org/query?q=packageid:{name}&take=1
*/
const NUGET_SEARCH = 'https://azuresearch-usnc.nuget.org/query';
const REQUEST_DELAY_MS = 300;
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Fetch NuGet download counts for a list of packages.
*
* @param {import('../package-catalog.mjs').CatalogEntry[]} packages
* @param {Object} [options]
* @param {Function} [options.fetchFn] - Fetch implementation (defaults to globalThis.fetch)
* @param {boolean} [options.verbose] - Log progress
* @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
*/
export async function collectNugetDownloads(packages, options = {}) {
const fetchFn = options.fetchFn || globalThis.fetch;
const verbose = options.verbose || false;
const results = [];
for (let i = 0; i < packages.length; i++) {
const pkg = packages[i];
if (verbose) {
process.stderr.write(` nuget ${i + 1}/${packages.length}: ${pkg.name}\n`);
}
try {
const url = `${NUGET_SEARCH}?q=packageid:${encodeURIComponent(pkg.name)}&take=1`;
const res = await fetchFn(url);
if (!res.ok) {
if (verbose) process.stderr.write(` nuget ${pkg.name}: ${res.status}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
} else {
const data = await res.json();
const entry = data?.data?.[0];
// NuGet returns total downloads, estimate monthly as total / 36 (3 years average)
const totalDownloads = entry?.totalDownloads || 0;
const monthlyEstimate = Math.round(totalDownloads / 36);
results.push({ name: pkg.name, downloads: monthlyEstimate, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
} catch (err) {
if (verbose) process.stderr.write(` nuget ${pkg.name} error: ${err.message}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
if (i < packages.length - 1) {
await sleep(REQUEST_DELAY_MS);
}
}
return {
packages: results.sort((a, b) => b.downloads - a.downloads),
period: 'last_month_estimated',
collectedAt: new Date().toISOString(),
};
}
-71
lib/census/collectors/nvd-cves.mjs
/**
* Collect crypto-related CVE counts from NVD (National Vulnerability Database).
*
* Endpoint: GET https://services.nvd.nist.gov/rest/json/cves/2.0?cweId=CWE-XXX&resultsPerPage=1
* - Free, no authentication required (API key optional for higher rate limits)
* - Rate limit: 5 requests per 30 seconds without API key
* - We use 7s delay between requests to stay well under limits
* - resultsPerPage=1 to minimize payload (we only need totalResults)
*/
const NVD_API = 'https://services.nvd.nist.gov/rest/json/cves/2.0';
const REQUEST_DELAY_MS = 7000;
const CRYPTO_CWES = [
{ id: 'CWE-327', name: 'Use of a Broken or Risky Cryptographic Algorithm' },
{ id: 'CWE-326', name: 'Inadequate Encryption Strength' },
{ id: 'CWE-328', name: 'Use of Weak Hash' },
];
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Fetch crypto-related CVE counts from NVD.
*
* @param {Object} [options]
* @param {Function} [options.fetchFn] - Fetch implementation (defaults to globalThis.fetch)
* @param {boolean} [options.verbose] - Log progress
* @returns {Promise<{cves: Array<{cweId: string, cweName: string, totalCount: number}>, collectedAt: string}>}
*/
export async function collectNvdCves(options = {}) {
const fetchFn = options.fetchFn || globalThis.fetch;
const verbose = options.verbose || false;
const results = [];
for (let i = 0; i < CRYPTO_CWES.length; i++) {
const cwe = CRYPTO_CWES[i];
const url = `${NVD_API}?cweId=${cwe.id}&resultsPerPage=1`;
if (verbose) {
process.stderr.write(` nvd ${i + 1}/${CRYPTO_CWES.length}: ${cwe.id} (${cwe.name})\n`);
}
try {
const res = await fetchFn(url);
if (!res.ok) {
if (verbose) process.stderr.write(` nvd ${cwe.id}: HTTP ${res.status}\n`);
results.push({ cweId: cwe.id, cweName: cwe.name, totalCount: 0 });
} else {
const data = await res.json();
const totalCount = data?.totalResults || 0;
results.push({ cweId: cwe.id, cweName: cwe.name, totalCount });
}
} catch (err) {
if (verbose) process.stderr.write(` nvd ${cwe.id} error: ${err.message}\n`);
results.push({ cweId: cwe.id, cweName: cwe.name, totalCount: 0 });
}
// Delay between requests (NVD rate limit)
if (i < CRYPTO_CWES.length - 1) {
await sleep(REQUEST_DELAY_MS);
}
}
return {
cves: results,
collectedAt: new Date().toISOString(),
};
}
-65
lib/census/collectors/packagist-downloads.mjs
/**
* Collect download counts from the Packagist API.
*
* Endpoint: GET https://packagist.org/packages/{name}.json
* - Returns total downloads and monthly downloads
* - No authentication required
* - Rate limit: be polite, 300ms between requests
*/
const PACKAGIST_API = 'https://packagist.org/packages';
const REQUEST_DELAY_MS = 300;
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Fetch Packagist download counts for a list of packages.
*
* @param {import('../package-catalog.mjs').CatalogEntry[]} packages
* @param {Object} [options]
* @param {Function} [options.fetchFn] - Fetch implementation (defaults to globalThis.fetch)
* @param {boolean} [options.verbose] - Log progress
* @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
*/
export async function collectPackagistDownloads(packages, options = {}) {
const fetchFn = options.fetchFn || globalThis.fetch;
const verbose = options.verbose || false;
const results = [];
for (let i = 0; i < packages.length; i++) {
const pkg = packages[i];
if (verbose) {
process.stderr.write(` packagist ${i + 1}/${packages.length}: ${pkg.name}\n`);
}
try {
const res = await fetchFn(`${PACKAGIST_API}/${pkg.name}.json`);
if (!res.ok) {
if (verbose) process.stderr.write(` packagist ${pkg.name}: ${res.status}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
} else {
const data = await res.json();
const monthlyDownloads = data?.package?.downloads?.monthly || 0;
results.push({ name: pkg.name, downloads: monthlyDownloads, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
} catch (err) {
if (verbose) process.stderr.write(` packagist ${pkg.name} error: ${err.message}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
if (i < packages.length - 1) {
await sleep(REQUEST_DELAY_MS);
}
}
return {
packages: results.sort((a, b) => b.downloads - a.downloads),
period: 'last_month',
collectedAt: new Date().toISOString(),
};
}
-65
lib/census/collectors/pub-downloads.mjs
/**
* Collect download counts from the pub.dev API.
*
* Endpoint: GET https://pub.dev/api/packages/{name}/score
* - Returns downloadCount30Days (or estimate from likes/popularity)
* - No authentication required
*/
const PUB_API = 'https://pub.dev/api/packages';
const REQUEST_DELAY_MS = 300;
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Fetch pub.dev download counts for a list of packages.
*
* @param {import('../package-catalog.mjs').CatalogEntry[]} packages
* @param {Object} [options]
* @param {Function} [options.fetchFn]
* @param {boolean} [options.verbose]
* @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
*/
export async function collectPubDownloads(packages, options = {}) {
const fetchFn = options.fetchFn || globalThis.fetch;
const verbose = options.verbose || false;
const results = [];
for (let i = 0; i < packages.length; i++) {
const pkg = packages[i];
if (verbose) {
process.stderr.write(` pub ${i + 1}/${packages.length}: ${pkg.name}\n`);
}
try {
const res = await fetchFn(`${PUB_API}/${pkg.name}/score`);
if (!res.ok) {
if (verbose) process.stderr.write(` pub ${pkg.name}: ${res.status}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
} else {
const data = await res.json();
// pub.dev score endpoint has downloadCount30Days
const downloads = data?.downloadCount30Days || 0;
results.push({ name: pkg.name, downloads: downloads, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
} catch (err) {
if (verbose) process.stderr.write(` pub ${pkg.name} error: ${err.message}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
if (i < packages.length - 1) {
await sleep(REQUEST_DELAY_MS);
}
}
return {
packages: results.sort((a, b) => b.downloads - a.downloads),
period: 'last_month',
collectedAt: new Date().toISOString(),
};
}
-67
lib/census/collectors/pypi-downloads.mjs
/**
* Collect download counts from PyPI Stats API.
*
* Endpoint: GET https://pypistats.org/api/packages/{pkg}/recent
* - Individual requests only (no batch endpoint)
* - No authentication required
* - 500ms delay between requests to be polite
*/
const PYPI_API = 'https://pypistats.org/api/packages';
const REQUEST_DELAY_MS = 500;
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Fetch PyPI download counts for a list of packages.
*
* @param {import('../package-catalog.mjs').CatalogEntry[]} packages
* @param {Object} [options]
* @param {Function} [options.fetchFn] - Fetch implementation (defaults to globalThis.fetch)
* @param {boolean} [options.verbose] - Log progress
* @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
*/
export async function collectPypiDownloads(packages, options = {}) {
const fetchFn = options.fetchFn || globalThis.fetch;
const verbose = options.verbose || false;
const results = [];
for (let i = 0; i < packages.length; i++) {
const pkg = packages[i];
const url = `${PYPI_API}/${pkg.name}/recent`;
if (verbose) {
process.stderr.write(` pypi ${i + 1}/${packages.length}: ${pkg.name}\n`);
}
try {
const res = await fetchFn(url);
if (!res.ok) {
if (verbose) process.stderr.write(` pypi ${pkg.name}: ${res.status}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
} else {
const data = await res.json();
// Response: { data: { last_month: N, last_week: N, last_day: N }, ... }
const downloads = data?.data?.last_month || 0;
results.push({ name: pkg.name, downloads, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
} catch (err) {
if (verbose) process.stderr.write(` pypi ${pkg.name} error: ${err.message}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
// Delay between requests
if (i < packages.length - 1) {
await sleep(REQUEST_DELAY_MS);
}
}
return {
packages: results.sort((a, b) => b.downloads - a.downloads),
period: 'last_month',
collectedAt: new Date().toISOString(),
};
}
-67
lib/census/collectors/rubygems-downloads.mjs
/**
* Collect download counts from the RubyGems API.
*
* Endpoint: GET https://rubygems.org/api/v1/gems/{name}.json
* - Returns total downloads (no monthly breakdown)
* - Estimate monthly = total / 120 (approx 10 years of data)
* - No authentication required
*/
const RUBYGEMS_API = 'https://rubygems.org/api/v1/gems';
const REQUEST_DELAY_MS = 300;
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Fetch RubyGems download counts for a list of packages.
*
* @param {import('../package-catalog.mjs').CatalogEntry[]} packages
* @param {Object} [options]
* @param {Function} [options.fetchFn]
* @param {boolean} [options.verbose]
* @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
*/
export async function collectRubygemsDownloads(packages, options = {}) {
const fetchFn = options.fetchFn || globalThis.fetch;
const verbose = options.verbose || false;
const results = [];
for (let i = 0; i < packages.length; i++) {
const pkg = packages[i];
if (verbose) {
process.stderr.write(` rubygems ${i + 1}/${packages.length}: ${pkg.name}\n`);
}
try {
const res = await fetchFn(`${RUBYGEMS_API}/${pkg.name}.json`);
if (!res.ok) {
if (verbose) process.stderr.write(` rubygems ${pkg.name}: ${res.status}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
} else {
const data = await res.json();
// RubyGems only provides total downloads; estimate monthly
const totalDownloads = data?.downloads || 0;
const monthlyEstimate = Math.round(totalDownloads / 120);
results.push({ name: pkg.name, downloads: monthlyEstimate, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
} catch (err) {
if (verbose) process.stderr.write(` rubygems ${pkg.name} error: ${err.message}\n`);
results.push({ name: pkg.name, downloads: 0, tier: pkg.tier, category: pkg.category, replacedBy: pkg.replacedBy, algorithms: pkg.algorithms, note: pkg.note });
}
if (i < packages.length - 1) {
await sleep(REQUEST_DELAY_MS);
}
}
return {
packages: results.sort((a, b) => b.downloads - a.downloads),
period: 'estimated_monthly',
collectedAt: new Date().toISOString(),
};
}
lib/census/package-catalog.mjs

Sorry, the diff of this file is too big to display

bin/cryptoserve.mjs

Sorry, the diff of this file is too big to display