Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

@antora/content-aggregator

Package Overview
Dependencies
Maintainers
2
Versions
86
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@antora/content-aggregator - npm Package Compare versions

Comparing version 3.0.3 to 3.1.0

lib/compute-origin.js
lib/deep-clone.js
lib/deep-flatten.js
lib/logger.js
lib/posixify.js
lib/remove-git-suffix.js

293

lib/aggregate-content.js
'use strict'
const camelCaseKeys = require('camelcase-keys')
const computeOrigin = require('./compute-origin')
const { createHash } = require('crypto')
const createGitHttpPlugin = require('./git-plugin-http')
const decodeUint8Array = require('./decode-uint8-array')
const deepClone = require('./deep-clone')
const deepFlatten = require('./deep-flatten')
const EventEmitter = require('events')

@@ -11,3 +13,2 @@ const expandPath = require('@antora/expand-path-helper')

const filterRefs = require('./filter-refs')
const flattenDeep = require('./flatten-deep')
const fs = require('fs')

@@ -21,2 +22,3 @@ const { promises: fsp } = fs

const invariably = require('./invariably')
const logger = require('./logger')
const { makeMatcherRx, versionMatcherOpts: VERSION_MATCHER_OPTS } = require('./matcher')

@@ -26,3 +28,4 @@ const MultiProgress = require('multi-progress') // calls require('progress') as a peer dependencies

const { posix: path } = ospath
const posixify = ospath.sep === '\\' ? (p) => p.replace(/\\/g, '/') : undefined
const posixify = require('./posixify')
const removeGitSuffix = require('./remove-git-suffix')
const { fs: resolvePathGlobsFs, git: resolvePathGlobsGit } = require('./resolve-path-globs')

@@ -50,8 +53,5 @@ const { pipeline, Writable } = require('stream')

const VENTILATED_CSV_RX = /\s*,\s+/
const EDIT_URL_TEMPLATE_VAR_RX = /\{(web_url|ref(?:hash|name)|path)\}/g
const GIT_SUFFIX_RX = /(?:(?:(?:\.git)?\/)?\.git|\/)$/
const GIT_URI_DETECTOR_RX = /:(?:\/\/|[^/\\])/
const HEADS_DIR_RX = /^heads\//
const HOSTED_GIT_REPO_RX = /^(?:https?:\/\/|.+@)(git(?:hub|lab)\.com|bitbucket\.org|pagure\.io)[/:](.+?)(?:\.git)?$/
const HTTP_ERROR_CODE_RX = new RegExp('^' + git.Errors.HttpError.code + '$', 'i')
const NEWLINE_RX = /\n/g
const PATH_SEPARATOR_RX = /[/]/g

@@ -161,3 +161,3 @@ const SHORTEN_REF_RX = /^refs\/(?:heads|remotes\/[^/]+|tags)\//

return [
...flattenDeep(componentVersionBuckets)
...deepFlatten(componentVersionBuckets)
.reduce((accum, batch) => {

@@ -167,4 +167,5 @@ const key = batch.version + '@' + batch.name

if (!entry) return accum.set(key, batch)
const files = batch.files
const { files, origins } = batch
;(batch.files = entry.files).push(...files)
;(batch.origins = entry.origins).push(origins[0])
Object.assign(entry, batch)

@@ -266,5 +267,13 @@ return accum

const originUrl = repo.url || (await resolveRemoteUrl(repo, remoteName))
return selectReferences(source, repo, remoteName).then((refs) =>
Promise.all(refs.map((ref) => collectFilesFromReference(source, repo, remoteName, authStatus, ref, originUrl)))
)
return selectReferences(source, repo, remoteName).then((refs) => {
if (!refs.length) {
const { url, branches, tags, startPath, startPaths } = source
const startPathInfo =
'startPaths' in source ? { 'start paths': startPaths || undefined } : { 'start path': startPath || undefined }
const sourceInfo = yaml.dump({ url, branches, tags, ...startPathInfo }, { flowLevel: 1 }).trimRight()
logger.info(`No matching references found for content source entry (${sourceInfo.replace(NEWLINE_RX, ' | ')})`)
return []
}
return Promise.all(refs.map((it) => collectFilesFromReference(source, repo, remoteName, authStatus, it, originUrl)))
})
}

@@ -277,3 +286,3 @@

const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
const noWorktree = repo.url ? undefined : null
const noWorktree = repo.url ? undefined : false
const refs = new Map()

@@ -414,4 +423,5 @@ if (

if (!startPaths.length) {
const refInfo = `ref: ${ref.fullname.replace(HEADS_DIR_RX, '')}${worktreePath ? ' <worktree>' : ''}`
throw new Error(`no start paths found in ${displayUrl} (${refInfo})`)
const where = worktreePath || (worktreePath === false ? repo.gitdir : displayUrl)
const flag = worktreePath ? ' <worktree>' : ref.remote && worktreePath === false ? ` <remotes/${ref.remote}>` : ''
throw new Error(`no start paths found in ${where} (${ref.type}: ${ref.shortname}${flag})`)
}

@@ -429,25 +439,26 @@ return Promise.all(

function collectFilesFromStartPath (startPath, repo, authStatus, ref, worktreePath, originUrl, editUrl, version) {
return (
worktreePath ? readFilesFromWorktree(worktreePath, startPath) : readFilesFromGitTree(repo, ref.oid, startPath)
)
.then((files) => {
const componentVersionBucket = loadComponentDescriptor(files, ref, version)
const origin = computeOrigin(originUrl, authStatus, repo.gitdir, ref, startPath, worktreePath, editUrl)
componentVersionBucket.files = files.map((file) => assignFileProperties(file, origin))
return componentVersionBucket
})
const origin = computeOrigin(originUrl, authStatus, repo.gitdir, ref, startPath, worktreePath, editUrl)
return (worktreePath ? readFilesFromWorktree(origin) : readFilesFromGitTree(repo, ref.oid, startPath))
.then((files) =>
Object.assign(deepClone((origin.descriptor = loadComponentDescriptor(files, ref, version))), {
files: files.map((file) => assignFileProperties(file, origin)),
origins: [origin],
})
)
.catch((err) => {
const msg = err.message
const refInfo = `ref: ${ref.fullname.replace(HEADS_DIR_RX, '')}${worktreePath ? ' <worktree>' : ''}`
const pathInfo = !startPath || msg.startsWith('the start path ') ? '' : ' | path: ' + startPath
throw Object.assign(err, { message: msg.replace(/$/m, ` in ${repo.url || repo.dir} (${refInfo}${pathInfo})`) })
const where = worktreePath || (worktreePath === false ? repo.gitdir : repo.url || repo.dir)
const flag = worktreePath ? ' <worktree>' : ref.remote && worktreePath === false ? ` <remotes/${ref.remote}>` : ''
const pathInfo = startPath ? (err.message.startsWith('the start path ') ? '' : ` | start path: ${startPath}`) : ''
const message = err.message.replace(/$/m, ` in ${where} (${ref.type}: ${ref.shortname}${flag}${pathInfo})`)
throw Object.assign(err, { message })
})
}
function readFilesFromWorktree (worktreePath, startPath) {
const cwd = ospath.join(worktreePath, startPath, '.') // . shaves off trailing slash
function readFilesFromWorktree (origin) {
const startPath = origin.startPath
const cwd = ospath.join(origin.worktree, startPath, '.') // . shaves off trailing slash
return fsp.stat(cwd).then(
(startPathStat) => {
if (!startPathStat.isDirectory()) throw new Error(`the start path '${startPath}' is not a directory`)
return srcFs(cwd)
return srcFs(cwd, origin)
},

@@ -460,3 +471,3 @@ () => {

function srcFs (cwd) {
function srcFs (cwd, origin) {
const relpathStart = cwd.length + 1

@@ -479,3 +490,7 @@ return new Promise((resolve, reject, cache = Object.create(null), files = []) =>

(readErr) => {
done(Object.assign(readErr, { message: readErr.message.replace(`'${abspath}'`, relpath) }))
const logObject = { file: { abspath, origin } }
readErr.code === 'ENOENT'
? logger.warn(logObject, `ENOENT: file or directory disappeared, ${readErr.syscall} ${relpath}`)
: logger.error(logObject, readErr.message.replace(`'${abspath}'`, relpath))
done()
}

@@ -485,11 +500,15 @@ )

(statErr) => {
const logObject = { file: { abspath, origin } }
if (statErr.symlink) {
statErr.message =
statErr.code === 'ELOOP'
? `Symbolic link cycle detected at ${relpath}`
: `Broken symbolic link detected at ${relpath}`
logger.error(
logObject,
(statErr.code === 'ELOOP' ? 'ELOOP: symbolic link cycle, ' : 'ENOENT: broken symbolic link, ') +
`${relpath} -> ${statErr.symlink}`
)
} else if (statErr.code === 'ENOENT') {
logger.warn(logObject, `ENOENT: file or directory disappeared, ${statErr.syscall} ${relpath}`)
} else {
statErr.message = statErr.message.replace(`'${abspath}'`, relpath)
logger.error(logObject, statErr.message.replace(`'${abspath}'`, relpath))
}
done(statErr)
done()
}

@@ -526,6 +545,6 @@ )

const files = []
createGitTreeWalker(repo, root, filterGitEntry)
.on('entry', (entry) => files.push(entryToFile(entry)))
createGitTreeWalker(repo, root, filterGitEntry, gitEntryToFile)
.on('entry', (file) => files.push(file))
.on('error', reject)
.on('end', () => resolve(Promise.all(files)))
.on('end', () => resolve(files))
.walk(start)

@@ -535,12 +554,8 @@ })

function createGitTreeWalker (repo, root, filter) {
function createGitTreeWalker (repo, root, filter, convert) {
return Object.assign(new EventEmitter(), {
walk (start) {
return (
visitGitTree(this, repo, root, filter, start)
// NOTE if error is thrown, promises already being resolved won't halt
.then(
() => this.emit('end'),
(err) => this.emit('error', err)
)
return visitGitTree(this, repo, root, filter, convert, start).then(
() => this.emit('end'),
(err) => this.emit('error', err)
)

@@ -551,3 +566,3 @@ },

function visitGitTree (emitter, repo, root, filter, parent, dirname = '', following = new Set()) {
function visitGitTree (emitter, repo, root, filter, convert, parent, dirname = '', following = new Set()) {
const reads = []

@@ -562,3 +577,3 @@ for (const entry of parent.tree) {

Object.assign(subtree, { dirname: path.join(parent.dirname, entry.path) })
return visitGitTree(emitter, repo, root, filter, subtree, vfilePath, following)
return visitGitTree(emitter, repo, root, filter, convert, subtree, vfilePath, following)
})

@@ -573,13 +588,14 @@ )

if (target.type === 'tree') {
return visitGitTree(emitter, repo, root, filter, target, vfilePath, new Set(following).add(entry.oid))
return visitGitTree(emitter, repo, root, filter, convert, target, vfilePath, target.following)
} else if (target.type === 'blob' && filterVerdict === true && (mode = FILE_MODES[target.mode])) {
emitter.emit('entry', Object.assign({ mode, oid: target.oid, path: vfilePath }, repo))
return convert(Object.assign({ mode, oid: target.oid, path: vfilePath }, repo)).then((result) =>
emitter.emit('entry', result)
)
}
},
(err) => {
// NOTE this error could be caught after promise chain has already been rejected
if (err instanceof NotFoundError) {
err.message = `Broken symbolic link detected at ${vfilePath}`
} else if (err.code === 'SymbolicLinkCycleError') {
err.message = `Symbolic link cycle detected at ${vfilePath}`
if (err.symlink) {
err.message =
(err.code === 'ELOOP' ? 'ELOOP: symbolic link cycle' : 'ENOENT: broken symbolic link') +
`, ${vfilePath} -> ${err.symlink}`
}

@@ -591,3 +607,7 @@ throw err

} else if ((mode = FILE_MODES[entry.mode])) {
emitter.emit('entry', Object.assign({ mode, oid: entry.oid, path: vfilePath }, repo))
reads.push(
convert(Object.assign({ mode, oid: entry.oid, path: vfilePath }, repo)).then((result) =>
emitter.emit('entry', result)
)
)
}

@@ -597,30 +617,39 @@ }

}
return Promise.all(reads)
// NOTE preserve scan order so error for symbolic link cycle is deterministic; ensures no rejections after resolve
return Promise.allSettled(reads).then((results) => {
const rejected = results.find(({ reason }) => reason)
if (rejected) throw rejected.reason
})
}
function readGitSymlink (repo, root, parent, { oid }, following) {
if (following.size !== (following = new Set(following).add(oid)).size) {
return git.readBlob(Object.assign({ oid }, repo)).then(({ blob: target }) => {
target = decodeUint8Array(target)
let targetParent
if (parent.dirname) {
const dirname = parent.dirname + '/'
target = path.join(dirname, target) // join doesn't remove trailing separator
if (target.startsWith(dirname)) {
target = target.substr(dirname.length)
targetParent = parent
} else {
targetParent = root
}
} else {
target = path.normalize(target) // normalize doesn't remove trailing separator
targetParent = root
function readGitSymlink (repo, root, parent, { oid, path: name }, following) {
const dirname = parent.dirname
if (following.size === (following = new Set(following)).add(oid).size) {
const err = { name: 'SymbolicLinkCycleError', code: 'ELOOP', oid, path: `${path.join(dirname, name)}` }
return Promise.reject(Object.assign(new Error(`Symbolic link cycle detected at ${oid}:${err.path}`), err))
}
return git.readBlob(Object.assign({ oid }, repo)).then(({ blob: symlink }) => {
symlink = decodeUint8Array(symlink)
let target
let targetParent = root
if (dirname) {
if (!(target = path.join('/', dirname, symlink).substr(1)) || target === dirname) {
target = '.'
} else if (target.startsWith(dirname + '/')) {
target = target.substr(dirname.length + 1) // join doesn't remove trailing separator
targetParent = parent
}
const targetSegments = target.split('/')
if (!targetSegments[targetSegments.length - 1]) targetSegments.pop()
return readGitObjectAtPath(repo, root, targetParent, targetSegments, following)
} else {
target = path.normalize(symlink) // normalize doesn't remove trailing separator
}
if (target === '.') {
const err = { name: 'SymbolicLinkCycleError', code: 'ELOOP', oid, path: `${path.join(dirname, name)}`, symlink }
return Promise.reject(Object.assign(new Error(`Symbolic link cycle detected at ${oid}:${err.path}`), err))
}
const targetSegments = target.split('/')
targetSegments[targetSegments.length - 1] || targetSegments.pop()
return readGitObjectAtPath(repo, root, targetParent, targetSegments, following).catch((err) => {
throw Object.assign(err, { symlink })
})
}
const err = { name: 'SymbolicLinkCycleError', code: 'SymbolicLinkCycleError', oid }
return Promise.reject(Object.assign(new Error(`Symbolic link cycle found at oid: ${err.oid}`), err))
})
}

@@ -638,3 +667,3 @@

? readGitObjectAtPath(repo, root, subtree, pathSegments, following)
: Object.assign(subtree, { type: 'tree' })
: Object.assign(subtree, { type: 'tree', following }) // Q: should this create copy?
})

@@ -646,3 +675,3 @@ : entry.mode === SYMLINK_FILE_MODE

}
return Promise.reject(new NotFoundError(`No file or directory found at "${parent.oid}:${pathSegments.join('/')}"`))
return Promise.reject(new NotFoundError(`${parent.oid}:${pathSegments.join('/')}`))
}

@@ -662,3 +691,3 @@

function entryToFile (entry) {
function gitEntryToFile (entry) {
return git.readBlob(entry).then(({ blob: contents }) => {

@@ -717,51 +746,5 @@ contents = Buffer.from(contents.buffer)

data.version = version
return camelCaseKeys(data, { deep: true, stopPaths: ['asciidoc'] })
return camelCaseKeys(data, ['asciidoc'])
}
function computeOrigin (url, authStatus, gitdir, ref, startPath, worktreePath = undefined, editUrl = true) {
const { shortname: refname, oid: refhash, type: reftype } = ref
const origin = { type: 'git', url, gitdir, refname, [reftype]: refname, startPath }
if (authStatus) origin.private = authStatus
if (worktreePath === undefined) {
origin.refhash = refhash
} else {
if (worktreePath) {
origin.fileUriPattern =
(posixify ? 'file:///' + posixify(worktreePath) : 'file://' + worktreePath) +
(startPath ? '/' + startPath + '/%s' : '/%s')
} else {
origin.refhash = refhash
}
origin.worktree = worktreePath
if (url.startsWith('file://')) url = undefined
}
if (url) origin.webUrl = url.replace(GIT_SUFFIX_RX, '')
if (editUrl === true) {
let match
if (url && (match = url.match(HOSTED_GIT_REPO_RX))) {
const host = match[1]
let action
let category = ''
if (host === 'pagure.io') {
action = 'blob'
category = 'f'
} else if (host === 'bitbucket.org') {
action = 'src'
} else {
action = reftype === 'branch' ? 'edit' : 'blob'
}
origin.editUrlPattern = 'https://' + path.join(match[1], match[2], action, refname, category, startPath, '%s')
}
} else if (editUrl) {
const vars = {
path: () => (startPath ? path.join(startPath, '%s') : '%s'),
refhash: () => refhash,
refname: () => refname,
web_url: () => origin.webUrl || '',
}
origin.editUrlPattern = editUrl.replace(EDIT_URL_TEMPLATE_VAR_RX, (_, name) => vars[name]())
}
return origin
}
function assignFileProperties (file, origin) {

@@ -850,3 +833,3 @@ if (!file.src) file.src = {}

if (phaseIdx) ratio += (phaseIdx * scaleFactor) / GIT_PROGRESS_PHASES.length
// NOTE: updates are automatically throttled based on renderThrottle option
// NOTE updates are automatically throttled based on renderThrottle option
this.update(ratio > scaleFactor ? scaleFactor : ratio)

@@ -858,6 +841,6 @@ }

if (err) {
// TODO: could use progressBar.interrupt() to replace bar with message instead
// TODO could use progressBar.interrupt() to replace bar with message instead
this.chars.incomplete = '?'
this.update(0)
// NOTE: force progress bar to update regardless of throttle setting
// NOTE force progress bar to update regardless of throttle setting
this.render(undefined, true)

@@ -900,3 +883,3 @@ } else {

if (posixify) normalizedUrl = posixify(normalizedUrl)
normalizedUrl = normalizedUrl.replace(GIT_SUFFIX_RX, '')
normalizedUrl = removeGitSuffix(normalizedUrl)
const basename = normalizedUrl.split(ANY_SEPARATOR_RX).pop()

@@ -944,5 +927,10 @@ const hash = createHash('sha1')

if (!lstat.isSymbolicLink()) return lstat
return fsp.stat(path_).catch((statErr) => {
throw Object.assign(statErr, { symlink: true })
})
return fsp.stat(path_).catch((statErr) =>
fsp
.readlink(path_)
.catch(invariably.void)
.then((symlink) => {
throw Object.assign(statErr, { symlink })
})
)
})

@@ -1025,5 +1013,5 @@ }

}
const wrappedErr = new Error(`${wrappedMsg} (url: ${displayUrl})`)
wrappedErr.stack += `\nCaused by: ${err.stack || 'unknown'}`
return wrappedErr
const errWrapper = new Error(`${wrappedMsg} (url: ${displayUrl})`)
errWrapper.stack += `\nCaused by: ${err.stack || 'unknown'}`
return errWrapper
}

@@ -1035,4 +1023,12 @@

function coerceToString (value) {
return value == null ? '' : String(value)
function camelCaseKeys (o, stopPaths = [], p = '') {
if (Array.isArray(o)) return o.map((it) => camelCaseKeys(it, stopPaths, p))
if (o == null || o.constructor !== Object) return o
const pathPrefix = p && p + '.'
const accum = {}
for (const [k, v] of Object.entries(o)) {
const camelKey = k.toLowerCase().replace(/[_-]([a-z0-9])/g, (_, l, idx) => (idx ? l.toUpperCase() : l))
accum[camelKey] = ~stopPaths.indexOf(pathPrefix + camelKey) ? v : camelCaseKeys(v, stopPaths, pathPrefix + camelKey)
}
return accum
}

@@ -1044,2 +1040,6 @@

function coerceToString (value) {
return value == null ? '' : String(value)
}
function findWorktrees (repo, patterns) {

@@ -1081,2 +1081,1 @@ if (!patterns.length) return new Map()

module.exports = aggregateContent
module.exports._computeOrigin = computeOrigin

4

lib/resolve-path-globs.js
'use strict'
const flattenDeep = require('./flatten-deep')
const deepFlatten = require('./deep-flatten')
const { promises: fsp } = require('fs')

@@ -59,3 +59,3 @@ const git = require('./git')

if (explicit) dirents = dirents.filter((dirent) => !explicit.has(dirent.name))
const discovered = flattenDeep(
const discovered = deepFlatten(
await Promise.all(

@@ -62,0 +62,0 @@ dirents.map((dirent) =>

21

package.json
{
"name": "@antora/content-aggregator",
"version": "3.0.3",
"version": "3.1.0",
"description": "Fetches and aggregates content from distributed sources for use in an Antora documentation pipeline.",

@@ -19,11 +19,22 @@ "license": "MPL-2.0",

"main": "lib/index.js",
"exports": {
".": "./lib/index.js",
"./git": "./lib/git.js",
"./git/http-plugin": "./lib/git-plugin-http.js",
"./lib/git-plugin-http": "./lib/git-plugin-http.js",
"./package.json": "./package.json"
},
"imports": {
"#compute-origin": "./lib/compute-origin.js",
"#constants": "./lib/constants.js"
},
"dependencies": {
"@antora/expand-path-helper": "~2.0",
"@antora/logger": "3.1.0",
"@antora/user-require-helper": "~2.0",
"braces": "~3.0",
"cache-directory": "~2.0",
"camelcase-keys": "~7.0",
"glob-stream": "~7.0",
"hpagent": "~0.1.0",
"isomorphic-git": "~1.10",
"hpagent": "~1.0",
"isomorphic-git": "~1.19",
"js-yaml": "~4.1",

@@ -38,3 +49,3 @@ "multi-progress": "~4.0",

"engines": {
"node": ">=12.21.0"
"node": ">=16.0.0"
},

@@ -41,0 +52,0 @@ "files": [

lib/flatten-deep.js
SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc