crawler.plugin.queue
Advanced tools
Readme
用于发现html文档中的地址链接
当前支持的模式
partten: role:crawler.plugin.queue,cmd:analyze 测试地址：http://172.16.112.215:9003/act POST
返回数据结构:
{
"type":"array",
"description":"返回的数据结构",
"items":{
"type":"object",
"properties":{
"url":{"type":"string","title":"下载链接的详细地址"},
"path":{"type":"string","title":"下载链接的路径"},
"query":{"type":"string","title":"下载链接的地址栏参数信息"},
"protocol":{"type":"string","title":"下载链接的协议"},
"port":{"type":"number","title":"下载链接的端口"},
"hostname":{"type":"string","title":"下载链接的域名",
"depth":{"type":"number","title":"下载链接深度"},
}
}
}
参数数据结构:
{
"type":"object",
"description":"参数",
"required":["queueItem"],
"properties":{
"discoverConfig":{
"type":"object",
"description":"地址过滤配置",
"properties":{
"parseHTMLComments":{"type":"boolean","title":"是否查找注释中的地址"},
"parseScriptTags":{"type":"boolean","title":"是否查找script标签中的地址"},
"ignoreRobots":{"type":"boolean","title":"是否忽略机器人应答"},
"maxDepth":{"type":"number","title":"最大深度"},
"fetchWhitelistedMimeTypesBelowMaxDepth":{"type":"boolean","title":"是否开启最大深度过滤"},
"whitePathList":{"type":"array","title":"路径白名单","items":{
"type":"object"
"title":"单个路径规则",
"properties":{
"path":{
"type":"string",
"title":"规则，支持正则"
},
"enable":{
"type":"boolean",
"title":"是否开启"
}
}
}},
"allowedProtocols":{"type":"array","title":"支持的协议","items":{"type":"string"}}
}
},
"queueConfig":{
"type":"object",
"description":"域名过滤配置",
"properties":{
"ignoreWWWDomain":{"type":"boolean","title":"忽略www的域名"},
"stripWWWDomain":{"type":"boolean","title":"是否去掉www域名，使用根域名"},
"scanSubdomains":{"type":"boolean","title":"搜索子域名"},
"host":{"type":"string","title":"初始host"},
"initialProtocol":{"type":"string","title":"初始协议"},
"initialPort":{"type":"number","title":"初始端口",
"stripQuerystring":{"type":"boolean","title":"去掉地址栏后面的query参数"},
"allowQueryParams":{"type":"array","title":"过滤query参数，选择需要的","items":{"type":"string","title":"单个参数"}},
"domainWhiteList":{"type":"array","title":"域名白名单，支持正则","items":{"type":"string","title":"单个域名规则"}},
"filterByDomain":{"type":"boolean","title":"是否开启过滤域名白名单"}
}
},
"queueItem":{
"type":"object",
"description":"下载的页面的链接信息",
"required":["url","responseBody"],
"properties":{
"url":{"type":"string","title":"下载链接的详细地址"},
"path":{"type":"string","title":"下载链接的路径"},
"query":{"type":"string","title":"下载链接的地址栏参数信息"},
"protocol":{"type":"string","title":"下载链接的协议"},
"port":{"type":"number","title":"下载链接的端口"},
"hostname":{"type":"string","title":"下载链接的域名",
"depth":{"type":"number","title":"下载链接深度"},
"responseBody":{"type":"string","title":"html文档"}
}
}
}
}
测试数据
{
"queueConfig": {
"ignoreWWWDomain": false,
"stripWWWDomain": false,
"scanSubdomains": true,
"host": "www.yaolan.com",
"initialProtocol": "http",
"initialPort": 80,
"stripQuerystring": true,
"fetchConditions": [],
"domainWhiteList": ["(.*?).yaolan.com"],
"filterByDomain": true
},
"discoverConfig": {
"parseHTMLComments": false,
"parseScriptTags": false,
"allowedProtocols": ["http", "https"],
"whitePathList": [{ "path": "/(.*?)", "enable": true }],
"userAgent": "",
"fetchWhitelistedMimeTypesBelowMaxDepth": false,
"maxDepth": 0,
"ignoreRobots": true
},
"queueItem": {
"responseBody": "摇篮网首页的html代码",
"url": "http://www.yaolan.com"
}
}
partten: role:crawler.plugin.queue,cmd:queue
返回数据结构:
{
"type":"array",
"description":"返回的数据结构",
"items":{
"type":"object",
"properties":{
"url":{"type":"string","title":"下载链接的详细地址"},
"path":{"type":"string","title":"下载链接的路径"},
"query":{"type":"string","title":"下载链接的地址栏参数信息"},
"protocol":{"type":"string","title":"下载链接的协议"},
"port":{"type":"number","title":"下载链接的端口"},
"hostname":{"type":"string","title":"下载链接的域名",
"depth":{"type":"number","title":"下载链接深度"},
}
}
}
参数数据结构:
{
"type":"object",
"description":"参数",
"required":["queueItem"],
"properties":{
"queueConfig":{
"type":"object",
"description":"域名过滤配置",
"properties":{
"ignoreWWWDomain":{"type":"boolean","title":"忽略www的域名"},
"stripWWWDomain":{"type":"boolean","title":"是否去掉www域名，使用根域名"},
"scanSubdomains":{"type":"boolean","title":"搜索子域名"},
"host":{"type":"string","title":"初始host"},
"initialProtocol":{"type":"string","title":"初始协议"},
"initialPort":{"type":"number","title":"初始端口",
"stripQuerystring":{"type":"boolean","title":"去掉地址栏后面的query参数"},
"allowQueryParams":{"type":"array","title":"过滤query参数，选择需要的","items":{"type":"string","title":"单个参数"}},
"domainWhiteList":{"type":"array","title":"域名白名单，支持正则","items":{"type":"string","title":"单个域名规则"}},
"filterByDomain":{"type":"boolean","title":"是否开启过滤域名白名单"}
}
},
"urls":{
"type":"array",
"description":"下载的页面的链接信息",
"items":{
"type":"string",
"title":"链接"
}
}
}
}
测试数据
{
"queueConfig": {
"ignoreWWWDomain": false,
"stripWWWDomain": false,
"scanSubdomains": true,
"host": "www.yaolan.com",
"initialProtocol": "http",
"initialPort": 80,
"stripQuerystring": true,
"fetchConditions": [],
"domainWhiteList": ["(.*?).yaolan.com"],
"filterByDomain": true
},
"urls": ["http://www.yaolan.com","http://bbs.yaolan.com"]
}
FAQs
用于发现html文档中的地址链接
The npm package crawler.plugin.queue receives a total of 19 weekly downloads. As such, crawler.plugin.queue popularity was classified as not popular.
We found that crawler.plugin.queue demonstrated a not healthy version release cadence and project activity because the last version was released a year ago. It has 1 open source maintainer collaborating on the project.
Did you know?
Socket for GitHub automatically highlights issues in each pull request and monitors the health of all your open source dependencies. Discover the contents of your packages and block harmful activity before you install or update your dependencies.
Security News
Tea.xyz, a crypto project aimed at rewarding open source contributions, is once again facing backlash due to an influx of spam packages flooding public package registries.
Security News
As cyber threats become more autonomous, AI-powered defenses are crucial for businesses to stay ahead of attackers who can exploit software vulnerabilities at scale.
Security News
UnitedHealth Group disclosed that the ransomware attack on Change Healthcare compromised protected health information for millions in the U.S., with estimated costs to the company expected to reach $1 billion.