507 lines
12 KiB
JavaScript
507 lines
12 KiB
JavaScript
/**
|
|
* 模拟抓取校验脚本
|
|
* 自动遍历全站sitemap.xml，抓取全部公开页面，校验页面质量
|
|
*/
|
|
|
|
const https = require('https');
|
|
const http = require('http');
|
|
const { JSDOM } = require('jsdom');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
// 配置
|
|
const CONFIG = {
|
|
baseUrl: process.env.BASE_URL || 'http://localhost:3000',
|
|
timeout: 10000,
|
|
outputDir: path.join(__dirname, '../reports'),
|
|
};
|
|
|
|
// 测试结果
|
|
const results = {
|
|
totalPages: 0,
|
|
passedPages: 0,
|
|
failedPages: 0,
|
|
warnings: 0,
|
|
errors: [],
|
|
warningsList: [],
|
|
details: [],
|
|
};
|
|
|
|
/**
|
|
* 获取页面内容
|
|
*/
|
|
function fetchPage(url) {
|
|
return new Promise((resolve, reject) => {
|
|
const protocol = url.startsWith('https') ? https : http;
|
|
const timeout = CONFIG.timeout;
|
|
|
|
const req = protocol.get(url, { timeout }, (res) => {
|
|
let data = '';
|
|
|
|
res.on('data', (chunk) => {
|
|
data += chunk;
|
|
});
|
|
|
|
res.on('end', () => {
|
|
resolve({
|
|
status: res.statusCode,
|
|
headers: res.headers,
|
|
body: data,
|
|
url: url,
|
|
});
|
|
});
|
|
});
|
|
|
|
req.on('error', (err) => {
|
|
reject(err);
|
|
});
|
|
|
|
req.on('timeout', () => {
|
|
req.destroy();
|
|
reject(new Error('请求超时'));
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* 解析sitemap.xml
|
|
*/
|
|
async function parseSitemap() {
|
|
try {
|
|
const response = await fetchPage(`${CONFIG.baseUrl}/sitemap.xml`);
|
|
const dom = new JSDOM(response.body);
|
|
const urls = dom.window.document.querySelectorAll('url');
|
|
|
|
const sitemapUrls = [];
|
|
urls.forEach((url) => {
|
|
const loc = url.querySelector('loc');
|
|
if (loc) {
|
|
sitemapUrls.push(loc.textContent);
|
|
}
|
|
});
|
|
|
|
return sitemapUrls;
|
|
} catch (error) {
|
|
console.error('解析sitemap失败:', error.message);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 校验H1唯一性
|
|
*/
|
|
function validateH1(html, url) {
|
|
const dom = new JSDOM(html);
|
|
const h1Elements = dom.window.document.querySelectorAll('h1');
|
|
|
|
if (h1Elements.length === 0) {
|
|
return {
|
|
valid: false,
|
|
message: '缺少H1标签',
|
|
};
|
|
}
|
|
|
|
if (h1Elements.length > 1) {
|
|
return {
|
|
valid: false,
|
|
message: `存在${h1Elements.length}个H1标签，应唯一`,
|
|
};
|
|
}
|
|
|
|
return {
|
|
valid: true,
|
|
message: 'H1标签唯一',
|
|
content: h1Elements[0].textContent.trim(),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* 校验标题层级
|
|
*/
|
|
function validateHeadings(html, url) {
|
|
const dom = new JSDOM(html);
|
|
const headings = dom.window.document.querySelectorAll('h1, h2, h3, h4, h5, h6');
|
|
|
|
const headingLevels = [];
|
|
headings.forEach((h) => {
|
|
const level = parseInt(h.tagName.substring(1));
|
|
headingLevels.push(level);
|
|
});
|
|
|
|
// 检查层级是否连续
|
|
let previousLevel = 0;
|
|
let isValid = true;
|
|
let errorMessage = '';
|
|
|
|
for (let i = 0; i < headingLevels.length; i++) {
|
|
const currentLevel = headingLevels[i];
|
|
|
|
// H1应该在最前面
|
|
if (i === 0 && currentLevel !== 1) {
|
|
isValid = false;
|
|
errorMessage = '第一个标题应该是H1';
|
|
break;
|
|
}
|
|
|
|
// 检查层级跳跃
|
|
if (currentLevel > previousLevel + 1 && previousLevel !== 0) {
|
|
isValid = false;
|
|
errorMessage = `标题层级跳跃：从H${previousLevel}到H${currentLevel}`;
|
|
break;
|
|
}
|
|
|
|
previousLevel = currentLevel;
|
|
}
|
|
|
|
return {
|
|
valid: isValid,
|
|
message: errorMessage || '标题层级规范',
|
|
headings: headingLevels,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* 校验JSON-LD结构化数据
|
|
*/
|
|
function validateJsonLd(html, url) {
|
|
const dom = new JSDOM(html);
|
|
const scripts = dom.window.document.querySelectorAll('script[type="application/ld+json"]');
|
|
|
|
const jsonLdData = [];
|
|
let isValid = true;
|
|
let errorMessage = '';
|
|
|
|
scripts.forEach((script) => {
|
|
try {
|
|
const data = JSON.parse(script.textContent);
|
|
jsonLdData.push(data);
|
|
|
|
// 基本结构校验
|
|
if (!data['@context']) {
|
|
isValid = false;
|
|
errorMessage = 'JSON-LD缺少@context';
|
|
}
|
|
if (!data['@type']) {
|
|
isValid = false;
|
|
errorMessage = 'JSON-LD缺少@type';
|
|
}
|
|
} catch (error) {
|
|
isValid = false;
|
|
errorMessage = `JSON-LD语法错误: ${error.message}`;
|
|
}
|
|
});
|
|
|
|
return {
|
|
valid: isValid,
|
|
message: errorMessage || 'JSON-LD结构化数据合法',
|
|
count: jsonLdData.length,
|
|
data: jsonLdData,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* 校验内部链接
|
|
*/
|
|
function validateInternalLinks(html, baseUrl) {
|
|
const dom = new JSDOM(html);
|
|
const links = dom.window.document.querySelectorAll('a[href]');
|
|
|
|
const internalLinks = [];
|
|
const brokenLinks = [];
|
|
|
|
links.forEach((link) => {
|
|
const href = link.getAttribute('href');
|
|
|
|
// 跳过外部链接和锚点
|
|
if (href.startsWith('http') || href.startsWith('#') || href.startsWith('mailto:')) {
|
|
return;
|
|
}
|
|
|
|
// 构建完整URL
|
|
let fullUrl;
|
|
if (href.startsWith('/')) {
|
|
fullUrl = `${CONFIG.baseUrl}${href}`;
|
|
} else {
|
|
fullUrl = `${CONFIG.baseUrl}/${href}`;
|
|
}
|
|
|
|
internalLinks.push({
|
|
text: link.textContent.trim(),
|
|
href: href,
|
|
fullUrl: fullUrl,
|
|
});
|
|
});
|
|
|
|
return {
|
|
valid: true,
|
|
message: `发现${internalLinks.length}个内部链接`,
|
|
links: internalLinks,
|
|
brokenLinks: brokenLinks,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* 校验单个页面
|
|
*/
|
|
async function validatePage(url) {
|
|
const pageResult = {
|
|
url: url,
|
|
status: 'passed',
|
|
checks: [],
|
|
errors: [],
|
|
warnings: [],
|
|
};
|
|
|
|
try {
|
|
// 获取页面
|
|
const response = await fetchPage(url);
|
|
|
|
// 校验HTTP状态码
|
|
if (response.status !== 200) {
|
|
pageResult.status = 'failed';
|
|
pageResult.errors.push(`HTTP状态码异常: ${response.status}`);
|
|
results.errors.push({
|
|
url: url,
|
|
error: `HTTP状态码异常: ${response.status}`,
|
|
});
|
|
}
|
|
|
|
pageResult.checks.push({
|
|
name: 'HTTP状态码',
|
|
status: response.status === 200 ? 'passed' : 'failed',
|
|
message: `状态码: ${response.status}`,
|
|
});
|
|
|
|
// 校验H1唯一性
|
|
const h1Result = validateH1(response.body, url);
|
|
pageResult.checks.push({
|
|
name: 'H1唯一性',
|
|
status: h1Result.valid ? 'passed' : 'failed',
|
|
message: h1Result.message,
|
|
content: h1Result.content,
|
|
});
|
|
|
|
if (!h1Result.valid) {
|
|
pageResult.status = 'failed';
|
|
pageResult.errors.push(h1Result.message);
|
|
results.errors.push({
|
|
url: url,
|
|
error: h1Result.message,
|
|
});
|
|
}
|
|
|
|
// 校验标题层级
|
|
const headingsResult = validateHeadings(response.body, url);
|
|
pageResult.checks.push({
|
|
name: '标题层级',
|
|
status: headingsResult.valid ? 'passed' : 'failed',
|
|
message: headingsResult.message,
|
|
});
|
|
|
|
if (!headingsResult.valid) {
|
|
pageResult.status = 'failed';
|
|
pageResult.errors.push(headingsResult.message);
|
|
results.errors.push({
|
|
url: url,
|
|
error: headingsResult.message,
|
|
});
|
|
}
|
|
|
|
// 校验JSON-LD
|
|
const jsonLdResult = validateJsonLd(response.body, url);
|
|
pageResult.checks.push({
|
|
name: 'JSON-LD结构化数据',
|
|
status: jsonLdResult.valid ? 'passed' : 'failed',
|
|
message: jsonLdResult.message,
|
|
count: jsonLdResult.count,
|
|
});
|
|
|
|
if (!jsonLdResult.valid) {
|
|
pageResult.status = 'failed';
|
|
pageResult.errors.push(jsonLdResult.message);
|
|
results.errors.push({
|
|
url: url,
|
|
error: jsonLdResult.message,
|
|
});
|
|
}
|
|
|
|
// 校验内部链接
|
|
const linksResult = validateInternalLinks(response.body, url);
|
|
pageResult.checks.push({
|
|
name: '内部链接',
|
|
status: linksResult.valid ? 'passed' : 'failed',
|
|
message: linksResult.message,
|
|
linkCount: linksResult.links.length,
|
|
});
|
|
|
|
} catch (error) {
|
|
pageResult.status = 'failed';
|
|
pageResult.errors.push(`页面抓取失败: ${error.message}`);
|
|
results.errors.push({
|
|
url: url,
|
|
error: `页面抓取失败: ${error.message}`,
|
|
});
|
|
}
|
|
|
|
return pageResult;
|
|
}
|
|
|
|
/**
|
|
* 生成测试报告
|
|
*/
|
|
function generateReport() {
|
|
// 创建报告目录
|
|
if (!fs.existsSync(CONFIG.outputDir)) {
|
|
fs.mkdirSync(CONFIG.outputDir, { recursive: true });
|
|
}
|
|
|
|
const reportPath = path.join(CONFIG.outputDir, 'crawler_report.json');
|
|
const reportContent = {
|
|
summary: {
|
|
totalTime: new Date().toISOString(),
|
|
baseUrl: CONFIG.baseUrl,
|
|
totalPages: results.totalPages,
|
|
passedPages: results.passedPages,
|
|
failedPages: results.failedPages,
|
|
warnings: results.warnings,
|
|
passRate: ((results.passedPages / results.totalPages) * 100).toFixed(2) + '%',
|
|
},
|
|
errors: results.errors,
|
|
warnings: results.warningsList,
|
|
details: results.details,
|
|
};
|
|
|
|
fs.writeFileSync(reportPath, JSON.stringify(reportContent, null, 2));
|
|
|
|
// 生成可读报告
|
|
const readableReportPath = path.join(CONFIG.outputDir, 'crawler_report.md');
|
|
let readableReport = `# 爬虫校验报告
|
|
|
|
生成时间: ${new Date().toISOString()}
|
|
|
|
## 摘要
|
|
|
|
- 总页面数: ${results.totalPages}
|
|
- 通过页面: ${results.passedPages}
|
|
- 失败页面: ${results.failedPages}
|
|
- 警告: ${results.warnings}
|
|
- 通过率: ${((results.passedPages / results.totalPages) * 100).toFixed(2)}%
|
|
|
|
## 失败详情
|
|
|
|
`;
|
|
|
|
if (results.errors.length === 0) {
|
|
readableReport += '无失败项\n\n';
|
|
} else {
|
|
results.errors.forEach((error, index) => {
|
|
readableReport += `${index + 1}. ${error.url}\n - ${error.error}\n\n`;
|
|
});
|
|
}
|
|
|
|
readableReport += `## 页面详情
|
|
|
|
`;
|
|
|
|
results.details.forEach((detail) => {
|
|
readableReport += `### ${detail.url}
|
|
|
|
状态: ${detail.status}
|
|
|
|
`;
|
|
detail.checks.forEach((check) => {
|
|
readableReport += `- ${check.name}: ${check.status} - ${check.message}\n`;
|
|
});
|
|
readableReport += '\n';
|
|
});
|
|
|
|
fs.writeFileSync(readableReportPath, readableReport);
|
|
|
|
console.log(`报告已生成:`);
|
|
console.log(` - JSON报告: ${reportPath}`);
|
|
console.log(` - 可读报告: ${readableReportPath}`);
|
|
}
|
|
|
|
/**
|
|
* 主函数
|
|
*/
|
|
async function main() {
|
|
console.log('开始爬虫校验...');
|
|
console.log(`目标URL: ${CONFIG.baseUrl}`);
|
|
|
|
// 获取sitemap
|
|
console.log('正在解析sitemap...');
|
|
const sitemapUrls = await parseSitemap();
|
|
|
|
if (sitemapUrls.length === 0) {
|
|
console.log('未找到sitemap，使用默认页面列表');
|
|
// 默认页面列表
|
|
sitemapUrls.push(
|
|
`${CONFIG.baseUrl}/`,
|
|
`${CONFIG.baseUrl}/about`,
|
|
`${CONFIG.baseUrl}/evolution`,
|
|
`${CONFIG.baseUrl}/evolution/method`,
|
|
`${CONFIG.baseUrl}/value`,
|
|
`${CONFIG.baseUrl}/value/creation`,
|
|
`${CONFIG.baseUrl}/value/transfer`,
|
|
`${CONFIG.baseUrl}/value/delivery`,
|
|
`${CONFIG.baseUrl}/value/support`,
|
|
`${CONFIG.baseUrl}/amb`,
|
|
`${CONFIG.baseUrl}/amb/a`,
|
|
`${CONFIG.baseUrl}/amb/m`,
|
|
`${CONFIG.baseUrl}/amb/b`,
|
|
`${CONFIG.baseUrl}/capabilities`,
|
|
`${CONFIG.baseUrl}/cases`,
|
|
`${CONFIG.baseUrl}/knowledge`,
|
|
`${CONFIG.baseUrl}/concepts`,
|
|
`${CONFIG.baseUrl}/faq`,
|
|
`${CONFIG.baseUrl}/match`,
|
|
`${CONFIG.baseUrl}/contact`
|
|
);
|
|
}
|
|
|
|
console.log(`共发现${sitemapUrls.length}个页面`);
|
|
|
|
// 逐个校验页面
|
|
for (let i = 0; i < sitemapUrls.length; i++) {
|
|
const url = sitemapUrls[i];
|
|
console.log(`[${i + 1}/${sitemapUrls.length}] 正在校验: ${url}`);
|
|
|
|
const pageResult = await validatePage(url);
|
|
results.details.push(pageResult);
|
|
results.totalPages++;
|
|
|
|
if (pageResult.status === 'passed') {
|
|
results.passedPages++;
|
|
console.log(` ✓ 通过`);
|
|
} else {
|
|
results.failedPages++;
|
|
console.log(` ✗ 失败`);
|
|
pageResult.errors.forEach((error) => {
|
|
console.log(` - ${error}`);
|
|
});
|
|
}
|
|
}
|
|
|
|
// 生成报告
|
|
console.log('\n正在生成报告...');
|
|
generateReport();
|
|
|
|
console.log('\n校验完成!');
|
|
console.log(`通过率: ${((results.passedPages / results.totalPages) * 100).toFixed(2)}%`);
|
|
|
|
// 返回结果
|
|
return results;
|
|
}
|
|
|
|
// 如果直接运行此脚本
|
|
if (require.main === module) {
|
|
main().catch(console.error);
|
|
}
|
|
|
|
module.exports = {
|
|
main,
|
|
validatePage,
|
|
parseSitemap,
|
|
CONFIG,
|
|
}; |