chore: prepare project for server deployment

2026-09-09 17:43:07 +08:00
commit d0bcff2311
83 changed files with 24163 additions and 0 deletions
@@ -0,0 +1,507 @@
+/**
+ * 模拟抓取校验脚本
+ * 自动遍历全站sitemap.xml，抓取全部公开页面，校验页面质量
+ */
+
+const https = require('https');
+const http = require('http');
+const { JSDOM } = require('jsdom');
+const fs = require('fs');
+const path = require('path');
+
+// 配置
+const CONFIG = {
+  baseUrl: process.env.BASE_URL || 'http://localhost:3000',
+  timeout: 10000,
+  outputDir: path.join(__dirname, '../reports'),
+};
+
+// 测试结果
+const results = {
+  totalPages: 0,
+  passedPages: 0,
+  failedPages: 0,
+  warnings: 0,
+  errors: [],
+  warningsList: [],
+  details: [],
+};
+
+/**
+ * 获取页面内容
+ */
+function fetchPage(url) {
+  return new Promise((resolve, reject) => {
+    const protocol = url.startsWith('https') ? https : http;
+    const timeout = CONFIG.timeout;
+    
+    const req = protocol.get(url, { timeout }, (res) => {
+      let data = '';
+      
+      res.on('data', (chunk) => {
+        data += chunk;
+      });
+      
+      res.on('end', () => {
+        resolve({
+          status: res.statusCode,
+          headers: res.headers,
+          body: data,
+          url: url,
+        });
+      });
+    });
+    
+    req.on('error', (err) => {
+      reject(err);
+    });
+    
+    req.on('timeout', () => {
+      req.destroy();
+      reject(new Error('请求超时'));
+    });
+  });
+}
+
+/**
+ * 解析sitemap.xml
+ */
+async function parseSitemap() {
+  try {
+    const response = await fetchPage(`${CONFIG.baseUrl}/sitemap.xml`);
+    const dom = new JSDOM(response.body);
+    const urls = dom.window.document.querySelectorAll('url');
+    
+    const sitemapUrls = [];
+    urls.forEach((url) => {
+      const loc = url.querySelector('loc');
+      if (loc) {
+        sitemapUrls.push(loc.textContent);
+      }
+    });
+    
+    return sitemapUrls;
+  } catch (error) {
+    console.error('解析sitemap失败:', error.message);
+    return [];
+  }
+}
+
+/**
+ * 校验H1唯一性
+ */
+function validateH1(html, url) {
+  const dom = new JSDOM(html);
+  const h1Elements = dom.window.document.querySelectorAll('h1');
+  
+  if (h1Elements.length === 0) {
+    return {
+      valid: false,
+      message: '缺少H1标签',
+    };
+  }
+  
+  if (h1Elements.length > 1) {
+    return {
+      valid: false,
+      message: `存在${h1Elements.length}个H1标签，应唯一`,
+    };
+  }
+  
+  return {
+    valid: true,
+    message: 'H1标签唯一',
+    content: h1Elements[0].textContent.trim(),
+  };
+}
+
+/**
+ * 校验标题层级
+ */
+function validateHeadings(html, url) {
+  const dom = new JSDOM(html);
+  const headings = dom.window.document.querySelectorAll('h1, h2, h3, h4, h5, h6');
+  
+  const headingLevels = [];
+  headings.forEach((h) => {
+    const level = parseInt(h.tagName.substring(1));
+    headingLevels.push(level);
+  });
+  
+  // 检查层级是否连续
+  let previousLevel = 0;
+  let isValid = true;
+  let errorMessage = '';
+  
+  for (let i = 0; i < headingLevels.length; i++) {
+    const currentLevel = headingLevels[i];
+    
+    // H1应该在最前面
+    if (i === 0 && currentLevel !== 1) {
+      isValid = false;
+      errorMessage = '第一个标题应该是H1';
+      break;
+    }
+    
+    // 检查层级跳跃
+    if (currentLevel > previousLevel + 1 && previousLevel !== 0) {
+      isValid = false;
+      errorMessage = `标题层级跳跃：从H${previousLevel}到H${currentLevel}`;
+      break;
+    }
+    
+    previousLevel = currentLevel;
+  }
+  
+  return {
+    valid: isValid,
+    message: errorMessage || '标题层级规范',
+    headings: headingLevels,
+  };
+}
+
+/**
+ * 校验JSON-LD结构化数据
+ */
+function validateJsonLd(html, url) {
+  const dom = new JSDOM(html);
+  const scripts = dom.window.document.querySelectorAll('script[type="application/ld+json"]');
+  
+  const jsonLdData = [];
+  let isValid = true;
+  let errorMessage = '';
+  
+  scripts.forEach((script) => {
+    try {
+      const data = JSON.parse(script.textContent);
+      jsonLdData.push(data);
+      
+      // 基本结构校验
+      if (!data['@context']) {
+        isValid = false;
+        errorMessage = 'JSON-LD缺少@context';
+      }
+      if (!data['@type']) {
+        isValid = false;
+        errorMessage = 'JSON-LD缺少@type';
+      }
+    } catch (error) {
+      isValid = false;
+      errorMessage = `JSON-LD语法错误: ${error.message}`;
+    }
+  });
+  
+  return {
+    valid: isValid,
+    message: errorMessage || 'JSON-LD结构化数据合法',
+    count: jsonLdData.length,
+    data: jsonLdData,
+  };
+}
+
+/**
+ * 校验内部链接
+ */
+function validateInternalLinks(html, baseUrl) {
+  const dom = new JSDOM(html);
+  const links = dom.window.document.querySelectorAll('a[href]');
+  
+  const internalLinks = [];
+  const brokenLinks = [];
+  
+  links.forEach((link) => {
+    const href = link.getAttribute('href');
+    
+    // 跳过外部链接和锚点
+    if (href.startsWith('http') || href.startsWith('#') || href.startsWith('mailto:')) {
+      return;
+    }
+    
+    // 构建完整URL
+    let fullUrl;
+    if (href.startsWith('/')) {
+      fullUrl = `${CONFIG.baseUrl}${href}`;
+    } else {
+      fullUrl = `${CONFIG.baseUrl}/${href}`;
+    }
+    
+    internalLinks.push({
+      text: link.textContent.trim(),
+      href: href,
+      fullUrl: fullUrl,
+    });
+  });
+  
+  return {
+    valid: true,
+    message: `发现${internalLinks.length}个内部链接`,
+    links: internalLinks,
+    brokenLinks: brokenLinks,
+  };
+}
+
+/**
+ * 校验单个页面
+ */
+async function validatePage(url) {
+  const pageResult = {
+    url: url,
+    status: 'passed',
+    checks: [],
+    errors: [],
+    warnings: [],
+  };
+  
+  try {
+    // 获取页面
+    const response = await fetchPage(url);
+    
+    // 校验HTTP状态码
+    if (response.status !== 200) {
+      pageResult.status = 'failed';
+      pageResult.errors.push(`HTTP状态码异常: ${response.status}`);
+      results.errors.push({
+        url: url,
+        error: `HTTP状态码异常: ${response.status}`,
+      });
+    }
+    
+    pageResult.checks.push({
+      name: 'HTTP状态码',
+      status: response.status === 200 ? 'passed' : 'failed',
+      message: `状态码: ${response.status}`,
+    });
+    
+    // 校验H1唯一性
+    const h1Result = validateH1(response.body, url);
+    pageResult.checks.push({
+      name: 'H1唯一性',
+      status: h1Result.valid ? 'passed' : 'failed',
+      message: h1Result.message,
+      content: h1Result.content,
+    });
+    
+    if (!h1Result.valid) {
+      pageResult.status = 'failed';
+      pageResult.errors.push(h1Result.message);
+      results.errors.push({
+        url: url,
+        error: h1Result.message,
+      });
+    }
+    
+    // 校验标题层级
+    const headingsResult = validateHeadings(response.body, url);
+    pageResult.checks.push({
+      name: '标题层级',
+      status: headingsResult.valid ? 'passed' : 'failed',
+      message: headingsResult.message,
+    });
+    
+    if (!headingsResult.valid) {
+      pageResult.status = 'failed';
+      pageResult.errors.push(headingsResult.message);
+      results.errors.push({
+        url: url,
+        error: headingsResult.message,
+      });
+    }
+    
+    // 校验JSON-LD
+    const jsonLdResult = validateJsonLd(response.body, url);
+    pageResult.checks.push({
+      name: 'JSON-LD结构化数据',
+      status: jsonLdResult.valid ? 'passed' : 'failed',
+      message: jsonLdResult.message,
+      count: jsonLdResult.count,
+    });
+    
+    if (!jsonLdResult.valid) {
+      pageResult.status = 'failed';
+      pageResult.errors.push(jsonLdResult.message);
+      results.errors.push({
+        url: url,
+        error: jsonLdResult.message,
+      });
+    }
+    
+    // 校验内部链接
+    const linksResult = validateInternalLinks(response.body, url);
+    pageResult.checks.push({
+      name: '内部链接',
+      status: linksResult.valid ? 'passed' : 'failed',
+      message: linksResult.message,
+      linkCount: linksResult.links.length,
+    });
+    
+  } catch (error) {
+    pageResult.status = 'failed';
+    pageResult.errors.push(`页面抓取失败: ${error.message}`);
+    results.errors.push({
+      url: url,
+      error: `页面抓取失败: ${error.message}`,
+    });
+  }
+  
+  return pageResult;
+}
+
+/**
+ * 生成测试报告
+ */
+function generateReport() {
+  // 创建报告目录
+  if (!fs.existsSync(CONFIG.outputDir)) {
+    fs.mkdirSync(CONFIG.outputDir, { recursive: true });
+  }
+  
+  const reportPath = path.join(CONFIG.outputDir, 'crawler_report.json');
+  const reportContent = {
+    summary: {
+      totalTime: new Date().toISOString(),
+      baseUrl: CONFIG.baseUrl,
+      totalPages: results.totalPages,
+      passedPages: results.passedPages,
+      failedPages: results.failedPages,
+      warnings: results.warnings,
+      passRate: ((results.passedPages / results.totalPages) * 100).toFixed(2) + '%',
+    },
+    errors: results.errors,
+    warnings: results.warningsList,
+    details: results.details,
+  };
+  
+  fs.writeFileSync(reportPath, JSON.stringify(reportContent, null, 2));
+  
+  // 生成可读报告
+  const readableReportPath = path.join(CONFIG.outputDir, 'crawler_report.md');
+  let readableReport = `# 爬虫校验报告
+
+生成时间: ${new Date().toISOString()}
+
+## 摘要
+
+- 总页面数: ${results.totalPages}
+- 通过页面: ${results.passedPages}
+- 失败页面: ${results.failedPages}
+- 警告: ${results.warnings}
+- 通过率: ${((results.passedPages / results.totalPages) * 100).toFixed(2)}%
+
+## 失败详情
+
+`;
+  
+  if (results.errors.length === 0) {
+    readableReport += '无失败项\n\n';
+  } else {
+    results.errors.forEach((error, index) => {
+      readableReport += `${index + 1}. ${error.url}\n   - ${error.error}\n\n`;
+    });
+  }
+  
+  readableReport += `## 页面详情
+
+`;
+  
+  results.details.forEach((detail) => {
+    readableReport += `### ${detail.url}
+
+状态: ${detail.status}
+
+`;
+    detail.checks.forEach((check) => {
+      readableReport += `- ${check.name}: ${check.status} - ${check.message}\n`;
+    });
+    readableReport += '\n';
+  });
+  
+  fs.writeFileSync(readableReportPath, readableReport);
+  
+  console.log(`报告已生成:`);
+  console.log(`  - JSON报告: ${reportPath}`);
+  console.log(`  - 可读报告: ${readableReportPath}`);
+}
+
+/**
+ * 主函数
+ */
+async function main() {
+  console.log('开始爬虫校验...');
+  console.log(`目标URL: ${CONFIG.baseUrl}`);
+  
+  // 获取sitemap
+  console.log('正在解析sitemap...');
+  const sitemapUrls = await parseSitemap();
+  
+  if (sitemapUrls.length === 0) {
+    console.log('未找到sitemap，使用默认页面列表');
+    // 默认页面列表
+    sitemapUrls.push(
+      `${CONFIG.baseUrl}/`,
+      `${CONFIG.baseUrl}/about`,
+      `${CONFIG.baseUrl}/evolution`,
+      `${CONFIG.baseUrl}/evolution/method`,
+      `${CONFIG.baseUrl}/value`,
+      `${CONFIG.baseUrl}/value/creation`,
+      `${CONFIG.baseUrl}/value/transfer`,
+      `${CONFIG.baseUrl}/value/delivery`,
+      `${CONFIG.baseUrl}/value/support`,
+      `${CONFIG.baseUrl}/amb`,
+      `${CONFIG.baseUrl}/amb/a`,
+      `${CONFIG.baseUrl}/amb/m`,
+      `${CONFIG.baseUrl}/amb/b`,
+      `${CONFIG.baseUrl}/capabilities`,
+      `${CONFIG.baseUrl}/cases`,
+      `${CONFIG.baseUrl}/knowledge`,
+      `${CONFIG.baseUrl}/concepts`,
+      `${CONFIG.baseUrl}/faq`,
+      `${CONFIG.baseUrl}/match`,
+      `${CONFIG.baseUrl}/contact`
+    );
+  }
+  
+  console.log(`共发现${sitemapUrls.length}个页面`);
+  
+  // 逐个校验页面
+  for (let i = 0; i < sitemapUrls.length; i++) {
+    const url = sitemapUrls[i];
+    console.log(`[${i + 1}/${sitemapUrls.length}] 正在校验: ${url}`);
+    
+    const pageResult = await validatePage(url);
+    results.details.push(pageResult);
+    results.totalPages++;
+    
+    if (pageResult.status === 'passed') {
+      results.passedPages++;
+      console.log(`  ✓ 通过`);
+    } else {
+      results.failedPages++;
+      console.log(`  ✗ 失败`);
+      pageResult.errors.forEach((error) => {
+        console.log(`    - ${error}`);
+      });
+    }
+  }
+  
+  // 生成报告
+  console.log('\n正在生成报告...');
+  generateReport();
+  
+  console.log('\n校验完成!');
+  console.log(`通过率: ${((results.passedPages / results.totalPages) * 100).toFixed(2)}%`);
+  
+  // 返回结果
+  return results;
+}
+
+// 如果直接运行此脚本
+if (require.main === module) {
+  main().catch(console.error);
+}
+
+module.exports = {
+  main,
+  validatePage,
+  parseSitemap,
+  CONFIG,
+};