EnlightenCode

node爬虫脚本puppeteer ​

通过youtube链接获取视频信息 ​

使用 YouTube Player API ​

首先，我们使用 YouTube Player API 来加载视频，这样我们就可以获取一些基本信息和控制视频播放。

js
<!DOCTYPE html>
<html>

<body>
  <!-- 1. The <iframe> (and video player) will replace this <div> tag. -->
  <div id="player"></div>

  <script>
    // 2. This code loads the IFrame Player API code asynchronously.
    var tag = document.createElement('script');
    tag.src = "https://www.youtube.com/iframe_api";
    var firstScriptTag = document.getElementsByTagName('script')[0];
    firstScriptTag.parentNode.insertBefore(tag, firstScriptTag);

    // 3. This function creates an <iframe> (and YouTube player)
    //    after the API code downloads.
    var player;
    function onYouTubeIframeAPIReady() {
      player = new YT.Player('player', {
        height: '360',
        width: '640',
        videoId: 'M7lc1UVf-VE',
        events: {
          'onReady': onPlayerReady,
          'onStateChange': onPlayerStateChange
        }
      });
    }

    // 4. The API will call this function when the video player is ready.
    function onPlayerReady(event) {
      event.target.playVideo();
      getVideoInfo();
    }

    // 5. The API calls this function when the player's state changes.
    function onPlayerStateChange(event) {
      if (event.data == YT.PlayerState.PLAYING) {
        // Video is playing
      }
    }

    function getVideoInfo() {
      // 获取视频信息
      var videoData = player.getVideoData();
      console.log("Video Data:", videoData);
      console.log("Title:", videoData.title);
      console.log("Video ID:", videoData.video_id);
      console.log("Author:", videoData.author);

      // 获取视频时长（秒）
      var duration = player.getDuration();
      console.log("Duration:", duration);

      // 获取缩略图
      var videoId = player.getVideoData().video_id;
      var thumbnailUrl = `https://img.youtube.com/vi/${videoId}/0.jpg`;
      console.log("Thumbnail URL:", thumbnailUrl);
    }
  </script>
</body>

</html>

使用 oEmbed API 获取额外信息 ​

YouTube Player API 无法提供所有你需要的信息，比如作者头像和视频简介。我们可以结合使用 oEmbed API 来获取一些额外信息。

js
  async function getVideoInfo(url) {
    const oembedUrl = `https://www.youtube.com/oembed?url=${encodeURIComponent("https://www.youtube.com/watch?v=Af4CR16VP2I&t=217s")}&format=json`;
    try {
      const response = await fetch(oembedUrl);
      const data = await response.json();
      console.log(data);
      // 处理视频信息
    } catch (error) {
      console.error('Error:', error);
    }
  }

使用Youtube Data API ​

Youtube Data API 可以获取更多的信息，但是需要获取API密钥，并且有每天10000的配额限制。

js
  <script src="https://apis.google.com/js/api.js"></script>

 // 替换为你的 API key
      const API_KEY = '';

      // 替换为你想获取信息的视频 ID
      const VIDEO_ID = '';

      // 加载 YouTube API 客户端库
      const loadClient = () => {
        return new Promise((resolve, reject) => {
          gapi.load('client', () => {
            gapi.client.setApiKey(API_KEY);
            gapi.client.load('youtube', 'v3')
              .then(() => resolve())
              .catch(err => reject('Error loading YouTube API:', err));
          });
        });
      }

      // 执行 API 请求
      const executeRequest = () => {
        return gapi.client.youtube.videos.list({
          "part": [
            "snippet,contentDetails,statistics"
          ],
          "id": VIDEO_ID
        });
      }

      // 主函数
      const getVideoInfo = async () => {
        try {
          await loadClient();
          const response = await executeRequest();
          console.log('Video info:', response.result.items[0]);
          // 处理获取到的视频信息
          const videoInfo = response.result.items[0];
          // 例如，获取视频标题
          console.log('Video title:', videoInfo.snippet.title);
          // 获取缩略图 URL
          console.log('Thumbnail URL:', videoInfo.snippet.thumbnails.default.url);
        } catch (err) {
          console.error('Error:', err);
        }
      }

      // 调用主函数
      getVideoInfo();

node爬虫脚本puppeteer ​

使用脚本来爬取youtube视频链接的部分信息 ​

结合node-fetch获取信息

js
const fetch = require('node-fetch');
const puppeteer = require('puppeteer');
const fs = require('fs');

function extractUsernameFromUrl(url) {
  const match = url.match(/\/user\/([^/]+)$/);
  return match ? match[1] : '';
}
async function scrapeData(urls) {
  const browser = await puppeteer.launch({
    headless: "new",
    args: ['--no-sandbox', '--disable-setuid-sandbox']
  });

  const results = [];

  try {
    for (const url of urls) {
      const page = await browser.newPage();
      await page.setViewport({ width: 1280, height: 800 });

      await page.goto(url, { waitUntil: 'networkidle2' });
      await page.waitForSelector('body');

      const extractedData = await page.evaluate((url) => {
        const ytInitialData = JSON.parse(document.body.innerHTML.match(/var ytInitialData = (.+?);<\/script>/)[1]);

        const videoId = new URL(url).searchParams.get('v');

        const videoTitle = ytInitialData.playerOverlays?.playerOverlayRenderer?.videoDetails?.playerOverlayVideoDetailsRenderer?.title?.simpleText;

        let videoDescription = '';
        let authorAvatar = '';
        let authorName = '';
        let authorEmail = '';

        const contents = ytInitialData.contents?.twoColumnWatchNextResults?.results?.results?.contents;
        if (contents) {
          const data = contents.find(item => item.videoSecondaryInfoRenderer);
          if (data) {
            const videoSecondaryInfoRenderer = data.videoSecondaryInfoRenderer;
            videoDescription = videoSecondaryInfoRenderer.attributedDescription?.content || '';
            authorAvatar = videoSecondaryInfoRenderer.owner?.videoOwnerRenderer?.thumbnail?.thumbnails?.[0]?.url || '';
            authorName = videoSecondaryInfoRenderer.owner?.videoOwnerRenderer?.title?.runs?.[0]?.text || '';
            authorEmail = videoSecondaryInfoRenderer.owner?.videoOwnerRenderer?.title?.runs?.[0]?.text?.navigationEndpoint?.commandMetadata?.webCommandMetadata?.url || '';
            thumbnailUrl = `https://img.youtube.com/vi/${videoId}/0.jpg`
          }
        }

        return {
          videoId,
          videoTitle,
          videoDescription,
          authorAvatar,
          authorName,
          authorEmail,
          thumbnailUrl
        };
      }, url);

      if (extractedData) {
        console.log(`Successfully extracted data for video ID: ${extractedData.videoId}`);

        // 调用 oEmbed API
        const oembedUrl = `https://www.youtube.com/oembed?url=${encodeURIComponent(url)}&format=json`;
        try {
          const response = await fetch(oembedUrl);
          const oembedData = await response.json();

          oembedData.id = extractedData.videoId;
          const email = extractUsernameFromUrl(oembedData.author_url);
          oembedData.email = "@" + email;

          // 合并数据
          const mergedData = {
            ...extractedData,
            ...oembedData,
            thumbnail_url: oembedData.thumbnail_url,
            title: oembedData.title,
            author_url: oembedData.author_url
          };

          results.push(mergedData);
        } catch (error) {
          console.error('Error fetching oEmbed data:', error);
          results.push(extractedData);
        }
      } else {
        console.log(`Failed to extract data for URL: ${url}`);
      }

      await page.close();
    }

    // 将所有结果保存为 JSON 文件
    fs.writeFileSync('extractedVideosData.json', JSON.stringify(results, null, 2));
    console.log('All data saved to extractedVideosData.json');

  } catch (error) {
    console.error('Error scraping data:', error);
  } finally {
    await browser.close();
  }

  return results;
}

const urls = [
  'https://www.youtube.com/watch?v=HpVOs5imUN0',
  'https://www.youtube.com/watch?v=MFRbfY0cZFo',
  'https://www.youtube.com/watch?v=wU9ouVuT7CQ',
  'https://www.youtube.com/watch?v=l1PQ2d8DfHo',
  'https://www.youtube.com/watch?v=YvB8wk_wJ6E',
  'https://www.youtube.com/watch?v=tpFEainx6Ak',
  'https://www.youtube.com/watch?v=h3J1os_YfiI',
  'https://www.youtube.com/watch?v=2WPdvIl3yYg',
  'https://www.youtube.com/watch?v=TZVioAWixJ4',
];

scrapeData(urls).then(results => {
  console.log(`Scraped data for ${results.length} videos`);
});

使用promise来处理并发的问题，提高爬取速度。

js
const puppeteer = require('puppeteer');
const fs = require('fs');

async function scrapeData(urls) {
  const browser = await puppeteer.launch({
    headless: "new",
    args: ['--no-sandbox', '--disable-setuid-sandbox']
  });

  const results = [];

  try {
    const pages = await Promise.all(urls.map(async (url) => {
      const page = await browser.newPage();
      await page.setViewport({ width: 1280, height: 800 });

      await page.goto(url, { waitUntil: 'networkidle2' });
      await page.waitForSelector('body');

      return {
        page,
        url
      };
    }));

    const extractedDataPromises = pages.map(async ({ page, url }) => {
      const extractedData = await page.evaluate((url) => {
        const ytInitialData = JSON.parse(document.body.innerHTML.match(/var ytInitialData = (.+?);<\/script>/)[1]);

        const videoId = new URL(url).searchParams.get('v');

        const videoTitle = ytInitialData.playerOverlays?.playerOverlayRenderer?.videoDetails?.playerOverlayVideoDetailsRenderer?.title?.simpleText;

        let videoDescription = '';
        let authorAvatar = '';
        let authorName = '';
        let authorEmail = '';

        const contents = ytInitialData.contents?.twoColumnWatchNextResults?.results?.results?.contents;
        if (contents) {
          const data = contents.find(item => item.videoSecondaryInfoRenderer);
          if (data) {
            const videoSecondaryInfoRenderer = data.videoSecondaryInfoRenderer;
            videoDescription = videoSecondaryInfoRenderer.attributedDescription?.content || '';
            authorAvatar = videoSecondaryInfoRenderer.owner?.videoOwnerRenderer?.thumbnail?.thumbnails?.[0]?.url || '';
            authorName = videoSecondaryInfoRenderer.owner?.videoOwnerRenderer?.title?.runs?.[0]?.text || '';
            authorEmail = videoSecondaryInfoRenderer.owner?.videoOwnerRenderer?.title?.runs?.[0]?.navigationEndpoint?.commandMetadata?.webCommandMetadata?.url || '';
          }
        }

        return {
          videoId,
          videoTitle,
          videoDescription,
          authorAvatar,
          authorName,
          authorEmail,
          url
        };
      }, url);

      return extractedData;
    });

    const extractedData = await Promise.all(extractedDataPromises);

    // 将数据按原始 URL 顺序存储到 results 数组中
    for (let i = 0; i < pages.length; i++) {
      if (extractedData[i]) {
        results.push(extractedData[i]);
      } else {
        console.log(`Failed to extract data for URL: ${pages[i].url}`);
      }
      await pages[i].page.close();
    }

    // 将所有结果保存为 JSON 文件
    fs.writeFileSync('extractedVideosData.json', JSON.stringify(results, null, 2));
    console.log('All data saved to extractedVideosData.json');

  } catch (error) {
    console.error('Error scraping data:', error);
  } finally {
    await browser.close();
  }

  return results;
}

const urls = [
  'https://www.youtube.com/watch?v=HpVOs5imUN0',
  'https://www.youtube.com/watch?v=MFRbfY0cZFo',
  'https://www.youtube.com/watch?v=wU9ouVuT7CQ',
  'https://www.youtube.com/watch?v=l1PQ2d8DfHo',
  'https://www.youtube.com/watch?v=YvB8wk_wJ6E',
  'https://www.youtube.com/watch?v=tpFEainx6Ak',
  'https://www.youtube.com/watch?v=h3J1os_YfiI',
  'https://www.youtube.com/watch?v=2WPdvIl3yYg',
  'https://www.youtube.com/watch?v=TZVioAWixJ4',
];

scrapeData(urls).then(results => {
  console.log(`Scraped data for ${results.length} videos`);
});

最终实现：多平台爬取并排序（部分需要下载图片到本地） ​

js
const path = require('path');
const fs = require('fs');
const puppeteer = require('puppeteer');
const axios = require('axios');
const crypto = require('crypto');
const sharp = require('sharp');

const storyDataDirectory = path.join(__dirname, 'public', 'assets', 'storydata');
clearDirectory(storyDataDirectory);
async function scrapeData(urlList) {
  const browser = await puppeteer.launch({
    headless: "new",
    args: ['--no-sandbox', '--disable-setuid-sandbox']
  });

  const results = [];


  try {
    const pages = await Promise.all(urlList.map(async (url) => {
      const page = await browser.newPage();
      await page.setViewport({ width: 1280, height: 800 });

      try {
        await page.goto(url, {
          waitUntil: 'networkidle2',
          timeout: 60000
        });
        await page.waitForSelector('body', { timeout: 60000 });
      } catch (error) {
        console.error(`Error loading page ${url}: ${error.message}`);
        return null; // 页面加载失败
      }

      return {
        page,
        url,
      };
    }));

    const extractedDataPromises = pages
      .filter(page => page !== null) // 过滤掉加载失败的页面
      .map(async ({ page, url }) => {
        try {
          const type = determineType(url);
          const extractedData = await page.evaluate(async (url, type) => {
            if (type === 'youtube') {
              const ytInitialData = JSON.parse(document.body.innerHTML.match(/var ytInitialData = (.+?);<\/script>/)[1]);
              const videoId = new URL(url).searchParams.get('v');
              const videoTitle = ytInitialData.playerOverlays?.playerOverlayRenderer?.videoDetails?.playerOverlayVideoDetailsRenderer?.title?.simpleText;
              let videoDescription = '';
              let authorAvatar = '';
              let authorName = '';
              let authorEmail = '';
              let viewCount = '';
              let date = ''

              const contents = ytInitialData.contents?.twoColumnWatchNextResults?.results?.results?.contents;
              if (contents) {
                const primaryData = contents.find(item => item.videoPrimaryInfoRenderer);
                const secondaryData = contents.find(item => item.videoSecondaryInfoRenderer);
                if (primaryData) {
                  viewCount = primaryData.videoPrimaryInfoRenderer.viewCount?.videoViewCountRenderer?.viewCount?.simpleText || '';
                  date = primaryData.videoPrimaryInfoRenderer.dateText?.simpleText || '';
                }
                if (secondaryData) {
                  const videoSecondaryInfoRenderer = secondaryData.videoSecondaryInfoRenderer;
                  videoDescription = videoSecondaryInfoRenderer.attributedDescription?.content || '';
                  authorAvatar = videoSecondaryInfoRenderer.owner?.videoOwnerRenderer?.thumbnail?.thumbnails?.[0]?.url || '';
                  authorName = videoSecondaryInfoRenderer.owner?.videoOwnerRenderer?.title?.runs?.[0]?.text || '';
                  authorEmail = videoSecondaryInfoRenderer.owner?.videoOwnerRenderer?.title?.runs?.[0]?.navigationEndpoint?.commandMetadata?.webCommandMetadata?.url?.slice(1) || '';
                  thumbnailUrl = `https://img.youtube.com/vi/${videoId}/0.jpg`
                }
              }

              return {
                videoId,
                videoTitle,
                videoDescription,
                authorAvatar,
                authorName,
                authorEmail,
                thumbnailUrl,
                viewCount,
                date,
              };
            } else if (type === 'x') {
              const match = url.match(/\/status\/(\d+)/);
              const videoId = (match && match[1]) ? match[1] : null;
              const date = document.querySelector('a time').textContent.trim();
              return {
                date,
                videoId
              };
            } else if (type === 'bilibili') {
              const videoTitle = document.querySelector('.video-title').textContent.trim();
              const authorName = document.querySelector('.up-name').textContent.trim();
              const date = document.querySelector('.pubdate-ip-text').textContent.trim();
              const videoDescription = document.querySelector('.desc-info-text').textContent.trim();
              let thumbnailUrl = document.querySelector('#wxwork-share-pic')?.getAttribute('src') || '';
              let authorUrl = document.querySelector('.up-avatar')?.href || '';
              let authorAvatar = document.querySelector('.bili-avatar-img')?.getAttribute('data-src') || '';
              thumbnailUrl = thumbnailUrl.split('@')[0];
              authorAvatar = authorAvatar.split('@')[0];
              return {
                videoTitle,
                authorName,
                authorAvatar,
                thumbnailUrl,
                videoDescription,
                date,
                authorUrl
              };
            } else if (type === 'xiaohongshu') {
              //  const ytInitialData = JSON.parse(document.body.innerHTML.match(/var ytInitialData = (.+?);<\/script>/)[1]);
              const thumbnailUrl = document.querySelector('.note-slider-img')?.getAttribute('src') || ''
              const element = document.querySelector('.xgplayer-poster');
              let videolUrl
              if (element) {
                const style = window.getComputedStyle(element);
                const backgroundImage = style.getPropertyValue('background-image');
                const urlMatch = backgroundImage.match(/url\(["']?(.*?)["']?\)/);
                if (urlMatch && urlMatch[1]) {
                  videolUrl = urlMatch[1];
                }
              }
              // const thumbnailUrl = document.querySelector('.note-slider-img')?.getAttribute('src') || ''
              const videoTitle = document.querySelector('#detail-title')?.textContent.trim() || ''
              const authorName = document.querySelector('.username')?.textContent.trim() || ''
              const date = document.querySelector('.date')?.textContent.trim() || '';
              const authorAvatar = document.querySelector('.avatar-item')?.getAttribute('src') || ''
              const videoDescription = document.querySelector('#detail-desc')?.children[0]?.textContent.trim() || ''
              return {
                thumbnailUrl,
                videoTitle,
                authorName,
                date,
                authorAvatar,
                videoDescription,
                videolUrl
              }
            } else if (type === 'facebook') {
              const html = document.body.innerHTML
              const endIndex = html.indexOf('<span>赞</span>');
              const truncatedHtml = endIndex !== -1 ? html.substring(0, endIndex) : html;
              const authorName = document.querySelector('strong span')?.textContent.trim() || ''
              const regex = /<div dir="auto" style="text-align:start">(.+?)<\/div>/g;
              const matches = [];
              let match;

              while ((match = regex.exec(truncatedHtml)) !== null) {
                const text = match[1].trim()
                if (!text.includes('<span')) {
                  matches.push(match[1].trim());
                }
              }
              console.log(matches)
              // const authorName = await page.$eval('strong span', el => el.textContent.trim())
              return {
                authorName,
                matches
              }
            }
          }, url, type);
          if (type === 'bilibili') {
            extractedData.thumbnailUrl = await downloadAndSaveImage(extractedData.thumbnailUrl, 'thumbnail', 434, 244);
            extractedData.authorAvatar = await downloadAndSaveImage(extractedData.authorAvatar, 'authorAvatar', 100, 100);
          }

          return {
            ...extractedData,
            type,
            url
          };
        } catch (error) {
          console.error(`Error extracting data from ${url}: ${error.message}`);
          return null;
        }
      });

    const extractedData = await Promise.all(extractedDataPromises);

    for (let i = 0; i < pages.length; i++) {
      if (pages[i] && extractedData[i]) {
        results.push(extractedData[i]);
      } else {
        console.log(`Failed to extract data for URL: ${pages[i]?.url || 'Unknown URL'}`);
      }
      if (pages[i]) {
        await pages[i].page.close();
      }
    }
    return results;
  } catch (error) {
    console.error('Error scraping data:', error);
  } finally {
    await browser.close();
  }

  return results;
}

function determineType(url) {
  const typeMap = {
    'youtube.com': 'youtube',
    'twitter.com': 'x',
    'x.com': 'x',
    'bilibili.com': 'bilibili',
    'xiaohongshu.com': 'xiaohongshu',
    "facebook.com": 'facebook'
  };

  for (const domain in typeMap) {
    if (url.includes(domain)) {
      return typeMap[domain];
    }
  }

  return 'unknown';
}

//bilibili图片直接引用会跨域，保存到本地 evaluate是浏览器环境，无法使用path
async function downloadAndSaveImage(url, prefix, width = 240, height = 240) {
  if (!url) return null;
  const cleanUrl = url.split('@')[0];  // 移除尺寸参数
  const hash = crypto.createHash('md5').update(cleanUrl).digest('hex');
  const filename = `${prefix}_${hash}.jpg`;
  const filepath = path.join(__dirname, 'public', 'assets', 'storydata', 'bilibili', filename);

  try {
    // 下载原始图片
    const response = await axios({
      url: cleanUrl,
      method: 'GET',
      responseType: 'stream'
    });

    // 使用 sharp 调整尺寸
    await new Promise((resolve, reject) => {
      response.data
        .pipe(sharp())
        .resize(width, height) // 调整尺寸
        .toFile(filepath, (err, info) => {
          if (err) {
            reject(err);
          } else {
            resolve(info);
          }
        });
    });

    return `/assets/storydata/bilibili/${filename}`;
  } catch (error) {
    console.error(`Error downloading and resizing image from ${url}:`, error);
    return null;
  }
}


function parseDate(dateString) {
  const formats = [
    /(\d{4})年(\d{1,2})月(\d{1,2})日/, // "2024年6月12日"
    /(\d{4})-(\d{1,2})-(\d{1,2})/, // "2024-06-18"
    /(\d{1,2}):(\d{1,2}) · (\d{4})年(\d{1,2})月(\d{1,2})日/, // "上午10:59 · 2024年8月20日"
    /(\d{2})-(\d{2})/, // "07-29"
  ];

  for (const format of formats) {
    console.log(dateString, 'dateString');
    if (dateString) {
      const match = dateString.match(format);
      if (match) {
        let year, month, day
        if (match.length === 3) {
          year = new Date().getFullYear()
          month = parseInt(match[1], 10) - 1;
          day = parseInt(match[2], 10);
        } else {
          year = parseInt(match[1], 10);
          month = parseInt(match[2], 10) - 1; // JavaScript 月份是从 0 开始的
          day = parseInt(match[3], 10);
        }
        return new Date(year, month, day);
      }
    }
  }
  return ''
}

//格式化时间并排序
function sortByDate(arr, descending = true) {
  return arr.map(item => {
    const parsedDate = parseDate(item.date);
    if (parsedDate) {
      item.formattedDate = parsedDate?.toISOString()?.substring(0, 10);
    }
    return item;
  }).sort((a, b) => {
    const dateA = new Date(a.formattedDate);
    const dateB = new Date(b.formattedDate);

    if (descending) {
      return dateB - dateA; // 降序
    } else {
      return dateA - dateB; // 升序
    }
  });
}

function clearDirectory(directoryPath) {
  fs.readdir(directoryPath, (err, files) => {
    if (err) {
      console.error(`Error reading directory: ${directoryPath}`, err);
      return;
    }
    files.forEach(file => {
      fs.unlink(path.join(directoryPath, file), err => {
        if (err) {
          console.error(`Error deleting file: ${file}`, err);
        }
      });
    });
  });
}

async function crawlXHS (){
  
}

const URL_LIST = [
  "https://www.xiaohongshu.com/explore/66a786500000000009015489?xsec_token=ABnIt-lH6fhQX9QtRxZ26efsL5Rev0lciTN03JYhURzP0=&xsec_source=pc_search",
  "https://www.xiaohongshu.com/explore/66c05ab2000000000901580d?xsec_token=ABy3jJHBMU9kaAYMOVsxNuzVZxBO2o4kvkGq9wrC2yxe0=&xsec_source=pc_search",
  "https://www.xiaohongshu.com/explore/65a351dd0000000011018408?xsec_token=AB2ianaz-rA6X_XjMhN6MKLIvrNOtQBqOVguOpB5fS428=&xsec_source=pc_search",
  "https://www.xiaohongshu.com/explore/658a974e000000001d025c16?xsec_token=ABc9YG3s0uBPFN_37JyuYaA2W4awVyZtEB4JuEp6mJXB0=&xsec_source=pc_search",
  "https://www.xiaohongshu.com/explore/6588fb0b0000000034034823?xsec_token=ABYyhmQaV6NqmnuR9Bf9hNoMfsq-Rm4yofCZoG7f8w1cs=&xsec_source=pc_search",
  "https://www.xiaohongshu.com/explore/66601df600000000060046a2?xsec_token=ABCnM0d4p41wI5QpoSnzuyQiHVr4cCz1jnlj5jXIQjOTo=&xsec_source=pc_search",
  "https://www.xiaohongshu.com/explore/665b39e60000000005006c49?xsec_token=ABWS8Bf4uJrTFPSHyu1WxVAh9DhfsSfs1ZwkYfUelGZ18=&xsec_source=pc_search",
  "https://www.xiaohongshu.com/explore/666c4e23000000000e032bbc?xsec_token=ABsugUDWchlnDbuj99QHNXtEgvcy-teJ10CNAFLoJJ9sU=&xsec_source=pc_search",

  "https://www.facebook.com/share/p/hLyXkajHg9AMd4n1/",
  "https://www.facebook.com/share/p/WAzpRTrrYAh6QWUB/",
  "https://www.facebook.com/share/p/n1u7NA2Btzcg5cw3/",
  "https://www.facebook.com/share/p/fzVXBkgAwxfMtqAB/",
  "https://www.facebook.com/share/v/PDftY4EMLzH6jdui/",
  "https://www.facebook.com/share/p/cu27hN8uUFJKm2CN/",

  "https://twitter.com/DeemosTech/status/1825729382016745811",
  "https://twitter.com/Gorden_Sun/status/1796578685161783527",
  "https://twitter.com/ntamotsu_/status/1782273273843294611",
  "https://twitter.com/satori_sz9/status/1796015122021765505",
  "https://twitter.com/DeemosTech/status/1812909941105303604",
  "https://twitter.com/grmchn4ai/status/1798119923098804242",
  "https://twitter.com/kuronagirai/status/1782257115022925983",
  "https://twitter.com/DeemosTech/status/1825810156262928478",
  "https://twitter.com/DeemosTech/status/1799112017309217082",
  "https://twitter.com/akinoriosamura/status/1760124253398237202",
  "https://twitter.com/DeemosTech/status/1823375756220838364",
  "https://twitter.com/akinoriosamura/status/1755428565053771925",
  "https://twitter.com/WilliamLamkin/status/1792934921595469858",
  "https://twitter.com/DeemosTech/status/1805236190066708493",
  "https://twitter.com/DeemosTech/status/1802329656030633994",
  "https://twitter.com/DeemosTech/status/1815026598695112841",
  "https://twitter.com/DeemosTech/status/1809969250817454325",
  "https://twitter.com/Koba_1975/status/1796712777802731984",
  "https://twitter.com/labibjaffar/status/1818236302073925664",
  "https://twitter.com/DeemosTech/status/1803253026708398228",
  "https://twitter.com/Gorden_Sun/status/1796578685161783527",
  "https://twitter.com/DeemosTech/status/1825729382016745811",
  "https://twitter.com/DeemosTech/status/1827736182610153795",
  "https://twitter.com/DeemosTech/status/1826692188077785112",

  "https://www.youtube.com/watch?v=HpVOs5imUN0",
  "https://www.youtube.com/watch?v=MFRbfY0cZFo",
  "https://www.youtube.com/watch?v=tpFEainx6Ak",
  "https://www.youtube.com/watch?v=h3J1os_YfiI",
  "https://www.youtube.com/watch?v=2WPdvIl3yYg",
  "https://www.youtube.com/watch?v=TZVioAWixJ4",
  "https://www.youtube.com/watch?v=wU9ouVuT7CQ",
  "https://www.youtube.com/watch?v=l1PQ2d8DfHo",
  "https://www.youtube.com/watch?v=YvB8wk_wJ6E",

  "https://www.bilibili.com/video/BV1L6421Z7ng/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1ky411b7ti/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV13142117Dx/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1bje8eVEdj/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1Dw4m1y72v/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV19WpoeAEJX/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1ii421Q7bQ/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1g4421Q7bp/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1d9TyeFEMY/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1pW421R7hd/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1fm421N7yu/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1jw4m1i7qT/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1xy41187bb/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1Wi421v7JR/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1Vb421n76D/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1Yi421U759/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1Hj42197bz/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1sT421v7LK/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1nw4m197wo/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",
  "https://www.bilibili.com/video/BV1SvVYeNEsC/?spm_id_from=333.337.search-card.all.click&vd_source=2cf827ae246d7e0c43844bc723259237",


];


scrapeData(URL_LIST).then(results => {
  console.log(`Scraped data for ${results.length} videos`);
  const sortedResults = sortByDate(results);
  fs.writeFileSync('public/assets/storydata/extractedVideosData.json', JSON.stringify(sortedResults, null, 2));
  console.log('All data saved to extractedVideosData.json');
});

优化爬取脚本 ​

js
const puppeteer = require('puppeteer');
const cheerio = require('cheerio');
const axios = require('axios');
const fs = require('fs').promises;

const SITE_CONFIGS = {
  XIAOHONGSHU: {
    dataKey: 'note',
    identifier: 'xiaohongshu.com',
    parseStrategies: [
      {
        name: 'initialState',
        pattern: /window\.__INITIAL_STATE__\s*=\s*({[^;]+})/,
        extract: (match) => JSON.parse(match[1]).note
      },
      {
        name: 'bracketMatching',
        pattern: null,
        extract: (content, config) => parseByBracketMatching(content, config)
      }
    ]
  },
  BILIBILI: {
    dataKey: 'videoData',
    identifier: 'bilibili.com',
    parseStrategies: [
      {
        name: 'initialState',
        pattern: /window\.__INITIAL_STATE__\s*=\s*({[^;]+})/,
        extract: (match) => JSON.parse(match[1]).videoData
      },
      {
        name: 'bracketMatching',
        pattern: null,
        extract: (content, config) => parseByBracketMatching(content, config)
      }
    ]
  }
};

function parseByBracketMatching(scriptContent, siteConfig) {
  const dataKeyIndex = scriptContent.indexOf(`,"${siteConfig.dataKey}":`);
  if (dataKeyIndex === -1) return;
  const dataStartIndex = scriptContent.indexOf('{', dataKeyIndex);
  if (dataStartIndex === -1) return;
  const stack = ['{'];
  let dataEndIndex = dataStartIndex + 1;

  while (dataEndIndex < scriptContent.length && stack.length > 0) {
    const char = scriptContent[dataEndIndex];
    if (char === '{') stack.push('{');
    else if (char === '}') stack.pop();
    dataEndIndex++;
  }

  if (stack.length === 0) {
    const dataStr = scriptContent.substring(dataStartIndex, dataEndIndex);
    return JSON.parse(dataStr);
  }
}

async function crawlVideoByHtml(videoUrl) {
  try {
    const { data: html } = await axios.get(videoUrl);
    const $ = cheerio.load(html);

    const siteConfig = Object.values(SITE_CONFIGS).find(config =>
      videoUrl.includes(config.identifier)
    );

    if (!siteConfig) {
      throw new Error('不支持的网站');
    }

    let videoData = null;
    $('script').each((index, element) => {
      const scriptContent = $(element).html();
      if (!scriptContent) return;

      // 遍历所有解析策略
      for (const strategy of siteConfig.parseStrategies) {
        try {
          if (strategy.pattern) {
            const match = scriptContent.match(strategy.pattern);
            if (match) {
              videoData = strategy.extract(match, siteConfig);
              if (videoData) {
                console.log(`使用 ${strategy.name} 策略成功解析数据 --- `);
                return false;
              }
            }
          } else {
            videoData = strategy.extract(scriptContent, siteConfig);
            if (videoData) {
              console.log(`使用 ${strategy.name} 策略成功解析数据`);
              return false;
            }
          }
        } catch (err) {
          console.error(`${strategy.name} 策略解析失败:`, err);
          continue;
        }
      }
    });

    return videoData;
  } catch (error) {
    console.error('爬取失败:', error);
    throw error;
  }
}

async function main() {
  // 测试URL
  const xhsUrl = 'https://www.xiaohongshu.com/explore/672c8dcf000000003c01c673?xsec_token=ABue8fMPYeoMxLuK-hG69Oi1gdn8lLd_clZlSlJhS0juU=&xsec_source=pc_feed';
  const biliUrl = 'https://www.bilibili.com/video/BV1i8D6YGEcp/?spm_id_from=333.1007.tianma.1-2-2.click&vd_source=2cf827ae246d7e0c43844bc723259237';

  try {
    // 测试小红书
    const xhsData = await crawlVideoByHtml(xhsUrl);
    if (xhsData) {
      await fs.writeFile(
        'xhs_data.json',
        JSON.stringify(xhsData, null, 2),
        'utf8'
      );
      console.log('小红书数据已成功写入 xhs_data.json');
    }

    // 测试B站
    const biliData = await crawlVideoByHtml(biliUrl);
    if (biliData) {
      await fs.writeFile(
        'bili_data.json',
        JSON.stringify(biliData, null, 2),
        'utf8'
      );
      console.log('B站数据已成功写入 bili_data.json');
    }
  } catch (error) {
    console.error('爬取失败:', error);
  }
}

main()

脚本数据反爬机制 ​

html
<script>
  window.__INIT_STATE__={}
  (function() {
            var s;
            (s = document.currentScript || document.scripts[document.scripts.length - 1]).parentNode.removeChild(s);
        }());
</script>