yaoyaoai/scripts/nodejs/douyin_profile_scraper.js

"use strict";

/**
 * 抖音用户主页采集（对齐 Electron ipAgent:analyzeDouyinAccount）
 */

const https = require("https");
const { DouyinBrowserManager } = require("./douyin_browser_manager.js");

const PROFILE_URL_RE =
  /douyin\.com\/user|iesdouyin\.com\/share\/user|v\.douyin\.com/i;

const SCRAPE_SCRIPT = `
(() => {
  try {
    let accountName = "";
    const nicknameEl = document.querySelector(
      '[class*="nickname"], [class*="account"], h1'
    );
    if (nicknameEl) accountName = nicknameEl.textContent?.trim() || "";

    if (!accountName) {
      const metaEl = document.querySelector('meta[property="og:title"]');
      if (metaEl) accountName = metaEl.getAttribute("content") || "";
    }

    if (!accountName) {
      const allHeaders = document.querySelectorAll(
        'h1, h2, .dy-name, [data-testid*="name"], [class*="userName"]'
      );
      for (const el of allHeaders) {
        const text = el.textContent?.trim();
        if (text && text.length > 2 && text.length < 30) {
          accountName = text;
          break;
        }
      }
    }

    if (!accountName) {
      const titleText = document.title || "";
      const match = titleText.match(/(.+?)(?:的抖音|抖音号|-)/);
      accountName = match ? match[1].trim() : titleText.split("-")[0]?.trim() || "";
    }

    const videoTitles = [];
    const seenTexts = new Set();
    const excludeWords = [
      "抖音号", "IP属地", "获赞", "粉丝", "关注", "作品", "点赞", "评论",
      "分享", "收藏", "私信", "粉丝团", "加关注", "最新作品", "合作推荐", "喜欢作者",
    ];

    const shouldExclude = (text) => {
      if (!text || text.length < 5 || text.length > 300) return true;
      return excludeWords.some((word) => text.includes(word));
    };

    const allElements = document.querySelectorAll(
      'p, span[class*="desc"], div[class*="title"]'
    );
    allElements.forEach((el) => {
      if (videoTitles.length >= 6) return;
      const text = el.textContent?.trim();
      if (
        text &&
        text.length > 10 &&
        text.length < 500 &&
        !shouldExclude(text) &&
        !seenTexts.has(text)
      ) {
        videoTitles.push(text);
        seenTexts.add(text);
      }
    });

    if (videoTitles.length < 3) {
      const titleSelectors = [
        '[class*="title"]',
        '[class*="desc"]',
        '[class*="caption"]',
        '[data-testid*="title"]',
        '[class*="feed-item"]',
        '[class*="video-item"]',
        ".dy-video-title",
        ".aweme-detail",
        '[class*="aweme"]',
      ];
      for (const selector of titleSelectors) {
        if (videoTitles.length >= 6) break;
        document.querySelectorAll(selector).forEach((el) => {
          if (videoTitles.length >= 6) return;
          const text = el.textContent?.trim();
          if (text && !shouldExclude(text) && text.length > 5 && !seenTexts.has(text)) {
            videoTitles.push(text);
            seenTexts.add(text);
          }
        });
      }
    }

    const cleanTitles = videoTitles
      .map((title) => title.replace(/^\\d+\\s+/, "").trim())
      .filter((t) => t && t.length > 5);

    return {
      accountName: accountName || "未知账号",
      videoTitles: Array.from(new Set(cleanTitles)).slice(0, 6),
      pageTitle: document.title,
      pageUrl: window.location.href,
    };
  } catch (err) {
    return {
      accountName: document.title?.split("-")[0] || "错误",
      videoTitles: [],
      pageTitle: document.title,
      pageUrl: window.location.href,
      error: err.message,
    };
  }
})()
`;

function emitProgress(msg) {
  try {
    globalThis.__native?.emitProgress?.(msg);
  } catch (_) {
    /* ignore */
  }
}

function resolveShortUrl(shortUrl) {
  return new Promise((resolve, reject) => {
    let u;
    try {
      u = new URL(shortUrl);
    } catch (e) {
      reject(e);
      return;
    }
    const opts = {
      hostname: u.hostname,
      path: u.pathname + u.search,
      method: "HEAD",
      headers: {
        "User-Agent":
          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
      },
      timeout: 30000,
    };
    const req = https.request(opts, (res) => {
      if (
        res.statusCode &&
        res.statusCode >= 300 &&
        res.statusCode < 400 &&
        res.headers.location
      ) {
        resolve(res.headers.location);
      } else {
        resolve(shortUrl);
      }
    });
    req.on("error", reject);
    req.on("timeout", () => {
      req.destroy();
      reject(new Error("短链接解析超时"));
    });
    req.end();
  });
}

async function normalizeProfileUrl(input) {
  let url = String(input || "").trim();
  if (!url) throw new Error("请输入账号主页或分享链接");

  const urlMatch = url.match(/https?:\/\/[^\s\u4e00-\u9fa5"'<>]+/i);
  if (urlMatch) url = urlMatch[0];
  else if (!url.startsWith("http")) url = `https://${url}`;

  if (/v\.douyin\.com|iesdouyin\.com/i.test(url) && !/\/user\//i.test(url)) {
    emitProgress("正在解析分享链接...");
    url = await resolveShortUrl(url);
  }

  if (!PROFILE_URL_RE.test(url)) {
    throw new Error("请输入抖音账号主页或用户分享链接（支持 v.douyin.com 短链）");
  }
  return url;
}

async function scrapeDouyinProfile(profileUrl) {
  const manager = DouyinBrowserManager.getInstance();
  emitProgress("正在打开浏览器并采集数据，请稍候...");
  await manager.initBrowser();
  const page = await manager.newPage();
  try {
    await page.goto(profileUrl, { waitUntil: "domcontentloaded", timeout: 60000 });
    emitProgress("页面已加载，正在提取账号与视频标题...");
    await new Promise((r) => setTimeout(r, 5000));
    const data = await page.evaluate(SCRAPE_SCRIPT);
    if (data?.error) {
      throw new Error(data.error);
    }
    return data;
  } finally {
    try {
      await page.close();
    } catch (_) {
      /* ignore */
    }
  }
}

module.exports = {
  normalizeProfileUrl,
  scrapeDouyinProfile,
};