import { chromium } from 'playwright' import { ensureDataFiles, readCache, writeCache, writeStatus } from './lib/cache.js' const PAGE_URL = 'https://www1.gdtv.cn/tvColumn/768' function parseEpisodeItem(rawItem) { try { const dataObj = typeof rawItem.data === 'string' ? JSON.parse(rawItem.data) : rawItem.data const videoObj = typeof dataObj.videoUrl === 'string' ? JSON.parse(dataObj.videoUrl) : dataObj.videoUrl return { id: dataObj.id || rawItem.id, title: dataObj.title || '', coverUrl: dataObj.coverUrl || '', releasedAt: dataObj.releasedAt || 0, timeLength: dataObj.timeLength || 0, videoUrl: videoObj?.hd || videoObj?.sd || '', raw: dataObj } } catch (error) { console.error('解析单条节目失败:', error) return null } } function mergeCache(oldCache, responseUrl, data) { const url = new URL(responseUrl) const currentPage = Number(url.searchParams.get('currentPage') || 1) const beginScore = Number(url.searchParams.get('beginScore') || 0) const parsedItems = (data.list || []) .map(parseEpisodeItem) .filter(Boolean) const itemMap = new Map((oldCache.items || []).map(item => [item.id, item])) for (const item of parsedItems) { itemMap.set(item.id, item) } return { ...oldCache, updatedAt: new Date().toISOString(), name: data.name || oldCache.name || '七十二家房客', coverUrl: data.coverUrl || oldCache.coverUrl || '', displayType: data.displayType ?? oldCache.displayType ?? 0, beginScoreMap: { ...(oldCache.beginScoreMap || {}), [currentPage]: beginScore }, pages: { ...(oldCache.pages || {}), [currentPage]: { currentPage, beginScore, count: parsedItems.length, capturedAt: new Date().toISOString() } }, items: Array.from(itemMap.values()).sort((a, b) => (b.releasedAt || 0) - (a.releasedAt || 0)) } } function log(...args) { console.log('[capture]', ...args) } async function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)) } async function getPageScrollInfo(page) { return await page.evaluate(() => { return { scrollTop: window.scrollY || document.documentElement.scrollTop || document.body.scrollTop || 0, scrollHeight: Math.max( document.body.scrollHeight, document.documentElement.scrollHeight ), innerHeight: window.innerHeight } }) } async function scrollStep(page, distance = 700) { await page.evaluate((step) => { window.scrollBy(0, step) }, distance) } async function findLoadMoreButton(page) { const selectors = [ '.index__load-component___1Ht2U button', 'button:has-text("点击加载更多")', 'button:has-text("加载更多")', '.ant-btn:has-text("点击加载更多")' ] for (const selector of selectors) { const locator = page.locator(selector).first() const count = await page.locator(selector).count().catch(() => 0) if (!count) continue const visible = await locator.isVisible().catch(() => false) if (visible) { return locator } } return null } async function getDomItemCount(page) { return await page.evaluate(() => { const selectors = [ '.index__item-component___1q1ob', '.index__list-item', '.ant-list-item', 'li' ] for (const selector of selectors) { const count = document.querySelectorAll(selector).length if (count > 0) return count } return 0 }).catch(() => 0) } async function clickLoadMoreAndWait(page) { const button = await findLoadMoreButton(page) if (!button) return false const responsePromise = page.waitForResponse( (response) => response.url().includes('gdtv-api.gdtv.cn/api/tvColumn/v1/news') && response.request().method() === 'GET' && response.status() === 200, { timeout: 15000 } ).catch(() => null) await button.scrollIntoViewIfNeeded().catch(() => {}) await sleep(800) await button.click({ timeout: 10000 }).catch(async () => { await button.click({ force: true, timeout: 10000 }) }) const response = await responsePromise await sleep(1800) return !!response } async function autoCollectByScrollAndClick(page, maxRounds = 120) { let round = 0 let noChangeRounds = 0 let lastDomCount = await getDomItemCount(page) let lastScrollTop = 0 while (round < maxRounds) { round += 1 const button = await findLoadMoreButton(page) if (button) { log(`第 ${round} 轮:发现“加载更多”按钮,准备点击`) const clicked = await clickLoadMoreAndWait(page) const currentDomCount = await getDomItemCount(page) if (clicked) { log(`第 ${round} 轮:点击成功,当前列表数量 ${currentDomCount}`) } else { log(`第 ${round} 轮:点击后未捕获到新响应`) } if (currentDomCount <= lastDomCount) { noChangeRounds += 1 } else { noChangeRounds = 0 lastDomCount = currentDomCount } await sleep(1200) continue } await scrollStep(page, 700) await sleep(1400) const { scrollTop, scrollHeight, innerHeight } = await getPageScrollInfo(page) const currentDomCount = await getDomItemCount(page) log( `第 ${round} 轮:继续下滑,scrollTop=${scrollTop},scrollHeight=${scrollHeight},items=${currentDomCount}` ) if (currentDomCount <= lastDomCount && scrollTop === lastScrollTop) { noChangeRounds += 1 } else { if (currentDomCount > lastDomCount) { lastDomCount = currentDomCount } noChangeRounds = 0 } lastScrollTop = scrollTop const nearBottom = scrollTop + innerHeight >= scrollHeight - 80 if (nearBottom) { log(`第 ${round} 轮:已经接近页面底部`) await sleep(1800) const retryButton = await findLoadMoreButton(page) if (!retryButton) { noChangeRounds += 1 } } if (noChangeRounds >= 6) { log('连续多轮没有新内容,停止采集') break } } log(`自动采集流程结束,共执行 ${round} 轮`) } async function main() { ensureDataFiles() writeStatus({ running: true, lastMessage: '正在启动浏览器采集', updatedAt: new Date().toISOString() }) const browser = await chromium.launchPersistentContext('./.pw-user-data', { headless: false }) const page = await browser.newPage() page.on('response', async (response) => { const url = response.url() if (!url.includes('gdtv-api.gdtv.cn/api/tvColumn/v1/news')) return if (response.status() !== 200) return try { const data = await response.json() const oldCache = readCache() const nextCache = mergeCache(oldCache, url, data) writeCache(nextCache) const currentPage = new URL(url).searchParams.get('currentPage') const capturedPages = Object.keys(nextCache.pages || {}).length const totalItems = nextCache.items?.length || 0 writeStatus({ running: true, lastMessage: `已采集第 ${currentPage} 页`, updatedAt: new Date().toISOString(), capturedPages, totalItems }) log(`采集成功: page=${currentPage} totalItems=${totalItems}`) } catch (error) { console.error('[capture] 解析响应失败:', error) } }) await page.goto(PAGE_URL, { waitUntil: 'domcontentloaded' }) await page.waitForTimeout(5000) log('已打开页面:', PAGE_URL) log('等待第一页接口加载...') await page.waitForResponse( (response) => response.url().includes('gdtv-api.gdtv.cn/api/tvColumn/v1/news') && response.request().method() === 'GET' && response.status() === 200, { timeout: 15000 } ).catch(() => null) await sleep(2500) writeStatus({ running: true, lastMessage: '开始自动下滑并检测加载更多按钮', updatedAt: new Date().toISOString() }) await autoCollectByScrollAndClick(page, 150) writeStatus({ running: false, lastMessage: '自动采集完成', updatedAt: new Date().toISOString() }) log('自动采集完成,数据已写入 data/qishier-cache.json') } main().catch((error) => { console.error('采集启动失败:', error) writeStatus({ running: false, lastMessage: `采集启动失败: ${error.message}`, updatedAt: new Date().toISOString() }) })