Cloudflare Browser Rendering で AI ベースの Web スクレイピング

export async function extractWithAI<T>(
  url: string,
  prompt: string,
  schema: Record<string, unknown>,
  env: Env
): Promise<T> {
  const endpoint = `https://api.cloudflare.com/client/v4/accounts/${env.CLOUDFLARE_ACCOUNT_ID}/browser-rendering/json`;

  const payload = {
    url,
    prompt,
    response_format: {
      type: 'json_schema',
      schema,
    },
  };

  const response = await fetch(endpoint, {
    method: 'POST',
    headers: {
      Authorization: `Bearer ${env.CLOUDFLARE_API_TOKEN}`,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify(payload),
  });

  const data = await response.json();
  return data.result;
}

export async function parseWithAI(
  url: string,
  name: string,
  sinceDate: string,
  env: Env
): Promise<Article[]> {
  const prompt = `
You are extracting articles from a blog or news website.

Instructions:
1. Find all articles, blog posts, or news items on this page
2. For each article, extract:
   - Title: The main heading or title of the article
   - URL: The link to the full article
   - Date: The publication date in YYYY-MM-DD format

3. Date handling (CRITICAL):
   - If the date is visible on the list page, use it
   - If NOT visible, navigate to the article detail page and extract the date
   - Convert all dates to YYYY-MM-DD format

4. Filtering:
   - Only include articles published on or after ${sinceDate}

Return the results as a structured JSON array.
`;

  const schema = {
    type: 'object',
    properties: {
      articles: {
        type: 'array',
        items: {
          type: 'object',
          properties: {
            title: { type: 'string' },
            url: { type: 'string' },
            date: { type: 'string' },
          },
          required: ['title', 'url', 'date'],
        },
      },
    },
    required: ['articles'],
  };

  const result = await extractWithAI<{ articles: Article[] }>(
    url,
    prompt,
    schema,
    env
  );

  return result.articles.map(article => ({
    ...article,
    source: name,
  }));
}

#!/bin/bash

CLOUDFLARE_ACCOUNT_ID="your-account-id"
CLOUDFLARE_API_TOKEN="your-api-token"
TEST_URL="https://kusumoto.app/blog"

ENDPOINT="https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/browser-rendering/json"

REQUEST_BODY=$(cat <<EOF
{
  "url": "$TEST_URL",
  "prompt": "このページから最新の記事を抽出してください。各記事のタイトル、URL、日付を取得してください。",
  "response_format": {
    "type": "json_schema",
    "schema": {
      "type": "object",
      "properties": {
        "articles": {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "title": { "type": "string" },
              "url": { "type": "string" },
              "date": { "type": "string" }
            },
            "required": ["title", "url"]
          }
        }
      },
      "required": ["articles"]
    }
  }
}
EOF
)

curl -X POST "$ENDPOINT" \
  -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \
  -H "Content-Type: application/json" \
  -d "$REQUEST_BODY" | jq .

{
  "success": true,
  "result": {
    "articles": [
      {
        "title": "iOS アプリ開発 〜 Mac のカメラを使ってシミュレータでカメラの動作確認",
        "url": "https://kusumoto.app/blog/rocketsim-simulator-camera",
        "date": "2025-11-07"
      },
      ...
    ]
  }
}