yigitkonur/scrape-do-mcp-server

# Simple Mode (default - only scrape_links tool)
claude mcp add scrape-do npx --scope user \
  -e SCRAPEDO_API_KEY=your_scrape_do_key \
  -- -y @yigitkonur/scrape-do-mcp-server

# Research Suite Mode (all 6 tools + search + LLM processing)
claude mcp add scrape-do npx --scope user \
  -e SCRAPEDO_API_KEY=your_scrape_do_key \
  -e SERPER_API_KEY=your_serper_key \
  -e ADVANCED_MODE=true \
  -e LLM_ENABLED=true \
  -e OPENAI_BASE_URL=https://openrouter.ai/api/v1 \
  -e OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxx \
  -e OPENAI_MODEL=google/gemini-2.0-flash-exp:free \
  -e OPENAI_MODEL_SECONDARY=x-ai/grok-2-1212:free \
  -e OPENAI_MODEL_TERTIARY=openai/gpt-4o-mini \
  -- -y @yigitkonur/scrape-do-mcp-server

# Advanced Mode (all 6 tools, LLM disabled for speed)
claude mcp add scrape-do npx --scope user \
  -e SCRAPEDO_API_KEY=your_scrape_do_key \
  -e SERPER_API_KEY=your_serper_key \
  -e ADVANCED_MODE=true \
  -e LLM_ENABLED=false \
  -- -y @yigitkonur/scrape-do-mcp-server

# Step 1: Install globally
npm install -g @yigitkonur/scrape-do-mcp-server

# Step 2: Add to Claude (Simple Mode - default)
claude mcp add scrape-do scrape-do-mcp-server --scope user \
  -e SCRAPEDO_API_KEY=your_scrape_do_key

# Step 2: Add to Claude (Research Suite Mode with LLM Failover)
claude mcp add scrape-do scrape-do-mcp-server --scope user \
  -e SCRAPEDO_API_KEY=your_scrape_do_key \
  -e SERPER_API_KEY=your_serper_key \
  -e ADVANCED_MODE=true \
  -e LLM_ENABLED=true \
  -e OPENAI_BASE_URL=https://openrouter.ai/api/v1 \
  -e OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxx \
  -e OPENAI_MODEL=google/gemini-2.0-flash-exp:free \
  -e OPENAI_MODEL_SECONDARY=x-ai/grok-2-1212:free \
  -e OPENAI_MODEL_TERTIARY=openai/gpt-4o-mini

npm install -g @yigitkonur/scrape-do-mcp-server

{
  "mcpServers": {
    "scrape-do": {
      "command": "scrape-do-mcp-server",
      "env": {
        "SCRAPEDO_API_KEY": "your_scrape_do_key"
      }
    }
  }
}

{
  "mcpServers": {
    "scrape-do": {
      "command": "scrape-do-mcp-server",
      "env": {
        "SCRAPEDO_API_KEY": "your_scrape_do_key",
        "SERPER_API_KEY": "your_serper_key",
        "ADVANCED_MODE": "true",
        "LLM_ENABLED": "true",
        "OPENAI_BASE_URL": "https://openrouter.ai/api/v1",
        "OPENAI_API_KEY": "sk-xxxxxxxxxxxxxxxxx",
        "OPENAI_MODEL": "google/gemini-2.0-flash-exp:free",
        "OPENAI_MODEL_SECONDARY": "x-ai/grok-2-1212:free",
        "OPENAI_MODEL_TERTIARY": "openai/gpt-4o-mini"
      }
    }
  }
}

{
  "mcpServers": {
    "scrape-do": {
      "command": "scrape-do-mcp-server",
      "env": {
        "SCRAPEDO_API_KEY": "your_scrape_do_key",
        "SERPER_API_KEY": "your_serper_key",
        "ADVANCED_MODE": "true",
        "LLM_ENABLED": "false"
      }
    }
  }
}

# Check if binary is available
which scrape-do-mcp-server

# Verify Claude MCP connection
claude mcp list

# Set environment variables first
export SCRAPEDO_API_KEY=your_key

npx @modelcontextprotocol/inspector --cli \
  'scrape-do-mcp-server' \
  --method tools/call --tool-name scrape_single \
  --tool-arg url="https://example.com" \
  --tool-arg use_llm=false

# Set environment variables first
export SCRAPEDO_API_KEY=your_key
export OPENAI_API_KEY=your_openrouter_key
export OPENAI_BASE_URL=https://openrouter.ai/api/v1

npx @modelcontextprotocol/inspector --cli \
  'scrape-do-mcp-server' \
  --method tools/call --tool-name scrape_single \
  --tool-arg url="https://example.com" \
  --tool-arg use_llm=true \
  --tool-arg what_to_extract="main content only"

# Test Simple Mode (default - 1 tool)
env SCRAPEDO_API_KEY=your_key \
npx @modelcontextprotocol/inspector --cli -- \
npx @yigitkonur/scrape-do-mcp-server --method tools/list

# Test Advanced Mode (6 tools)
env SCRAPEDO_API_KEY=your_key ADVANCED_MODE=true \
npx @modelcontextprotocol/inspector --cli -- \
npx @yigitkonur/scrape-do-mcp-server --method tools/list

# Test with LLM processing
env SCRAPEDO_API_KEY=your_key \
    OPENAI_API_KEY=your_openrouter_key \
    OPENAI_BASE_URL=https://openrouter.ai/api/v1 \
npx @modelcontextprotocol/inspector --cli -- \
npx @yigitkonur/scrape-do-mcp-server \
--method tools/call --tool-name scrape_links \
--tool-arg 'urls=["https://example.com"]' \
--tool-arg 'use_llm=true' \
--tool-arg 'what_to_extract="main content"'

# Process multiple URLs efficiently with automatic batching
scrape_links(urls=["site1.com", "site2.com", "site3.com"])

# Batch processes 10 URLs concurrently, then next 10, etc.
scrape_links(urls=[...30_urls...])  # Processed in 3 batches of 10

# Search multiple topics at once
get_multiple_search_results(keywords=["AI research", "machine learning", "deep learning"])

# Default mode - only scrape_links available
export SCRAPEDO_API_KEY=your_key
# ADVANCED_MODE=false (default)

# Enable all tools
export SCRAPEDO_API_KEY=your_key
export ADVANCED_MODE=true

# Basic batch search
get_multiple_search_results(
  keywords=["python tutorials", "machine learning", "data science"]
)

# Research workflow example
get_multiple_search_results(
  keywords=["competitor analysis", "market research 2024", "industry trends"]
)

# Basic usage - automatically retries with upgraded modes on 502 errors
scrape_links(
  urls=["https://news.ycombinator.com", "https://github.com/trending"],
  mode="basic",
  use_llm=false
)

# LLM-enhanced batch processing with targeted extraction
scrape_links(
  urls=["https://site1.com", "https://site2.com", "https://site3.com"],
  mode="basic",
  use_llm=true,
  what_to_extract="main content and key points"
)

# Advanced usage with JavaScript rendering and LLM processing
scrape_links(
  urls=["https://example.com"],
  mode="javascript",
  timeout=60,
  waitFor=5000,
  use_llm=true
)

# Batch processing - handles 10 concurrent URLs per batch
scrape_links(
  urls=[...30_different_urls...],  # Processes in 3 batches of 10
  mode="basic",
  use_llm=false
)

# Premium mode with geo-targeting and LLM extraction
scrape_links(
  urls=["https://geo-restricted.com"],
  mode="premium",
  country="US",
  use_llm=true,
  what_to_extract="product pricing and availability"
)

# Basic scraping (1 credit)
scrape_single(
  url="https://example.com",
  use_llm=false
)

# LLM-enhanced scraping with targeted extraction
scrape_single(
  url="https://example.com",
  use_llm=true,
  what_to_extract="main content and key points"
)

# Advanced options
scrape_single(
  url="https://example.com",
  followRedirects=true,
  timeout=30000,
  use_llm=true
)

# Basic premium scraping with residential proxy (10 credits)
scrape_premium(
  url="https://restricted-site.com",
  country="US",
  use_llm=false
)

# LLM-enhanced premium scraping
scrape_premium(
  url="https://restricted-site.com",
  country="US",
  use_llm=true,
  what_to_extract="product information and availability"
)

# Advanced options with sticky sessions
scrape_premium(
  url="https://restricted-site.com",
  country="US",
  sticky=true,
  sessionId="my-session-123",
  followRedirects=true,
  timeout=30000,
  use_llm=true
)

# Basic JavaScript rendering (5 credits)
scrape_javascript(
  url="https://spa-application.com",
  waitFor=5000,
  use_llm=false
)

# Wait for specific element to load
scrape_javascript(
  url="https://spa-application.com",
  waitSelector=".content-loaded",
  use_llm=false
)

# LLM-enhanced extraction from dynamic content
scrape_javascript(
  url="https://spa-application.com",
  waitFor=3000,
  use_llm=true,
  what_to_extract="product details and pricing"
)

# Advanced options
scrape_javascript(
  url="https://spa-application.com",
  waitFor=5000,
  waitSelector="#main-content",
  viewport={"width": 1920, "height": 1080},
  timeout=30000,
  use_llm=true
)

scrape_interactive(
  url="https://complex-site.com",
  actions=[
    {"type": "click", "selector": ".cookie-accept"},
    {"type": "fill", "selector": "input[name='search']", "value": "query"},
    {"type": "click", "selector": ".search-button"},
    {"type": "wait", "wait": 2000}
  ]
)

check_credits(includeDetails=true)

# ✅ Correct: Use environment variables
export SCRAPEDO_API_KEY=your_actual_key
export SERPER_API_KEY=your_actual_key
export OPENAI_API_KEY=your_actual_key

# ✅ Correct: Use .env file (automatically ignored)
cp .env.example .env
# Edit .env with your actual keys

# ❌ Wrong: Never put real keys in code or public repos

# Example: LLM enabled globally, but can be disabled per request
LLM_ENABLED=true  # Global default: ON
scrape_single(url="...", use_llm=false)  # This request: OFF

# Example: LLM disabled globally, but can be enabled per request
LLM_ENABLED=false  # Global default: OFF
scrape_single(url="...", use_llm=true)   # This request: ON

# High-volume processing
CACHE_TTL_SECONDS=300
RATE_LIMIT_REQUESTS=200

# Development/testing
CACHE_TTL_SECONDS=30
LOG_LEVEL=debug

# Production reliability
DEFAULT_TIMEOUT_MS=45000
MAX_TIMEOUT_MS=180000

# News articles and blogs
scrape_links(urls=["news-site.com"], mode="basic")

# E-commerce product pages
scrape_links(urls=["shop.com/product"], mode="premium", country="US")

# Single-page applications
scrape_links(urls=["app.com"], mode="javascript", waitFor=3000)

# Complex interactive sites
scrape_interactive(
  url="complex-site.com",
  actions=[{"type": "click", "selector": ".load-more"}]
)

# You only need to call once - the system handles retries automatically
scrape_links(urls=["site1.com", "site2.com"], mode="basic")

# Behind the scenes, for URLs that return 502 errors:
# 1. First attempt: basic mode (fast, 1 credit)
# 2. Auto-retry: javascript mode if basic returns 502 (5 credits)
# 3. Final retry: premium mode if javascript returns 502 (10 credits)
# 4. Returns best available result or clear error message

# For timeouts or other errors, manually escalate:
1. Try: scrape_links(mode="basic")
2. If timeout: scrape_links(mode="javascript", timeout=60)
3. If blocked: scrape_links(mode="premium", country="US")
4. Complex sites: scrape_interactive with custom actions

# Check credits before large operations
check_credits()

# Use appropriate modes to minimize costs
- Basic (1 credit): Static content
- JavaScript (5 credits): Dynamic content only when needed
- Premium (10 credits): When basic/javascript fail

LLM_ENABLED=true
OPENAI_API_KEY=your_openrouter_api_key
OPENAI_BASE_URL=https://openrouter.ai/api/v1
OPENAI_MODEL=moonshotai/kimi-k2-0905:nitro

scrape_single(
  url="https://messy-blog-post.com",
  use_llm=true
)

scrape_single(
  url="https://product-page.com",
  use_llm=true,
  what_to_extract="Extract only price, title, and availability status"
)

scrape_links(
  urls=["site1.com", "site2.com", "site3.com"],
  use_llm=true,
  what_to_extract="main content and key points"
)

# Only Scrape.do API key required
export SCRAPEDO_API_KEY=your_scrape_do_key
export LLM_ENABLED=false

# Usage: Pure scraping, maximum speed
scrape_single(url="https://example.com", use_llm=false)

# Scrape.do + OpenRouter configuration
export SCRAPEDO_API_KEY=your_scrape_do_key
export LLM_ENABLED=true
export OPENAI_API_KEY=your_openrouter_key
export OPENAI_BASE_URL=https://openrouter.ai/api/v1
export OPENAI_MODEL=moonshotai/kimi-k2-0905:nitro

# Usage: AI-enhanced content processing
scrape_single(url="https://example.com", use_llm=true, what_to_extract="key information")

# LLM available but disabled by default
export SCRAPEDO_API_KEY=your_scrape_do_key
export LLM_ENABLED=false
export OPENAI_API_KEY=your_openrouter_key
export OPENAI_BASE_URL=https://openrouter.ai/api/v1

# Usage: Choose per request
scrape_single(url="https://example.com", use_llm=false)  # Fast
scrape_single(url="https://example.com", use_llm=true)   # Enhanced

# Remove any existing configuration
claude mcp remove scrape-do -s user

# Use correct NPX syntax with proper argument separation
claude mcp add scrape-do npx --scope user \
  -e SCRAPEDO_API_KEY=your_key \
  -- -y @yigitkonur/scrape-do-mcp-server

# Verify connection
claude mcp list

# Install globally then use direct binary
npm install -g @yigitkonur/scrape-do-mcp-server
claude mcp add scrape-do scrape-do-mcp-server --scope user \
  -e SCRAPEDO_API_KEY=your_key

# Monitor your usage patterns
check_credits(includeDetails=true)

# Use appropriate timeouts
scrape_links(urls=["..."], timeout=30)  # Standard
scrape_links(urls=["..."], timeout=60)  # Slow sites

# Choose efficient modes
scrape_links(mode="basic")      # Try first
scrape_links(mode="javascript") # If content missing
scrape_links(mode="premium")    # If blocked

# Always pull before pushing (recommended workflow)
git pull --rebase origin main && git push origin main

# Use conventional commit format for proper versioning
git commit -m "feat: add new feature"    # triggers minor version bump
git commit -m "fix: resolve bug"        # triggers patch version bump
git commit -m "docs: update readme"     # no version bump