How could I remove part of page content before sending to LLM? about scrapegraph-ai HOT 9 CLOSED

File "/Users/david/.pyenv/versions/3.10.13/envs/scraper/lib/python3.10/site-packages/openai/_base_client.py", line 921, in request
    return self._request(
  File "/Users/david/.pyenv/versions/3.10.13/envs/scraper/lib/python3.10/site-packages/openai/_base_client.py", line 1020, in _request
    raise self._make_status_error_from_response(err.response) from None
openai.BadRequestError: Error code: 400 - {'detail': "This model's maximum context length is 32768 tokens. However, you requested 36317 tokens (32221 in the messages, 4096 in the completion). Please reduce the length of the messages or completion."}

import os
from dotenv import load_dotenv
from scrapegraphai.graphs import SmartScraperGraph
from scrapegraphai.utils import prettify_exec_info

load_dotenv()

deepseek_key = os.getenv("DEEPSEEK_KEY") 
graph_config = {
   "llm": {
      "api_key": deepseek_key,
      "model": "deepseek-chat",
      "temperature": 0.7,
      "max_tokens": 4096, ## max output tokens limited to 4k for gpt-4o,gpt-4-turbo
      "base_url": "https://api.deepseek.com/v1"
   },
   "embeddings": {
      "model": "ollama/nomic-embed-text",
      "temperature": 0,
      "base_url": "http://localhost:11434",  # set ollama URL
   },
   "headless": False,
   "verbose": True,
}

# ************************************************
# Create the SmartScraperGraph instance and run it
# ************************************************

smart_scraper_graph = SmartScraperGraph(
   prompt="extract all page title and links under，footage-card-wrapper as format of: [{\"title\": \"xxx\", \"link\":\"xxx\" }] ",
   # also accepts a string with the already downloaded HTML code
   source="https://stock.xinpianchang.com/footages/2997636.html",
   config=graph_config
)

result = smart_scraper_graph.run()
print(result)

openai.BadRequestError: Error code: 400 - {'detail': "This model's maximum context length is 32768 tokens. However, you requested 36294 tokens (32198 in the messages, 4096 in the completion). Please reduce the length of the messages or completion."}