awswaf challenge http status 202 about scrapy-playwright HOT 6 OPEN

from scrapy.spiders import Spider
import re
import scrapy
from urllib.parse import urlencode
from playwright.async_api import async_playwright


class PlaywrightSpider(Spider):
    name = "test01"
    custom_settings = {
        "PLAYWRIGHT_BROWSER_TYPE": "chromium",
        "PLAYWRIGHT_LAUNCH_OPTIONS": {
            "headless": False,
            "timeout": 20 * 1000,  # 20 seconds
        }
    }
    allowed_domains = [
        "cn.classic.warcraftlogs.com", "classic.warcraftlogs.com"
    ]

    players = [67849152]
    char_url = "https://cn.classic.warcraftlogs.com/character/id/{0}?mode=detailed&zone=1020#metric=dps"
    char_detail_url = "https://cn.classic.warcraftlogs.com/character/rankings-raids/{id}/default/1002/3/5000/5000/Any/rankings/0/0?dpstype=rdps&class=Any&signature={sign}"

    start_urls = [
        "https://cn.classic.warcraftlogs.com/character/id/67849152?mode=detailed&zone=1020#metric=dps"
    ]  # avoid using the default Scrapy downloader

    def start_requests(self):
        for player in self.players:
            eas_url = self.char_url.format(player)

            yield scrapy.Request(
                eas_url,
                meta={
                    "playwright": True,
                    "playwright_include_page": False,
                    "playwright_context_kwargs": {
                        "java_script_enabled": True,
                        "ignore_https_errors": True,
                    },
                    'id': player,
                    'referer': eas_url,
                },
                headers={
                    'Referer':
                    eas_url,
                    'agent':
                    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36',
                    'accept':
                    'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
                },
                callback=self.parse_player,
                errback=self.errback_close_page,
            )

    def parse_pass(self, response):
        pass

    async def parse_player(self, response, **kwargs):

        # page = response.meta["playwright_page"]
        # title = await page.title()

        # await page.close()
        # await page.context.close()
        # print("11111111111111", response.text)

        if response.status == 202:
            print("challenge")
            return

        meta = response.meta

        regex = r"rankings-raids.*signature=' \+ '(.*)'"
        res = re.search(regex, response.text)

        sign = None
        if res:
            sign = res.group(1)
        else:
            self.logger.info("ref=%s res=%s resp:%s", meta["referer"], res,
                             response.text)
            return

        meta["id"] = "67849152"

        # print('_token={}'.format(response.css("meta[name=csrf-token]::attr(content)").get()))
        self.logger.info(
            "_token=%s",
            response.css("meta[name=csrf-token]::attr(content)").get())

        yield response.follow(
            self.char_detail_url.format(id=meta["id"], sign=sign),
            method='POST',
            body=urlencode({
                '_token':
                response.css("meta[name=csrf-token]::attr(content)").get()
            }),
            headers={
                'Referer': self.char_url,
                'Content-Type':
                'application/x-www-form-urlencoded; charset=UTF-8'
            },
            meta=response.meta,
            #   dont_filter=True,
            #   priority=0,
            callback=self.parse_detail)
        pass

    def parse(self, response, **kwargs):
        # 'response' contains the page as seen by the browser
        return {"url": response.url}

    def parse_detail(self, response):
        # print(response.text)
        pass

    async def errback_close_page(self, failure):
        print(failure)
        # await page.close()