python selenium get devtools request headers data

Question

I have tried many searches and attempts, but I haven't achieved the results I wanted.

I need devtools >> network >> Headers >> cookie data

help!!!!

1. Please let me know how to retrieve information from devtools >> network >> headers using Selenium in Python.
   OR

2. If there's a better method than Selenium, feel free to let me know (python)

    url = "http://url.com"
    driver.get(url)
    
    time.sleep(0.3)
    logs = driver.get_log("performance")

search google, youtube .....

Answer 1

If your browser and the machine in which the script is being executed are the same, then you can leverage the debugUrl of chromium based web browsers and use the Chrome DevTools Protocol to intercept the required url and capture the headers.

Here is an attempt of the same in python. It sends the Fetch.enable message with the urlPattern to make the browser aware of the request to intercepted. Captures the headers from the intercepted requests and continues the intercepted requests as intended.

from selenium import webdriver
from selenium.webdriver.edge.options import Options
from selenium.webdriver.edge.service import Service as EdgeService
from webdriver_manager.microsoft import EdgeChromiumDriverManager

import time
import requests
import websockets
import asyncio
import json

options = Options()
options.add_argument('--ignore-ssl-errors=yes')
options.add_argument("--remote-debugging-port=9222")
options.add_argument("--remote-allow-origins=*")
options.add_argument('--ignore-certificate-errors')
driver = webdriver.Edge(service=EdgeService(EdgeChromiumDriverManager().install()), options=options)

def get_debugger_url():
    time.sleep(2)
    response = requests.get('http://localhost:9222/json')
    response_data = response.json()
    print(response_data)
    debugger_url = response_data[0]['webSocketDebuggerUrl']
    return debugger_url

async def listen_to_chrome_debugger():

    debugger_url = get_debugger_url()

    async with websockets.connect(uri=debugger_url, origin="*") as websocket:
        print("Connected to Debugger")

        # Enable interception of request based on the urlPattern provided
        await websocket.send(json.dumps({"id": 1, "method": "Fetch.enable", "params": {"patterns" : [{"urlPattern" : "*"}]}}))

        async for message in websocket:
            event = json.loads(message)
            # print("Received event:", event)

            # When request is captured, the requestPaused event is raised
            if event.get("method") == "Fetch.requestPaused":
                # Capture the cookies, headers, requests, etc..
                print(event['params']['request']['headers'])
                request_id = event["params"]["requestId"]

                # Continue the requests as it is or make changes to the intercepted request
                await websocket.send(json.dumps({
                    "id": 2,
                    "method": "Fetch.continueRequest",
                    "params": {"requestId": request_id}
                }))

driver.get("http://www.example.com")
print("Page title:", driver.title)

asyncio.run(listen_to_chrome_debugger())

time.sleep(10)

driver.quit()
print("Driver quit")

Resource: https://chromedevtools.github.io/devtools-protocol/tot/Fetch/