From d784732054c5fc5d34c9ddfecf70d39604feff54 Mon Sep 17 00:00:00 2001
From: vikas-gits-good <vikas.c.conappa@protonmail.com>
Date: Wed, 14 Jan 2026 22:21:58 +0530
Subject: [PATCH] Fix arun_many failing during deep crawling

---
 .claude/settings.local.json             |  28 --
 crawl4ai/async_configs.py               | 313 ++++++------
 crawl4ai/async_crawler_strategy.py      | 619 ++++++++++++++----------
 crawl4ai/async_dispatcher.py            | 285 ++++++-----
 crawl4ai/async_webcrawler.py            | 157 +++---
 crawl4ai/deep_crawling/base_strategy.py |  66 ++-
 crawl4ai/deep_crawling/bfs_strategy.py  |  78 +--
 crawl4ai/models.py                      | 108 +++--
 8 files changed, 968 insertions(+), 686 deletions(-)
 delete mode 100644 .claude/settings.local.json

diff --git a/.claude/settings.local.json b/.claude/settings.local.json
deleted file mode 100644
index 52e79a4fd..000000000
--- a/.claude/settings.local.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(cd:*)",
-      "Bash(python3:*)",
-      "Bash(python:*)",
-      "Bash(grep:*)",
-      "Bash(mkdir:*)",
-      "Bash(cp:*)",
-      "Bash(rm:*)",
-      "Bash(true)",
-      "Bash(./package-extension.sh:*)",
-      "Bash(find:*)",
-      "Bash(chmod:*)",
-      "Bash(rg:*)",
-      "Bash(/Users/unclecode/.npm-global/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg -A 5 -B 5 \"Script Builder\" docs/md_v2/apps/crawl4ai-assistant/)",
-      "Bash(/Users/unclecode/.npm-global/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg -A 30 \"generateCode\\(events, format\\)\" docs/md_v2/apps/crawl4ai-assistant/content/content.js)",
-      "Bash(/Users/unclecode/.npm-global/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg \"<style>\" docs/md_v2/apps/crawl4ai-assistant/index.html -A 5)",
-      "Bash(git checkout:*)",
-      "Bash(docker logs:*)",
-      "Bash(curl:*)",
-      "Bash(docker compose:*)",
-      "Bash(./test-final-integration.sh:*)",
-      "Bash(mv:*)"
-    ]
-  },
-  "enableAllProjectMcpServers": false
-}
\ No newline at end of file
diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py
index bfa0d398b..519b63670 100644
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@ -19,7 +19,10 @@
 from .extraction_strategy import ExtractionStrategy, LLMExtractionStrategy
 from .chunking_strategy import ChunkingStrategy, RegexChunking
 
-from .markdown_generation_strategy import MarkdownGenerationStrategy, DefaultMarkdownGenerator
+from .markdown_generation_strategy import (
+    MarkdownGenerationStrategy,
+    DefaultMarkdownGenerator,
+)
 from .content_scraping_strategy import ContentScrapingStrategy, LXMLWebScrapingStrategy
 from .deep_crawling import DeepCrawlStrategy
 from .table_extraction import TableExtractionStrategy, DefaultTableExtraction
@@ -35,15 +38,16 @@
 # Type alias for URL matching
 UrlMatcher = Union[str, Callable[[str], bool], List[Union[str, Callable[[str], bool]]]]
 
+
 class MatchMode(Enum):
     OR = "or"
     AND = "and"
 
-# from .proxy_strategy import ProxyConfig
 
+# from .proxy_strategy import ProxyConfig
 
 
-def to_serializable_dict(obj: Any, ignore_default_value : bool = False) -> Dict:
+def to_serializable_dict(obj: Any, ignore_default_value: bool = False) -> Dict:
     """
     Recursively convert an object to a serializable dictionary using {type, params} structure
     for complex objects.
@@ -64,7 +68,11 @@ def to_serializable_dict(obj: Any, ignore_default_value : bool = False) -> Dict:
         return obj.isoformat()
 
     # Handle lists, tuples, and sets, and basically any iterable
-    if isinstance(obj, (list, tuple, set)) or hasattr(obj, '__iter__') and not isinstance(obj, dict):
+    if (
+        isinstance(obj, (list, tuple, set))
+        or hasattr(obj, "__iter__")
+        and not isinstance(obj, dict)
+    ):
         return [to_serializable_dict(item) for item in obj]
 
     # Handle frozensets, which are not iterable
@@ -98,7 +106,7 @@ def to_serializable_dict(obj: Any, ignore_default_value : bool = False) -> Dict:
             if not (is_empty_value(value) and is_empty_value(param.default)):
                 if value != param.default and not ignore_default_value:
                     current_values[name] = to_serializable_dict(value)
-        
+
         # Don't serialize private __slots__ - they're internal implementation details
         # not constructor parameters. This was causing URLPatternFilter to fail
         # because _simple_suffixes was being serialized as 'simple_suffixes'
@@ -110,13 +118,8 @@ def to_serializable_dict(obj: Any, ignore_default_value : bool = False) -> Dict:
         #             if value is not None:
         #                 current_values[attr_name] = to_serializable_dict(value)
 
-            
-        
-        return {
-            "type": obj.__class__.__name__,
-            "params": current_values
-        }
-        
+        return {"type": obj.__class__.__name__, "params": current_values}
+
     return str(obj)
 
 
@@ -173,15 +176,13 @@ def is_empty_value(value: Any) -> bool:
         return True
     return False
 
+
 class GeolocationConfig:
     def __init__(
-        self,
-        latitude: float,
-        longitude: float,
-        accuracy: Optional[float] = 0.0
+        self, latitude: float, longitude: float, accuracy: Optional[float] = 0.0
     ):
         """Configuration class for geolocation settings.
-        
+
         Args:
             latitude: Latitude coordinate (e.g., 37.7749)
             longitude: Longitude coordinate (e.g., -122.4194)
@@ -190,24 +191,24 @@ def __init__(
         self.latitude = latitude
         self.longitude = longitude
         self.accuracy = accuracy
-    
+
     @staticmethod
     def from_dict(geo_dict: Dict) -> "GeolocationConfig":
         """Create a GeolocationConfig from a dictionary."""
         return GeolocationConfig(
             latitude=geo_dict.get("latitude"),
             longitude=geo_dict.get("longitude"),
-            accuracy=geo_dict.get("accuracy", 0.0)
+            accuracy=geo_dict.get("accuracy", 0.0),
         )
-    
+
     def to_dict(self) -> Dict:
         """Convert to dictionary representation."""
         return {
             "latitude": self.latitude,
             "longitude": self.longitude,
-            "accuracy": self.accuracy
+            "accuracy": self.accuracy,
         }
-    
+
     def clone(self, **kwargs) -> "GeolocationConfig":
         """Create a copy of this configuration with updated values.
 
@@ -221,6 +222,7 @@ def clone(self, **kwargs) -> "GeolocationConfig":
         config_dict.update(kwargs)
         return GeolocationConfig.from_dict(config_dict)
 
+
 class ProxyConfig:
     def __init__(
         self,
@@ -230,7 +232,7 @@ def __init__(
         ip: Optional[str] = None,
     ):
         """Configuration class for a single proxy.
-        
+
         Args:
             server: Proxy server URL (e.g., "http://127.0.0.1:8080")
             username: Optional username for proxy authentication
@@ -240,10 +242,10 @@ def __init__(
         self.server = server
         self.username = username
         self.password = password
-        
+
         # Extract IP from server if not explicitly provided
         self.ip = ip or self._extract_ip_from_server()
-    
+
     def _extract_ip_from_server(self) -> Optional[str]:
         """Extract IP address from server URL."""
         try:
@@ -256,7 +258,7 @@ def _extract_ip_from_server(self) -> Optional[str]:
                 return parts[0]
         except Exception:
             return None
-    
+
     @staticmethod
     def from_string(proxy_str: str) -> "ProxyConfig":
         """Create a ProxyConfig from a string.
@@ -287,12 +289,14 @@ def from_string(proxy_str: str) -> "ProxyConfig":
         parts = s.split(":")
         if len(parts) == 4:
             ip, port, username, password = parts
-            return ProxyConfig(server=f"http://{ip}:{port}", username=username, password=password)
+            return ProxyConfig(
+                server=f"http://{ip}:{port}", username=username, password=password
+            )
         if len(parts) == 2:
             ip, port = parts
             return ProxyConfig(server=f"http://{ip}:{port}")
         raise ValueError(f"Invalid proxy string format: {proxy_str}")
-    
+
     @staticmethod
     def from_dict(proxy_dict: Dict) -> "ProxyConfig":
         """Create a ProxyConfig from a dictionary."""
@@ -300,16 +304,16 @@ def from_dict(proxy_dict: Dict) -> "ProxyConfig":
             server=proxy_dict.get("server"),
             username=proxy_dict.get("username"),
             password=proxy_dict.get("password"),
-            ip=proxy_dict.get("ip")
+            ip=proxy_dict.get("ip"),
         )
-    
+
     @staticmethod
     def from_env(env_var: str = "PROXIES") -> List["ProxyConfig"]:
         """Load proxies from environment variable.
-        
+
         Args:
             env_var: Name of environment variable containing comma-separated proxy strings
-            
+
         Returns:
             List of ProxyConfig objects
         """
@@ -323,16 +327,16 @@ def from_env(env_var: str = "PROXIES") -> List["ProxyConfig"]:
         except Exception as e:
             print(f"Error loading proxies from environment: {e}")
         return proxies
-    
+
     def to_dict(self) -> Dict:
         """Convert to dictionary representation."""
         return {
             "server": self.server,
             "username": self.username,
             "password": self.password,
-            "ip": self.ip
+            "ip": self.ip,
         }
-    
+
     def clone(self, **kwargs) -> "ProxyConfig":
         """Create a copy of this configuration with updated values.
 
@@ -346,6 +350,7 @@ def clone(self, **kwargs) -> "ProxyConfig":
         config_dict.update(kwargs)
         return ProxyConfig.from_dict(config_dict)
 
+
 class BrowserConfig:
     """
     Configuration class for setting up a browser instance and its context in AsyncPlaywrightCrawlerStrategy.
@@ -455,9 +460,8 @@ def __init__(
         host: str = "localhost",
         enable_stealth: bool = False,
     ):
-        
         self.browser_type = browser_type
-        self.headless = headless 
+        self.headless = headless
         self.browser_mode = browser_mode
         self.use_managed_browser = use_managed_browser
         self.cdp_url = cdp_url
@@ -469,16 +473,22 @@ def __init__(
             self.channel = ""
             self.chrome_channel = ""
         if proxy:
-            warnings.warn("The 'proxy' parameter is deprecated and will be removed in a future release. Use 'proxy_config' instead.", UserWarning)
+            warnings.warn(
+                "The 'proxy' parameter is deprecated and will be removed in a future release. Use 'proxy_config' instead.",
+                UserWarning,
+            )
         self.proxy = proxy
         self.proxy_config = proxy_config
         if isinstance(self.proxy_config, dict):
             self.proxy_config = ProxyConfig.from_dict(self.proxy_config)
         if isinstance(self.proxy_config, str):
             self.proxy_config = ProxyConfig.from_string(self.proxy_config)
-        
+
         if self.proxy and self.proxy_config:
-            warnings.warn("Both 'proxy' and 'proxy_config' are provided. 'proxy_config' will take precedence.", UserWarning)
+            warnings.warn(
+                "Both 'proxy' and 'proxy_config' are provided. 'proxy_config' will take precedence.",
+                UserWarning,
+            )
             self.proxy = None
         elif self.proxy:
             # Convert proxy string to ProxyConfig if proxy_config is not provided
@@ -540,9 +550,13 @@ def __init__(
         # If persistent context is requested, ensure managed browser is enabled
         if self.use_persistent_context:
             self.use_managed_browser = True
-            
+
         # Validate stealth configuration
-        if self.enable_stealth and self.use_managed_browser and self.browser_mode == "builtin":
+        if (
+            self.enable_stealth
+            and self.use_managed_browser
+            and self.browser_mode == "builtin"
+        ):
             raise ValueError(
                 "enable_stealth cannot be used with browser_mode='builtin'. "
                 "Stealth mode requires a dedicated browser instance."
@@ -621,7 +635,6 @@ def to_dict(self):
             "enable_stealth": self.enable_stealth,
         }
 
-                
         return result
 
     def clone(self, **kwargs):
@@ -711,10 +724,14 @@ def set_nstproxy(
 
             # --- Handle API error response ---
             if isinstance(data, dict) and data.get("err"):
-                raise PermissionError(f"[NSTProxy] API Error: {data.get('msg', 'Unknown error')}")
+                raise PermissionError(
+                    f"[NSTProxy] API Error: {data.get('msg', 'Unknown error')}"
+                )
 
             if not isinstance(data, list) or not data:
-                raise ValueError("[NSTProxy] Invalid API response — expected a non-empty list")
+                raise ValueError(
+                    "[NSTProxy] Invalid API response — expected a non-empty list"
+                )
 
             proxy_info = data[0]
 
@@ -729,13 +746,14 @@ def set_nstproxy(
             print(f"[NSTProxy] ❌ Failed to set proxy: {e}")
             raise
 
+
 class VirtualScrollConfig:
     """Configuration for virtual scroll handling.
-    
+
     This config enables capturing content from pages with virtualized scrolling
     (like Twitter, Instagram feeds) where DOM elements are recycled as user scrolls.
     """
-    
+
     def __init__(
         self,
         container_selector: str,
@@ -745,13 +763,13 @@ def __init__(
     ):
         """
         Initialize virtual scroll configuration.
-        
+
         Args:
             container_selector: CSS selector for the scrollable container
             scroll_count: Maximum number of scrolls to perform
             scroll_by: Amount to scroll - can be:
                 - "container_height": scroll by container's height
-                - "page_height": scroll by viewport height  
+                - "page_height": scroll by viewport height
                 - int: fixed pixel amount
             wait_after_scroll: Seconds to wait after each scroll for content to load
         """
@@ -759,7 +777,7 @@ def __init__(
         self.scroll_count = scroll_count
         self.scroll_by = scroll_by
         self.wait_after_scroll = wait_after_scroll
-    
+
     def to_dict(self) -> dict:
         """Convert to dictionary for serialization."""
         return {
@@ -768,15 +786,16 @@ def to_dict(self) -> dict:
             "scroll_by": self.scroll_by,
             "wait_after_scroll": self.wait_after_scroll,
         }
-    
+
     @classmethod
     def from_dict(cls, data: dict) -> "VirtualScrollConfig":
         """Create instance from dictionary."""
         return cls(**data)
 
+
 class LinkPreviewConfig:
     """Configuration for link head extraction and scoring."""
-    
+
     def __init__(
         self,
         include_internal: bool = True,
@@ -788,14 +807,14 @@ def __init__(
         max_links: int = 100,
         query: Optional[str] = None,
         score_threshold: Optional[float] = None,
-        verbose: bool = False
+        verbose: bool = False,
     ):
         """
         Initialize link extraction configuration.
-        
+
         Args:
             include_internal: Whether to include same-domain links
-            include_external: Whether to include different-domain links  
+            include_external: Whether to include different-domain links
             include_patterns: List of glob patterns to include (e.g., ["*/docs/*", "*/api/*"])
             exclude_patterns: List of glob patterns to exclude (e.g., ["*/login*", "*/admin*"])
             concurrency: Number of links to process simultaneously
@@ -815,7 +834,7 @@ def __init__(
         self.query = query
         self.score_threshold = score_threshold
         self.verbose = verbose
-        
+
         # Validation
         if concurrency <= 0:
             raise ValueError("concurrency must be positive")
@@ -826,14 +845,16 @@ def __init__(
         if score_threshold is not None and not (0.0 <= score_threshold <= 1.0):
             raise ValueError("score_threshold must be between 0.0 and 1.0")
         if not include_internal and not include_external:
-            raise ValueError("At least one of include_internal or include_external must be True")
-    
+            raise ValueError(
+                "At least one of include_internal or include_external must be True"
+            )
+
     @staticmethod
     def from_dict(config_dict: Dict[str, Any]) -> "LinkPreviewConfig":
         """Create LinkPreviewConfig from dictionary (for backward compatibility)."""
         if not config_dict:
             return None
-        
+
         return LinkPreviewConfig(
             include_internal=config_dict.get("include_internal", True),
             include_external=config_dict.get("include_external", False),
@@ -844,9 +865,9 @@ def from_dict(config_dict: Dict[str, Any]) -> "LinkPreviewConfig":
             max_links=config_dict.get("max_links", 100),
             query=config_dict.get("query"),
             score_threshold=config_dict.get("score_threshold"),
-            verbose=config_dict.get("verbose", False)
+            verbose=config_dict.get("verbose", False),
         )
-    
+
     def to_dict(self) -> Dict[str, Any]:
         """Convert to dictionary format."""
         return {
@@ -859,9 +880,9 @@ def to_dict(self) -> Dict[str, Any]:
             "max_links": self.max_links,
             "query": self.query,
             "score_threshold": self.score_threshold,
-            "verbose": self.verbose
+            "verbose": self.verbose,
         }
-    
+
     def clone(self, **kwargs) -> "LinkPreviewConfig":
         """Create a copy with updated values."""
         config_dict = self.to_dict()
@@ -939,8 +960,8 @@ def load(data: dict) -> "HTTPCrawlerConfig":
             return config
         return HTTPCrawlerConfig.from_kwargs(config)
 
-class CrawlerRunConfig():
 
+class CrawlerRunConfig:
     """
     Configuration class for controlling how the crawler runs each crawl operation.
     This includes parameters for content extraction, page manipulation, waiting conditions,
@@ -966,13 +987,13 @@ class CrawlerRunConfig():
                           Default: False.
         css_selector (str or None): CSS selector to extract a specific portion of the page.
                                     Default: None.
-        
-        target_elements (list of str or None): List of CSS selectors for specific elements for Markdown generation 
-                                                and structured data extraction. When you set this, only the contents 
-                                                of these elements are processed for extraction and Markdown generation. 
-                                                If you do not set any value, the entire page is processed. 
-                                                The difference between this and css_selector is that this will shrink 
-                                                the initial raw HTML to the selected element, while this will only affect 
+
+        target_elements (list of str or None): List of CSS selectors for specific elements for Markdown generation
+                                                and structured data extraction. When you set this, only the contents
+                                                of these elements are processed for extraction and Markdown generation.
+                                                If you do not set any value, the entire page is processed.
+                                                The difference between this and css_selector is that this will shrink
+                                                the initial raw HTML to the selected element, while this will only affect
                                                 the extraction and Markdown generation.
                                     Default: None
         excluded_tags (list of str or None): List of HTML tags to exclude from processing.
@@ -1091,7 +1112,7 @@ class CrawlerRunConfig():
 
         # Virtual Scroll Parameters
         virtual_scroll_config (VirtualScrollConfig or dict or None): Configuration for handling virtual scroll containers.
-                                                                     Used for capturing content from pages with virtualized 
+                                                                     Used for capturing content from pages with virtualized
                                                                      scrolling (e.g., Twitter, Instagram feeds).
                                                                      Default: None.
 
@@ -1140,17 +1161,18 @@ class CrawlerRunConfig():
 
         # Experimental Parameters
         experimental (dict): Dictionary containing experimental parameters that are in beta phase.
-                            This allows passing temporary features that are not yet fully integrated 
+                            This allows passing temporary features that are not yet fully integrated
                             into the main parameter set.
                             Default: None.
 
         url: str = None  # This is not a compulsory parameter
     """
+
     _UNWANTED_PROPS = {
-        'disable_cache' : 'Instead, use cache_mode=CacheMode.DISABLED',
-        'bypass_cache' : 'Instead, use cache_mode=CacheMode.BYPASS',
-        'no_cache_read' : 'Instead, use cache_mode=CacheMode.WRITE_ONLY',
-        'no_cache_write' : 'Instead, use cache_mode=CacheMode.READ_ONLY',
+        "disable_cache": "Instead, use cache_mode=CacheMode.DISABLED",
+        "bypass_cache": "Instead, use cache_mode=CacheMode.BYPASS",
+        "no_cache_read": "Instead, use cache_mode=CacheMode.WRITE_ONLY",
+        "no_cache_write": "Instead, use cache_mode=CacheMode.READ_ONLY",
     }
 
     def __init__(
@@ -1283,7 +1305,7 @@ def __init__(
             self.proxy_config = ProxyConfig.from_string(proxy_config)
 
         self.proxy_rotation_strategy = proxy_rotation_strategy
-        
+
         # Browser Location and Identity Parameters
         self.locale = locale
         self.timezone_id = timezone_id
@@ -1338,10 +1360,12 @@ def __init__(
         self.exclude_external_images = exclude_external_images
         self.exclude_all_images = exclude_all_images
         self.table_score_threshold = table_score_threshold
-        
+
         # Table extraction strategy (default to DefaultTableExtraction if not specified)
         if table_extraction is None:
-            self.table_extraction = DefaultTableExtraction(table_score_threshold=table_score_threshold)
+            self.table_extraction = DefaultTableExtraction(
+                table_score_threshold=table_score_threshold
+            )
         else:
             self.table_extraction = table_extraction
 
@@ -1359,7 +1383,7 @@ def __init__(
         # Debugging and Logging Parameters
         self.verbose = verbose
         self.log_console = log_console
-        
+
         # Network and Console Capturing Parameters
         self.capture_network_requests = capture_network_requests
         self.capture_console_messages = capture_console_messages
@@ -1396,7 +1420,7 @@ def __init__(
 
         # Deep Crawl Parameters
         self.deep_crawl_strategy = deep_crawl_strategy
-        
+
         # Link Extraction Parameters
         if link_preview_config is None:
             self.link_preview_config = None
@@ -1406,8 +1430,10 @@ def __init__(
             # Convert dict to config object for backward compatibility
             self.link_preview_config = LinkPreviewConfig.from_dict(link_preview_config)
         else:
-            raise ValueError("link_preview_config must be LinkPreviewConfig object or dict")
-        
+            raise ValueError(
+                "link_preview_config must be LinkPreviewConfig object or dict"
+            )
+
         # Virtual Scroll Parameters
         if virtual_scroll_config is None:
             self.virtual_scroll_config = None
@@ -1415,22 +1441,25 @@ def __init__(
             self.virtual_scroll_config = virtual_scroll_config
         elif isinstance(virtual_scroll_config, dict):
             # Convert dict to config object for backward compatibility
-            self.virtual_scroll_config = VirtualScrollConfig.from_dict(virtual_scroll_config)
+            self.virtual_scroll_config = VirtualScrollConfig.from_dict(
+                virtual_scroll_config
+            )
         else:
-            raise ValueError("virtual_scroll_config must be VirtualScrollConfig object or dict")
-        
+            raise ValueError(
+                "virtual_scroll_config must be VirtualScrollConfig object or dict"
+            )
+
         # URL Matching Parameters
         self.url_matcher = url_matcher
         self.match_mode = match_mode
-        
+
         # Experimental Parameters
         self.experimental = experimental or {}
-        
+
         # Compile C4A scripts if provided
         if self.c4a_script and not self.js_code:
             self._compile_c4a_script()
 
-
     def _compile_c4a_script(self):
         """Compile C4A script to JavaScript"""
         try:
@@ -1439,35 +1468,35 @@ def _compile_c4a_script(self):
                 from .script import compile
             except ImportError:
                 from crawl4ai.script import compile
-                
+
             # Handle both string and list inputs
             if isinstance(self.c4a_script, str):
                 scripts = [self.c4a_script]
             else:
                 scripts = self.c4a_script
-                
+
             # Compile each script
             compiled_js = []
             for i, script in enumerate(scripts):
                 result = compile(script)
-                
+
                 if result.success:
                     compiled_js.extend(result.js_code)
                 else:
                     # Format error message following existing patterns
                     error = result.first_error
                     error_msg = (
-                        f"C4A Script compilation error (script {i+1}):\n"
+                        f"C4A Script compilation error (script {i + 1}):\n"
                         f"  Line {error.line}, Column {error.column}: {error.message}\n"
                         f"  Code: {error.source_line}"
                     )
                     if error.suggestions:
                         error_msg += f"\n  Suggestion: {error.suggestions[0].message}"
-                        
+
                     raise ValueError(error_msg)
-                    
+
             self.js_code = compiled_js
-            
+
         except ImportError:
             raise ValueError(
                 "C4A script compiler not available. "
@@ -1478,57 +1507,60 @@ def _compile_c4a_script(self):
             if "compilation error" not in str(e).lower():
                 raise ValueError(f"Failed to compile C4A script: {str(e)}")
             raise
-    
+
     def is_match(self, url: str) -> bool:
         """Check if this config matches the given URL.
-        
+
         Args:
             url: The URL to check against this config's matcher
-            
+
         Returns:
             bool: True if this config should be used for the URL or if no matcher is set.
         """
         if self.url_matcher is None:
             return True
-            
+
         if callable(self.url_matcher):
             # Single function matcher
             return self.url_matcher(url)
-        
+
         elif isinstance(self.url_matcher, str):
             # Single pattern string
             from fnmatch import fnmatch
+
             return fnmatch(url, self.url_matcher)
-        
+
         elif isinstance(self.url_matcher, list):
             # List of mixed matchers
             if not self.url_matcher:  # Empty list
                 return False
-                
+
             results = []
             for matcher in self.url_matcher:
                 if callable(matcher):
                     results.append(matcher(url))
                 elif isinstance(matcher, str):
                     from fnmatch import fnmatch
+
                     results.append(fnmatch(url, matcher))
                 else:
                     # Skip invalid matchers
                     continue
-            
+
             # Apply match mode logic
             if self.match_mode == MatchMode.OR:
                 return any(results) if results else False
             else:  # AND mode
                 return all(results) if results else False
-        
-        return False
 
+        return False
 
     def __getattr__(self, name):
         """Handle attribute access."""
         if name in self._UNWANTED_PROPS:
-            raise AttributeError(f"Getting '{name}' is deprecated. {self._UNWANTED_PROPS[name]}")
+            raise AttributeError(
+                f"Getting '{name}' is deprecated. {self._UNWANTED_PROPS[name]}"
+            )
         raise AttributeError(f"'{self.__class__.__name__}' has no attribute '{name}'")
 
     def __setattr__(self, name, value):
@@ -1538,8 +1570,10 @@ def __setattr__(self, name, value):
         all_params = sig.parameters  # Dictionary of parameter names and their details
 
         if name in self._UNWANTED_PROPS and value is not all_params[name].default:
-            raise AttributeError(f"Setting '{name}' is deprecated. {self._UNWANTED_PROPS[name]}")
-        
+            raise AttributeError(
+                f"Setting '{name}' is deprecated. {self._UNWANTED_PROPS[name]}"
+            )
+
         super().__setattr__(name, value)
 
     @staticmethod
@@ -1628,7 +1662,9 @@ def from_kwargs(kwargs: dict) -> "CrawlerRunConfig":
             exclude_domains=kwargs.get("exclude_domains", []),
             exclude_internal_links=kwargs.get("exclude_internal_links", False),
             score_links=kwargs.get("score_links", False),
-            preserve_https_for_internal_links=kwargs.get("preserve_https_for_internal_links", False),
+            preserve_https_for_internal_links=kwargs.get(
+                "preserve_https_for_internal_links", False
+            ),
             # Debugging and Logging Parameters
             verbose=kwargs.get("verbose", True),
             log_console=kwargs.get("log_console", False),
@@ -1650,7 +1686,7 @@ def from_kwargs(kwargs: dict) -> "CrawlerRunConfig":
             # URL Matching Parameters
             url_matcher=kwargs.get("url_matcher"),
             match_mode=kwargs.get("match_mode", MatchMode.OR),
-            # Experimental Parameters 
+            # Experimental Parameters
             experimental=kwargs.get("experimental"),
         )
 
@@ -1747,7 +1783,9 @@ def to_dict(self):
             "user_agent_mode": self.user_agent_mode,
             "user_agent_generator_config": self.user_agent_generator_config,
             "deep_crawl_strategy": self.deep_crawl_strategy,
-            "link_preview_config": self.link_preview_config.to_dict() if self.link_preview_config else None,
+            "link_preview_config": self.link_preview_config.to_dict()
+            if self.link_preview_config
+            else None,
             "url": self.url,
             "url_matcher": self.url_matcher,
             "match_mode": self.match_mode,
@@ -1780,6 +1818,7 @@ def clone(self, **kwargs):
         config_dict.update(kwargs)
         return CrawlerRunConfig.from_kwargs(config_dict)
 
+
 class LLMConfig:
     def __init__(
         self,
@@ -1792,7 +1831,7 @@ def __init__(
         frequency_penalty: Optional[float] = None,
         presence_penalty: Optional[float] = None,
         stop: Optional[List[str]] = None,
-        n: Optional[int] = None,    
+        n: Optional[int] = None,
     ):
         """Configuaration class for LLM provider and API token."""
         self.provider = provider
@@ -1809,7 +1848,7 @@ def __init__(
                     (prefix for prefix in prefixes if provider.startswith(prefix)),
                     None,
                 )
-                self.api_token = PROVIDER_MODELS_PREFIXES.get(selected_prefix)                    
+                self.api_token = PROVIDER_MODELS_PREFIXES.get(selected_prefix)
             else:
                 self.provider = DEFAULT_PROVIDER
                 self.api_token = os.getenv(DEFAULT_PROVIDER_API_KEY)
@@ -1834,7 +1873,7 @@ def from_kwargs(kwargs: dict) -> "LLMConfig":
             frequency_penalty=kwargs.get("frequency_penalty"),
             presence_penalty=kwargs.get("presence_penalty"),
             stop=kwargs.get("stop"),
-            n=kwargs.get("n")
+            n=kwargs.get("n"),
         )
 
     def to_dict(self):
@@ -1848,7 +1887,7 @@ def to_dict(self):
             "frequency_penalty": self.frequency_penalty,
             "presence_penalty": self.presence_penalty,
             "stop": self.stop,
-            "n": self.n
+            "n": self.n,
         }
 
     def clone(self, **kwargs):
@@ -1864,10 +1903,12 @@ def clone(self, **kwargs):
         config_dict.update(kwargs)
         return LLMConfig.from_kwargs(config_dict)
 
+
 class SeedingConfig:
     """
     Configuration class for URL discovery and pre-validation via AsyncUrlSeeder.
     """
+
     def __init__(
         self,
         source: str = "sitemap+cc",
@@ -1888,37 +1929,37 @@ def __init__(
     ):
         """
         Initialize URL seeding configuration.
-        
+
         Args:
-            source: Discovery source(s) to use. Options: "sitemap", "cc" (Common Crawl), 
+            source: Discovery source(s) to use. Options: "sitemap", "cc" (Common Crawl),
                    or "sitemap+cc" (both). Default: "sitemap+cc"
-            pattern: URL pattern to filter discovered URLs (e.g., "*example.com/blog/*"). 
+            pattern: URL pattern to filter discovered URLs (e.g., "*example.com/blog/*").
                     Supports glob-style wildcards. Default: "*" (all URLs)
-            live_check: Whether to perform HEAD requests to verify URL liveness. 
+            live_check: Whether to perform HEAD requests to verify URL liveness.
                        Default: False
             extract_head: Whether to fetch and parse <head> section for metadata extraction.
                          Required for BM25 relevance scoring. Default: False
-            max_urls: Maximum number of URLs to discover. Use -1 for no limit. 
+            max_urls: Maximum number of URLs to discover. Use -1 for no limit.
                      Default: -1
-            concurrency: Maximum concurrent requests for live checks/head extraction. 
+            concurrency: Maximum concurrent requests for live checks/head extraction.
                         Default: 1000
-            hits_per_sec: Rate limit in requests per second to avoid overwhelming servers. 
+            hits_per_sec: Rate limit in requests per second to avoid overwhelming servers.
                          Default: 5
-            force: If True, bypasses the AsyncUrlSeeder's internal .jsonl cache and 
+            force: If True, bypasses the AsyncUrlSeeder's internal .jsonl cache and
                   re-fetches URLs. Default: False
-            base_directory: Base directory for UrlSeeder's cache files (.jsonl). 
+            base_directory: Base directory for UrlSeeder's cache files (.jsonl).
                            If None, uses default ~/.crawl4ai/. Default: None
-            llm_config: LLM configuration for future features (e.g., semantic scoring). 
+            llm_config: LLM configuration for future features (e.g., semantic scoring).
                        Currently unused. Default: None
-            verbose: Override crawler's general verbose setting for seeding operations. 
+            verbose: Override crawler's general verbose setting for seeding operations.
                     Default: None (inherits from crawler)
-            query: Search query for BM25 relevance scoring (e.g., "python tutorials"). 
+            query: Search query for BM25 relevance scoring (e.g., "python tutorials").
                   Requires extract_head=True. Default: None
-            score_threshold: Minimum relevance score (0.0-1.0) to include URL. 
+            score_threshold: Minimum relevance score (0.0-1.0) to include URL.
                            Only applies when query is provided. Default: None
-            scoring_method: Scoring algorithm to use. Currently only "bm25" is supported. 
+            scoring_method: Scoring algorithm to use. Currently only "bm25" is supported.
                           Future: "semantic". Default: "bm25"
-            filter_nonsense_urls: Filter out utility URLs like robots.txt, sitemap.xml, 
+            filter_nonsense_urls: Filter out utility URLs like robots.txt, sitemap.xml,
                                  ads.txt, favicon.ico, etc. Default: True
         """
         self.source = source
@@ -1939,13 +1980,15 @@ def __init__(
 
     # Add to_dict, from_kwargs, and clone methods for consistency
     def to_dict(self) -> Dict[str, Any]:
-        return {k: v for k, v in self.__dict__.items() if k != 'llm_config' or v is not None}
+        return {
+            k: v for k, v in self.__dict__.items() if k != "llm_config" or v is not None
+        }
 
     @staticmethod
-    def from_kwargs(kwargs: Dict[str, Any]) -> 'SeedingConfig':
+    def from_kwargs(kwargs: Dict[str, Any]) -> "SeedingConfig":
         return SeedingConfig(**kwargs)
 
-    def clone(self, **kwargs: Any) -> 'SeedingConfig':
+    def clone(self, **kwargs: Any) -> "SeedingConfig":
         config_dict = self.to_dict()
         config_dict.update(kwargs)
         return SeedingConfig.from_kwargs(config_dict)
diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py
index 76977bb95..9e0c705c8 100644
--- a/crawl4ai/async_crawler_strategy.py
+++ b/crawl4ai/async_crawler_strategy.py
@@ -32,6 +32,7 @@
 import contextlib
 from functools import partial
 
+
 class AsyncCrawlerStrategy(ABC):
     """
     Abstract base class for crawler strategies.
@@ -42,6 +43,7 @@ class AsyncCrawlerStrategy(ABC):
     async def crawl(self, url: str, **kwargs) -> AsyncCrawlResponse:
         pass  # 4 + 3
 
+
 class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
     """
     Crawler strategy using Playwright.
@@ -72,7 +74,11 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
     """
 
     def __init__(
-        self, browser_config: BrowserConfig = None, logger: AsyncLogger = None, browser_adapter: BrowserAdapter = None, **kwargs
+        self,
+        browser_config: BrowserConfig = None,
+        logger: AsyncLogger = None,
+        browser_adapter: BrowserAdapter = None,
+        **kwargs,
     ):
         """
         Initialize the AsyncPlaywrightCrawlerStrategy with a browser configuration.
@@ -88,7 +94,7 @@ def __init__(
         # Initialize browser config, either from provided object or kwargs
         self.browser_config = browser_config or BrowserConfig.from_kwargs(kwargs)
         self.logger = logger
-        
+
         # Initialize browser adapter
         self.adapter = browser_adapter or PlaywrightAdapter()
 
@@ -110,9 +116,9 @@ def __init__(
 
         # Initialize browser manager with config
         self.browser_manager = BrowserManager(
-            browser_config=self.browser_config, 
+            browser_config=self.browser_config,
             logger=self.logger,
-            use_undetected=isinstance(self.adapter, UndetectedAdapter)
+            use_undetected=isinstance(self.adapter, UndetectedAdapter),
         )
 
     async def __aenter__(self):
@@ -375,7 +381,8 @@ async def process_iframes(self, page):
 
                     # Replace the iframe with a div containing the extracted content
                     _iframe = iframe_content.replace("`", "\\`")
-                    await self.adapter.evaluate(page,
+                    await self.adapter.evaluate(
+                        page,
                         f"""
                         () => {{
                             const iframe = document.getElementById('iframe-{i}');
@@ -384,7 +391,7 @@ async def process_iframes(self, page):
                             div.className = '{class_name}';
                             iframe.replaceWith(div);
                         }}
-                    """
+                    """,
                     )
                 else:
                     self.logger.warning(
@@ -419,11 +426,13 @@ async def create_session(self, **kwargs) -> str:
 
         user_agent = kwargs.get("user_agent", self.user_agent)
         # Use browser_manager to get a fresh page & context assigned to this session_id
-        page, context = await self.browser_manager.get_page(CrawlerRunConfig(
-            session_id=session_id,
-            user_agent=user_agent,
-            **kwargs,
-        ))
+        page, context = await self.browser_manager.get_page(
+            CrawlerRunConfig(
+                session_id=session_id,
+                user_agent=user_agent,
+                **kwargs,
+            )
+        )
         return session_id
 
     async def crawl(
@@ -455,7 +464,7 @@ async def crawl(
         elif url.startswith("file://"):
             # initialize empty lists for console messages
             captured_console = []
-            
+
             # Process local file
             local_file_path = url[7:]  # Remove 'file://' prefix
             if not os.path.exists(local_file_path):
@@ -465,7 +474,9 @@ async def crawl(
             if config.screenshot:
                 screenshot_data = await self._generate_screenshot_from_html(html)
             if config.capture_console_messages:
-                page, context = await self.browser_manager.get_page(crawlerRunConfig=config)
+                page, context = await self.browser_manager.get_page(
+                    crawlerRunConfig=config
+                )
                 captured_console = await self._capture_console_messages(page, url)
 
             return AsyncCrawlResponse(
@@ -477,7 +488,7 @@ async def crawl(
                 console_messages=captured_console,
             )
 
-        ##### 
+        #####
         # Since both "raw:" and "raw://" start with "raw:", the first condition is always true for both, so "raw://" will be sliced as "//...", which is incorrect.
         # Fix: Check for "raw://" first, then "raw:"
         # Also, the prefix "raw://" is actually 6 characters long, not 7, so it should be sliced accordingly: url[6:]
@@ -519,11 +530,11 @@ async def _crawl_web(
         response_headers = {}
         execution_result = None
         status_code = None
-        redirected_url = url 
+        redirected_url = url
 
         # Reset downloaded files list for new crawl
         self._downloaded_files = []
-        
+
         # Initialize capture lists
         captured_requests = []
         captured_console = []
@@ -552,10 +563,13 @@ async def _crawl_web(
             await context.add_init_script(load_js_script("navigator_overrider"))
 
         # Call hook after page creation
-        await self.execute_hook("on_page_context_created", page, context=context, config=config)
+        await self.execute_hook(
+            "on_page_context_created", page, context=context, config=config
+        )
 
         # Network Request Capturing
         if config.capture_network_requests:
+
             async def handle_request_capture(request):
                 try:
                     post_data_str = None
@@ -563,28 +577,42 @@ async def handle_request_capture(request):
                         # Be cautious with large post data
                         post_data = request.post_data_buffer
                         if post_data:
-                             # Attempt to decode, fallback to base64 or size indication
-                             try:
-                                 post_data_str = post_data.decode('utf-8', errors='replace')
-                             except UnicodeDecodeError:
-                                 post_data_str = f"[Binary data: {len(post_data)} bytes]"
+                            # Attempt to decode, fallback to base64 or size indication
+                            try:
+                                post_data_str = post_data.decode(
+                                    "utf-8", errors="replace"
+                                )
+                            except UnicodeDecodeError:
+                                post_data_str = f"[Binary data: {len(post_data)} bytes]"
                     except Exception:
                         post_data_str = "[Error retrieving post data]"
 
-                    captured_requests.append({
-                        "event_type": "request",
-                        "url": request.url,
-                        "method": request.method,
-                        "headers": dict(request.headers), # Convert Header dict
-                        "post_data": post_data_str,
-                        "resource_type": request.resource_type,
-                        "is_navigation_request": request.is_navigation_request(),
-                        "timestamp": time.time()
-                    })
+                    captured_requests.append(
+                        {
+                            "event_type": "request",
+                            "url": request.url,
+                            "method": request.method,
+                            "headers": dict(request.headers),  # Convert Header dict
+                            "post_data": post_data_str,
+                            "resource_type": request.resource_type,
+                            "is_navigation_request": request.is_navigation_request(),
+                            "timestamp": time.time(),
+                        }
+                    )
                 except Exception as e:
                     if self.logger:
-                        self.logger.warning(f"Error capturing request details for {request.url}: {e}", tag="CAPTURE")
-                    captured_requests.append({"event_type": "request_capture_error", "url": request.url, "error": str(e), "timestamp": time.time()})
+                        self.logger.warning(
+                            f"Error capturing request details for {request.url}: {e}",
+                            tag="CAPTURE",
+                        )
+                    captured_requests.append(
+                        {
+                            "event_type": "request_capture_error",
+                            "url": request.url,
+                            "error": str(e),
+                            "timestamp": time.time(),
+                        }
+                    )
 
             async def handle_response_capture(response):
                 try:
@@ -596,40 +624,66 @@ async def handle_response_capture(response):
                         body = None
                         # json_body = None
                         # text_body = None
-                    captured_requests.append({
-                        "event_type": "response",
-                        "url": response.url,
-                        "status": response.status,
-                        "status_text": response.status_text,
-                        "headers": dict(response.headers), # Convert Header dict
-                        "from_service_worker": response.from_service_worker,
-                        "request_timing": response.request.timing, # Detailed timing info
-                        "timestamp": time.time(),
-                        "body" : {
-                            # "raw": body,
-                            # "json": json_body,
-                            "text": text_body
+                    captured_requests.append(
+                        {
+                            "event_type": "response",
+                            "url": response.url,
+                            "status": response.status,
+                            "status_text": response.status_text,
+                            "headers": dict(response.headers),  # Convert Header dict
+                            "from_service_worker": response.from_service_worker,
+                            "request_timing": response.request.timing,  # Detailed timing info
+                            "timestamp": time.time(),
+                            "body": {
+                                # "raw": body,
+                                # "json": json_body,
+                                "text": text_body
+                            },
                         }
-                    })
+                    )
                 except Exception as e:
                     if self.logger:
-                        self.logger.warning(f"Error capturing response details for {response.url}: {e}", tag="CAPTURE")
-                    captured_requests.append({"event_type": "response_capture_error", "url": response.url, "error": str(e), "timestamp": time.time()})
+                        self.logger.warning(
+                            f"Error capturing response details for {response.url}: {e}",
+                            tag="CAPTURE",
+                        )
+                    captured_requests.append(
+                        {
+                            "event_type": "response_capture_error",
+                            "url": response.url,
+                            "error": str(e),
+                            "timestamp": time.time(),
+                        }
+                    )
 
             async def handle_request_failed_capture(request):
-                 try:
-                    captured_requests.append({
-                        "event_type": "request_failed",
-                        "url": request.url,
-                        "method": request.method,
-                        "resource_type": request.resource_type,
-                        "failure_text": str(request.failure) if request.failure else "Unknown failure",
-                        "timestamp": time.time()
-                    })
-                 except Exception as e:
+                try:
+                    captured_requests.append(
+                        {
+                            "event_type": "request_failed",
+                            "url": request.url,
+                            "method": request.method,
+                            "resource_type": request.resource_type,
+                            "failure_text": str(request.failure)
+                            if request.failure
+                            else "Unknown failure",
+                            "timestamp": time.time(),
+                        }
+                    )
+                except Exception as e:
                     if self.logger:
-                        self.logger.warning(f"Error capturing request failed details for {request.url}: {e}", tag="CAPTURE")
-                    captured_requests.append({"event_type": "request_failed_capture_error", "url": request.url, "error": str(e), "timestamp": time.time()})
+                        self.logger.warning(
+                            f"Error capturing request failed details for {request.url}: {e}",
+                            tag="CAPTURE",
+                        )
+                    captured_requests.append(
+                        {
+                            "event_type": "request_failed_capture_error",
+                            "url": request.url,
+                            "error": str(e),
+                            "timestamp": time.time(),
+                        }
+                    )
 
             page.on("request", handle_request_capture)
             page.on("response", handle_response_capture)
@@ -640,8 +694,12 @@ async def handle_request_failed_capture(request):
         handle_error = None
         if config.capture_console_messages:
             # Set up console capture using adapter
-            handle_console = await self.adapter.setup_console_capture(page, captured_console)
-            handle_error = await self.adapter.setup_error_capture(page, captured_console)
+            handle_console = await self.adapter.setup_console_capture(
+                page, captured_console
+            )
+            handle_error = await self.adapter.setup_error_capture(
+                page, captured_console
+            )
 
         # Set up console logging if requested
         # Note: For undetected browsers, console logging won't work directly
@@ -664,7 +722,9 @@ async def handle_request_failed_capture(request):
 
             # Handle page navigation and content loading
             if not config.js_only:
-                await self.execute_hook("before_goto", page, context=context, url=url, config=config)
+                await self.execute_hook(
+                    "before_goto", page, context=context, url=url, config=config
+                )
 
                 try:
                     # Generate a unique nonce for this request
@@ -685,7 +745,10 @@ async def handle_request_failed_capture(request):
                 except Error as e:
                     # Allow navigation to be aborted when downloading files
                     # This is expected behavior for downloads in some browser engines
-                    if 'net::ERR_ABORTED' in str(e) and self.browser_config.accept_downloads:
+                    if (
+                        "net::ERR_ABORTED" in str(e)
+                        and self.browser_config.accept_downloads
+                    ):
                         self.logger.info(
                             message=f"Navigation aborted, likely due to file download: {url}",
                             tag="GOTO",
@@ -696,7 +759,12 @@ async def handle_request_failed_capture(request):
                         raise RuntimeError(f"Failed on navigating ACS-GOTO:\n{str(e)}")
 
                 await self.execute_hook(
-                    "after_goto", page, context=context, url=url, response=response, config=config
+                    "after_goto",
+                    page,
+                    context=context,
+                    url=url,
+                    response=response,
+                    config=config,
                 )
 
                 # ──────────────────────────────────────────────────────────────
@@ -714,10 +782,10 @@ async def handle_request_failed_capture(request):
                     while req and req.redirected_from:
                         prev_req = req.redirected_from
                         prev_resp = await prev_req.response()
-                        if prev_resp:                       # keep earliest
+                        if prev_resp:  # keep earliest
                             first_resp = prev_resp
                         req = prev_req
-                
+
                     status_code = first_resp.status
                     response_headers = first_resp.headers
                 # if response is None:
@@ -870,7 +938,9 @@ async def handle_request_failed_capture(request):
             # Handle full page scanning
             if config.scan_full_page:
                 # await self._handle_full_page_scan(page, config.scroll_delay)
-                await self._handle_full_page_scan(page, config.scroll_delay, config.max_scroll_steps)
+                await self._handle_full_page_scan(
+                    page, config.scroll_delay, config.max_scroll_steps
+                )
 
             # Handle virtual scroll if configured
             if config.virtual_scroll_config:
@@ -897,8 +967,16 @@ async def handle_request_failed_capture(request):
                         params={"error": execution_result.get("error")},
                     )
 
-                await self.execute_hook("on_execution_started", page, context=context, config=config)
-                await self.execute_hook("on_execution_ended", page, context=context, config=config, result=execution_result)
+                await self.execute_hook(
+                    "on_execution_started", page, context=context, config=config
+                )
+                await self.execute_hook(
+                    "on_execution_ended",
+                    page,
+                    context=context,
+                    config=config,
+                    result=execution_result,
+                )
 
             # Handle user simulation
             if config.simulate_user or config.magic:
@@ -910,16 +988,18 @@ async def handle_request_failed_capture(request):
             # Handle wait_for condition
             # Todo: Decide how to handle this
             if not config.wait_for and config.css_selector and False:
-            # if not config.wait_for and config.css_selector:
+                # if not config.wait_for and config.css_selector:
                 config.wait_for = f"css:{config.css_selector}"
 
             if config.wait_for:
                 try:
                     # Use wait_for_timeout if specified, otherwise fall back to page_timeout
-                    timeout = config.wait_for_timeout if config.wait_for_timeout is not None else config.page_timeout
-                    await self.smart_wait(
-                        page, config.wait_for, timeout=timeout
+                    timeout = (
+                        config.wait_for_timeout
+                        if config.wait_for_timeout is not None
+                        else config.page_timeout
                     )
+                    await self.smart_wait(page, config.wait_for, timeout=timeout)
                 except Exception as e:
                     raise RuntimeError(f"Wait condition failed: {str(e)}")
 
@@ -944,7 +1024,9 @@ async def handle_request_failed_capture(request):
                 page = await self.process_iframes(page)
 
             # Pre-content retrieval hooks and delay
-            await self.execute_hook("before_retrieve_html", page, context=context, config=config)
+            await self.execute_hook(
+                "before_retrieve_html", page, context=context, config=config
+            )
             if config.delay_before_return_html:
                 await asyncio.sleep(config.delay_before_return_html)
 
@@ -955,31 +1037,42 @@ async def handle_request_failed_capture(request):
             if config.css_selector:
                 try:
                     # Handle comma-separated selectors by splitting them
-                    selectors = [s.strip() for s in config.css_selector.split(',')]
+                    selectors = [s.strip() for s in config.css_selector.split(",")]
                     html_parts = []
-                    
+
                     for selector in selectors:
                         try:
-                            content = await self.adapter.evaluate(page,
+                            content = await self.adapter.evaluate(
+                                page,
                                 f"""Array.from(document.querySelectorAll("{selector}"))
                                     .map(el => el.outerHTML)
-                                    .join('')"""
+                                    .join('')""",
                             )
                             html_parts.append(content)
                         except Error as e:
-                            print(f"Warning: Could not get content for selector '{selector}': {str(e)}")
-                    
+                            print(
+                                f"Warning: Could not get content for selector '{selector}': {str(e)}"
+                            )
+
                     # Wrap in a div to create a valid HTML structure
-                    html = f"<div class='crawl4ai-result'>\n" + "\n".join(html_parts) + "\n</div>"                    
+                    html = (
+                        f"<div class='crawl4ai-result'>\n"
+                        + "\n".join(html_parts)
+                        + "\n</div>"
+                    )
                 except Error as e:
                     raise RuntimeError(f"Failed to extract HTML content: {str(e)}")
             else:
                 html = await page.content()
-            
+
             # # Get final HTML content
             # html = await page.content()
             await self.execute_hook(
-                "before_return_html", page=page, html=html, context=context, config=config
+                "before_return_html",
+                page=page,
+                html=html,
+                context=context,
+                config=config,
             )
 
             # Handle PDF, MHTML and screenshot generation
@@ -1019,7 +1112,9 @@ async def get_delayed_content(delay: float = 5.0) -> str:
                 return await page.content()
 
             # For undetected browsers, retrieve console messages before returning
-            if config.capture_console_messages and hasattr(self.adapter, 'retrieve_console_messages'):
+            if config.capture_console_messages and hasattr(
+                self.adapter, "retrieve_console_messages"
+            ):
                 final_messages = await self.adapter.retrieve_console_messages(page)
                 captured_console.extend(final_messages)
 
@@ -1039,8 +1134,12 @@ async def get_delayed_content(delay: float = 5.0) -> str:
                 ),
                 redirected_url=redirected_url,
                 # Include captured data if enabled
-                network_requests=captured_requests if config.capture_network_requests else None,
-                console_messages=captured_console if config.capture_console_messages else None,
+                network_requests=captured_requests
+                if config.capture_network_requests
+                else None,
+                console_messages=captured_console
+                if config.capture_console_messages
+                else None,
             )
 
         except Exception as e:
@@ -1049,10 +1148,12 @@ async def get_delayed_content(delay: float = 5.0) -> str:
         finally:
             # If no session_id is given we should close the page
             all_contexts = page.context.browser.contexts
-            total_pages = sum(len(context.pages) for context in all_contexts)                
+            total_pages = sum(len(context.pages) for context in all_contexts)
             if config.session_id:
                 pass
-            elif total_pages <= 1 and (self.browser_config.use_managed_browser or self.browser_config.headless):
+            elif total_pages <= 1 and (
+                self.browser_config.use_managed_browser or self.browser_config.headless
+            ):
                 pass
             else:
                 # Detach listeners before closing to prevent potential errors during close
@@ -1062,18 +1163,27 @@ async def get_delayed_content(delay: float = 5.0) -> str:
                     page.remove_listener("requestfailed", handle_request_failed_capture)
                 if config.capture_console_messages:
                     # Retrieve any final console messages for undetected browsers
-                    if hasattr(self.adapter, 'retrieve_console_messages'):
-                        final_messages = await self.adapter.retrieve_console_messages(page)
+                    if hasattr(self.adapter, "retrieve_console_messages"):
+                        final_messages = await self.adapter.retrieve_console_messages(
+                            page
+                        )
                         captured_console.extend(final_messages)
-                    
+
                     # Clean up console capture
-                    await self.adapter.cleanup_console_capture(page, handle_console, handle_error)
-                
+                    await self.adapter.cleanup_console_capture(
+                        page, handle_console, handle_error
+                    )
+
                 # Close the page
                 await page.close()
 
     # async def _handle_full_page_scan(self, page: Page, scroll_delay: float = 0.1):
-    async def _handle_full_page_scan(self, page: Page, scroll_delay: float = 0.1, max_scroll_steps: Optional[int] = None):
+    async def _handle_full_page_scan(
+        self,
+        page: Page,
+        scroll_delay: float = 0.1,
+        max_scroll_steps: Optional[int] = None,
+    ):
         """
         Helper method to handle full page scanning.
 
@@ -1095,7 +1205,10 @@ async def _handle_full_page_scan(self, page: Page, scroll_delay: float = 0.1, ma
             viewport_size = page.viewport_size
             if viewport_size is None:
                 await page.set_viewport_size(
-                    {"width": self.browser_config.viewport_width, "height": self.browser_config.viewport_height}
+                    {
+                        "width": self.browser_config.viewport_width,
+                        "height": self.browser_config.viewport_height,
+                    }
                 )
                 viewport_size = page.viewport_size
 
@@ -1115,19 +1228,22 @@ async def _handle_full_page_scan(self, page: Page, scroll_delay: float = 0.1, ma
 
             scroll_step_count = 0
             while current_position < total_height:
-                #### 
+                ####
                 # NEW FEATURE: Check if we've reached the maximum allowed scroll steps
                 # This prevents infinite scrolling on very long pages or infinite scroll scenarios
                 # If max_scroll_steps is None, this check is skipped (unlimited scrolling - original behavior)
                 ####
-                if max_scroll_steps is not None and scroll_step_count >= max_scroll_steps:
+                if (
+                    max_scroll_steps is not None
+                    and scroll_step_count >= max_scroll_steps
+                ):
                     break
                 current_position = min(current_position + viewport_height, total_height)
                 await self.safe_scroll(page, 0, current_position, delay=scroll_delay)
 
                 # Increment the step counter for max_scroll_steps tracking
                 scroll_step_count += 1
-                
+
                 # await page.evaluate(f"window.scrollTo(0, {current_position})")
                 # await asyncio.sleep(scroll_delay)
 
@@ -1155,7 +1271,7 @@ async def _handle_virtual_scroll(self, page: Page, config: "VirtualScrollConfig"
         """
         Handle virtual scroll containers (e.g., Twitter-like feeds) by capturing
         content at different scroll positions and merging unique elements.
-        
+
         Following the design:
         1. Get container HTML
         2. Scroll by container height
@@ -1165,7 +1281,7 @@ async def _handle_virtual_scroll(self, page: Page, config: "VirtualScrollConfig"
            - New items added (appended): continue (items already in page)
            - Items replaced: capture HTML chunk and add to list
         5. After N scrolls, merge chunks if any were captured
-        
+
         Args:
             page: The Playwright page object
             config: Virtual scroll configuration
@@ -1173,17 +1289,17 @@ async def _handle_virtual_scroll(self, page: Page, config: "VirtualScrollConfig"
         try:
             # Import VirtualScrollConfig to avoid circular import
             from .async_configs import VirtualScrollConfig
-            
+
             # Ensure config is a VirtualScrollConfig instance
             if isinstance(config, dict):
                 config = VirtualScrollConfig.from_dict(config)
-            
+
             self.logger.info(
                 message="Starting virtual scroll capture for container: {selector}",
                 tag="VSCROLL",
-                params={"selector": config.container_selector}
+                params={"selector": config.container_selector},
             )
-            
+
             # JavaScript function to handle virtual scroll capture
             virtual_scroll_js = """
             async (config) => {
@@ -1295,30 +1411,32 @@ async def _handle_virtual_scroll(self, page: Page, config: "VirtualScrollConfig"
                 }
             }
             """
-            
+
             # Execute virtual scroll capture
-            result = await self.adapter.evaluate(page, virtual_scroll_js, config.to_dict())
-            
+            result = await self.adapter.evaluate(
+                page, virtual_scroll_js, config.to_dict()
+            )
+
             if result.get("replaced", False):
                 self.logger.success(
                     message="Virtual scroll completed. Merged {unique} unique elements from {chunks} chunks",
                     tag="VSCROLL",
                     params={
                         "unique": result.get("uniqueCount", 0),
-                        "chunks": result.get("chunksCount", 0)
-                    }
+                        "chunks": result.get("chunksCount", 0),
+                    },
                 )
             else:
                 self.logger.info(
                     message="Virtual scroll completed. Content was appended, no merging needed",
-                    tag="VSCROLL"
+                    tag="VSCROLL",
                 )
-            
+
         except Exception as e:
             self.logger.error(
                 message="Virtual scroll capture failed: {error}",
                 tag="VSCROLL",
-                params={"error": str(e)}
+                params={"error": str(e)},
             )
             # Continue with normal flow even if virtual scroll fails
 
@@ -1342,7 +1460,9 @@ async def _handle_download(self, download):
         """
         try:
             suggested_filename = download.suggested_filename
-            download_path = os.path.join(self.browser_config.downloads_path, suggested_filename)
+            download_path = os.path.join(
+                self.browser_config.downloads_path, suggested_filename
+            )
 
             self.logger.info(
                 message="Downloading {filename} to {path}",
@@ -1381,7 +1501,8 @@ async def remove_overlay_elements(self, page: Page) -> None:
         remove_overlays_js = load_js_script("remove_overlay_elements")
 
         try:
-            await self.adapter.evaluate(page,
+            await self.adapter.evaluate(
+                page,
                 f"""
                 (async () => {{
                     try {{
@@ -1396,7 +1517,7 @@ async def remove_overlay_elements(self, page: Page) -> None:
                         }};
                     }}
                 }})()
-            """
+            """,
             )
             await page.wait_for_timeout(500)  # Wait for any animations to complete
         except Exception as e:
@@ -1418,17 +1539,17 @@ async def export_pdf(self, page: Page) -> bytes:
         """
         pdf_data = await page.pdf(print_background=True)
         return pdf_data
-        
+
     async def capture_mhtml(self, page: Page) -> Optional[str]:
         """
         Captures the current page as MHTML using CDP.
-        
-        MHTML (MIME HTML) is a web page archive format that combines the HTML content 
+
+        MHTML (MIME HTML) is a web page archive format that combines the HTML content
         with its resources (images, CSS, etc.) into a single MIME-encoded file.
-        
+
         Args:
             page (Page): The Playwright page object
-            
+
         Returns:
             Optional[str]: The MHTML content as a string, or None if there was an error
         """
@@ -1438,10 +1559,10 @@ async def capture_mhtml(self, page: Page) -> Optional[str]:
                 # Wait for DOM content and network to be idle
                 await page.wait_for_load_state("domcontentloaded", timeout=5000)
                 await page.wait_for_load_state("networkidle", timeout=5000)
-                
+
                 # Give a little extra time for JavaScript execution
                 await page.wait_for_timeout(1000)
-                
+
                 # Wait for any animations to complete
                 await page.evaluate("""
                     () => new Promise(resolve => {
@@ -1459,19 +1580,19 @@ async def capture_mhtml(self, page: Page) -> Optional[str]:
                         tag="MHTML",
                         params={"error": str(e)},
                     )
-            
+
             # Create a new CDP session
             cdp_session = await page.context.new_cdp_session(page)
-            
+
             # Call Page.captureSnapshot with format "mhtml"
             result = await cdp_session.send("Page.captureSnapshot", {"format": "mhtml"})
-            
+
             # The result contains a 'data' field with the MHTML content
             mhtml_content = result.get("data")
-            
+
             # Detach the CDP session to clean up resources
             await cdp_session.detach()
-            
+
             return mhtml_content
         except Exception as e:
             # Log the error but don't raise it - we'll just return None for the MHTML
@@ -1514,11 +1635,11 @@ def handle_console_message(msg):
                     )
 
         page.on("console", handle_console_message)
-        
+
         await page.goto(file_path)
 
         return captured_console
-        
+
     async def take_screenshot(self, page, **kwargs) -> str:
         """
         Take a screenshot of the current page.
@@ -1618,28 +1739,34 @@ async def take_screenshot_scroller(self, page: Page, **kwargs) -> str:
                 # Special handling for the last segment
                 if i == num_segments - 1:
                     last_part_height = page_height % viewport_height
-                    
+
                     # If page_height is an exact multiple of viewport_height,
                     # we don't need an extra segment
                     if last_part_height == 0:
                         # Skip last segment if page height is exact multiple of viewport
                         break
-                    
+
                     # Adjust viewport to exactly match the remaining content height
-                    await page.set_viewport_size({"width": page_width, "height": last_part_height})
-                
+                    await page.set_viewport_size(
+                        {"width": page_width, "height": last_part_height}
+                    )
+
                 await page.evaluate(f"window.scrollTo(0, {y_offset})")
                 await asyncio.sleep(0.01)  # wait for render
-                
+
                 # Capture the current segment
                 # Note: Using compression options (format, quality) would go here
-                seg_shot = await page.screenshot(full_page=False, type="jpeg", quality=85)
+                seg_shot = await page.screenshot(
+                    full_page=False, type="jpeg", quality=85
+                )
                 # seg_shot = await page.screenshot(full_page=False)
                 img = Image.open(BytesIO(seg_shot)).convert("RGB")
                 segments.append(img)
 
             # Reset viewport to original size after capturing segments
-            await page.set_viewport_size({"width": page_width, "height": viewport_height})
+            await page.set_viewport_size(
+                {"width": page_width, "height": viewport_height}
+            )
 
             total_height = sum(img.height for img in segments)
             stitched = Image.new("RGB", (segments[0].width, total_height))
@@ -1782,12 +1909,13 @@ async def robust_execute_user_script(
                         # }})();
                         # """
                         # )
-                        
+
                         # """ NEW VERSION:
-                        # When {script} contains statements (e.g., const link = …; link.click();), 
+                        # When {script} contains statements (e.g., const link = …; link.click();),
                         # this forms invalid JavaScript, causing Playwright execution error: SyntaxError: Unexpected token 'const'.
                         # """
-                        result = await self.adapter.evaluate(page,
+                        result = await self.adapter.evaluate(
+                            page,
                             f"""
                         (async () => {{
                             try {{
@@ -1798,7 +1926,7 @@ async def robust_execute_user_script(
                                 return {{ success: false, error: err.toString(), stack: err.stack }};
                             }}
                         }})();
-                        """
+                        """,
                         )
                     except Error as e:
                         # If it's due to navigation destroying the context, handle gracefully
@@ -1909,7 +2037,8 @@ async def execute_user_script(
             for script in scripts:
                 try:
                     # Execute the script and wait for network idle
-                    result = await self.adapter.evaluate(page,
+                    result = await self.adapter.evaluate(
+                        page,
                         f"""
                         (() => {{
                             return new Promise((resolve) => {{
@@ -1943,14 +2072,13 @@ async def execute_user_script(
                                 }}
                             }});
                         }})()
-                    """
+                    """,
                     )
 
                     # Wait for network idle after script execution
                     t1 = time.time()
                     await page.wait_for_load_state("domcontentloaded", timeout=5000)
 
-
                     t1 = time.time()
                     await page.wait_for_load_state("networkidle", timeout=5000)
 
@@ -1993,7 +2121,8 @@ async def check_visibility(self, page):
         Returns:
             Boolean indicating visibility
         """
-        return await self.adapter.evaluate(page,
+        return await self.adapter.evaluate(
+            page,
             """
             () => {
                 const element = document.body;
@@ -2004,7 +2133,7 @@ async def check_visibility(self, page):
                                 style.opacity !== '0';
                 return isVisible;
             }
-        """
+        """,
         )
 
     async def safe_scroll(self, page: Page, x: int, y: int, delay: float = 0.1):
@@ -2034,7 +2163,8 @@ async def csp_scroll_to(self, page: Page, x: int, y: int) -> Dict[str, Any]:
             Dict containing scroll status and position information
         """
         try:
-            result = await self.adapter.evaluate(page,
+            result = await self.adapter.evaluate(
+                page,
                 f"""() => {{
                     try {{
                         const startX = window.scrollX;
@@ -2061,7 +2191,7 @@ async def csp_scroll_to(self, page: Page, x: int, y: int) -> Dict[str, Any]:
                             error: e.toString()
                         }};
                     }}
-                }}"""
+                }}""",
             )
 
             if not result["success"]:
@@ -2091,13 +2221,14 @@ async def get_page_dimensions(self, page: Page):
         Returns:
             Dict containing width and height of the page
         """
-        return await self.adapter.evaluate(page,
+        return await self.adapter.evaluate(
+            page,
             """
             () => {
                 const {scrollWidth, scrollHeight} = document.documentElement;
                 return {width: scrollWidth, height: scrollHeight};
             }
-        """
+        """,
         )
 
     async def page_need_scroll(self, page: Page) -> bool:
@@ -2111,14 +2242,15 @@ async def page_need_scroll(self, page: Page) -> bool:
             bool: True if page needs scrolling
         """
         try:
-            need_scroll = await self.adapter.evaluate(page,
+            need_scroll = await self.adapter.evaluate(
+                page,
                 """
             () => {
                 const scrollHeight = document.documentElement.scrollHeight;
                 const viewportHeight = window.innerHeight;
                 return scrollHeight > viewportHeight;
             }
-            """
+            """,
             )
             return need_scroll
         except Exception as e:
@@ -2134,18 +2266,22 @@ async def page_need_scroll(self, page: Page) -> bool:
 # HTTP Crawler Strategy
 ####################################################################################################
 
+
 class HTTPCrawlerError(Exception):
     """Base error class for HTTP crawler specific exceptions"""
+
     pass
 
 
 class ConnectionTimeoutError(HTTPCrawlerError):
     """Raised when connection timeout occurs"""
+
     pass
 
 
 class HTTPStatusError(HTTPCrawlerError):
     """Raised for unexpected status codes"""
+
     def __init__(self, status_code: int, message: str):
         self.status_code = status_code
         super().__init__(f"HTTP {status_code}: {message}")
@@ -2155,31 +2291,41 @@ class AsyncHTTPCrawlerStrategy(AsyncCrawlerStrategy):
     """
     Fast, lightweight HTTP-only crawler strategy optimized for memory efficiency.
     """
-    
-    __slots__ = ('logger', 'max_connections', 'dns_cache_ttl', 'chunk_size', '_session', 'hooks', 'browser_config')
+
+    __slots__ = (
+        "logger",
+        "max_connections",
+        "dns_cache_ttl",
+        "chunk_size",
+        "_session",
+        "hooks",
+        "browser_config",
+    )
 
     DEFAULT_TIMEOUT: Final[int] = 30
-    DEFAULT_CHUNK_SIZE: Final[int] = 64 * 1024  
+    DEFAULT_CHUNK_SIZE: Final[int] = 64 * 1024
     DEFAULT_MAX_CONNECTIONS: Final[int] = min(32, (os.cpu_count() or 1) * 4)
     DEFAULT_DNS_CACHE_TTL: Final[int] = 300
-    VALID_SCHEMES: Final = frozenset({'http', 'https', 'file', 'raw'})
-
-    _BASE_HEADERS: Final = MappingProxyType({
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-        'Accept-Language': 'en-US,en;q=0.5',
-        'Accept-Encoding': 'gzip, deflate, br',
-        'Connection': 'keep-alive',
-        'Upgrade-Insecure-Requests': '1',
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
-    })
-    
+    VALID_SCHEMES: Final = frozenset({"http", "https", "file", "raw"})
+
+    _BASE_HEADERS: Final = MappingProxyType(
+        {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+        }
+    )
+
     def __init__(
-        self, 
+        self,
         browser_config: Optional[HTTPCrawlerConfig] = None,
         logger: Optional[AsyncLogger] = None,
         max_connections: int = DEFAULT_MAX_CONNECTIONS,
         dns_cache_ttl: int = DEFAULT_DNS_CACHE_TTL,
-        chunk_size: int = DEFAULT_CHUNK_SIZE
+        chunk_size: int = DEFAULT_CHUNK_SIZE,
     ):
         """Initialize the HTTP crawler with config"""
         self.browser_config = browser_config or HTTPCrawlerConfig()
@@ -2188,22 +2334,21 @@ def __init__(
         self.dns_cache_ttl = dns_cache_ttl
         self.chunk_size = chunk_size
         self._session: Optional[aiohttp.ClientSession] = None
-        
+
         self.hooks = {
-            k: partial(self._execute_hook, k) 
-            for k in ('before_request', 'after_request', 'on_error')
+            k: partial(self._execute_hook, k)
+            for k in ("before_request", "after_request", "on_error")
         }
 
         # Set default hooks
-        self.set_hook('before_request', lambda *args, **kwargs: None)
-        self.set_hook('after_request', lambda *args, **kwargs: None)
-        self.set_hook('on_error', lambda *args, **kwargs: None)
-                      
+        self.set_hook("before_request", lambda *args, **kwargs: None)
+        self.set_hook("after_request", lambda *args, **kwargs: None)
+        self.set_hook("on_error", lambda *args, **kwargs: None)
 
     async def __aenter__(self) -> AsyncHTTPCrawlerStrategy:
         await self.start()
         return self
-        
+
     async def __aexit__(self, exc_type, exc_val, exc_tb) -> None:
         await self.close()
 
@@ -2223,11 +2368,7 @@ def set_hook(self, hook_type: str, hook_func: Callable) -> None:
             raise ValueError(f"Invalid hook type: {hook_type}")
 
     async def _execute_hook(
-        self, 
-        hook_type: str, 
-        hook_func: Callable,
-        *args: Any, 
-        **kwargs: Any
+        self, hook_type: str, hook_func: Callable, *args: Any, **kwargs: Any
     ) -> Any:
         if asyncio.iscoroutinefunction(hook_func):
             return await hook_func(*args, **kwargs)
@@ -2239,12 +2380,12 @@ async def start(self) -> None:
                 limit=self.max_connections,
                 ttl_dns_cache=self.dns_cache_ttl,
                 use_dns_cache=True,
-                force_close=False
+                force_close=False,
             )
             self._session = aiohttp.ClientSession(
                 headers=dict(self._BASE_HEADERS),
                 connector=connector,
-                timeout=ClientTimeout(total=self.DEFAULT_TIMEOUT)
+                timeout=ClientTimeout(total=self.DEFAULT_TIMEOUT),
             )
 
     async def close(self) -> None:
@@ -2254,141 +2395,131 @@ async def close(self) -> None:
             except asyncio.TimeoutError:
                 if self.logger:
                     self.logger.warning(
-                        message="Session cleanup timed out",
-                        tag="CLEANUP"
+                        message="Session cleanup timed out", tag="CLEANUP"
                     )
             finally:
                 self._session = None
 
     async def _stream_file(self, path: str) -> AsyncGenerator[memoryview, None]:
-        async with aiofiles.open(path, mode='rb') as f:
+        async with aiofiles.open(path, mode="rb") as f:
             while chunk := await f.read(self.chunk_size):
                 yield memoryview(chunk)
 
     async def _handle_file(self, path: str) -> AsyncCrawlResponse:
         if not os.path.exists(path):
             raise FileNotFoundError(f"Local file not found: {path}")
-            
+
         chunks = []
         async for chunk in self._stream_file(path):
-            chunks.append(chunk.tobytes().decode('utf-8', errors='replace'))
-            
-        return AsyncCrawlResponse(
-            html=''.join(chunks),
-            response_headers={},
-            status_code=200
-        )
+            chunks.append(chunk.tobytes().decode("utf-8", errors="replace"))
 
-    async def _handle_raw(self, content: str) -> AsyncCrawlResponse:
         return AsyncCrawlResponse(
-            html=content,
-            response_headers={},
-            status_code=200
+            html="".join(chunks), response_headers={}, status_code=200
         )
 
+    async def _handle_raw(self, content: str) -> AsyncCrawlResponse:
+        return AsyncCrawlResponse(html=content, response_headers={}, status_code=200)
 
     async def _handle_http(
-        self, 
-        url: str, 
-        config: CrawlerRunConfig
+        self, url: str, config: CrawlerRunConfig
     ) -> AsyncCrawlResponse:
         async with self._session_context() as session:
             timeout = ClientTimeout(
                 total=config.page_timeout or self.DEFAULT_TIMEOUT,
                 connect=10,
-                sock_read=30
+                sock_read=30,
             )
-            
+
             headers = dict(self._BASE_HEADERS)
             if self.browser_config.headers:
                 headers.update(self.browser_config.headers)
 
             request_kwargs = {
-                'timeout': timeout,
-                'allow_redirects': self.browser_config.follow_redirects,
-                'ssl': self.browser_config.verify_ssl,
-                'headers': headers
+                "timeout": timeout,
+                "allow_redirects": self.browser_config.follow_redirects,
+                "ssl": self.browser_config.verify_ssl,
+                "headers": headers,
             }
 
             if self.browser_config.method == "POST":
                 if self.browser_config.data:
-                    request_kwargs['data'] = self.browser_config.data
+                    request_kwargs["data"] = self.browser_config.data
                 if self.browser_config.json:
-                    request_kwargs['json'] = self.browser_config.json
+                    request_kwargs["json"] = self.browser_config.json
 
-            await self.hooks['before_request'](url, request_kwargs)
+            await self.hooks["before_request"](url, request_kwargs)
 
             try:
-                async with session.request(self.browser_config.method, url, **request_kwargs) as response:
+                async with session.request(
+                    self.browser_config.method, url, **request_kwargs
+                ) as response:
                     content = memoryview(await response.read())
-                    
+
                     if not (200 <= response.status < 300):
                         raise HTTPStatusError(
-                            response.status,
-                            f"Unexpected status code for {url}"
+                            response.status, f"Unexpected status code for {url}"
                         )
-                    
+
                     encoding = response.charset
                     if not encoding:
-                        encoding = chardet.detect(content.tobytes())['encoding'] or 'utf-8'                    
-                    
+                        encoding = (
+                            chardet.detect(content.tobytes())["encoding"] or "utf-8"
+                        )
+
                     result = AsyncCrawlResponse(
-                        html=content.tobytes().decode(encoding, errors='replace'),
+                        html=content.tobytes().decode(encoding, errors="replace"),
                         response_headers=dict(response.headers),
                         status_code=response.status,
-                        redirected_url=str(response.url)
+                        redirected_url=str(response.url),
                     )
-                    
-                    await self.hooks['after_request'](result)
+
+                    await self.hooks["after_request"](result)
                     return result
 
             except aiohttp.ServerTimeoutError as e:
-                await self.hooks['on_error'](e)
+                await self.hooks["on_error"](e)
                 raise ConnectionTimeoutError(f"Request timed out: {str(e)}")
-                
+
             except aiohttp.ClientConnectorError as e:
-                await self.hooks['on_error'](e)
+                await self.hooks["on_error"](e)
                 raise ConnectionError(f"Connection failed: {str(e)}")
-                
+
             except aiohttp.ClientError as e:
-                await self.hooks['on_error'](e)
+                await self.hooks["on_error"](e)
                 raise HTTPCrawlerError(f"HTTP client error: {str(e)}")
-            
+
             except asyncio.exceptions.TimeoutError as e:
-                await self.hooks['on_error'](e)
+                await self.hooks["on_error"](e)
                 raise ConnectionTimeoutError(f"Request timed out: {str(e)}")
-            
+
             except Exception as e:
-                await self.hooks['on_error'](e)
+                await self.hooks["on_error"](e)
                 raise HTTPCrawlerError(f"HTTP request failed: {str(e)}")
 
     async def crawl(
-        self, 
-        url: str, 
-        config: Optional[CrawlerRunConfig] = None, 
-        **kwargs
+        self, url: str, config: Optional[CrawlerRunConfig] = None, **kwargs
     ) -> AsyncCrawlResponse:
         config = config or CrawlerRunConfig.from_kwargs(kwargs)
-        
+
         parsed = urlparse(url)
-        scheme = parsed.scheme.rstrip('/')
-        
+        scheme = parsed.scheme.rstrip("/")
+
         if scheme not in self.VALID_SCHEMES:
             raise ValueError(f"Unsupported URL scheme: {scheme}")
-            
+
         try:
-            if scheme == 'file':
+            if scheme == "file":
                 return await self._handle_file(parsed.path)
-            elif scheme == 'raw':
+            elif scheme == "raw":
                 return await self._handle_raw(parsed.path)
             else:  # http or https
                 return await self._handle_http(url, config)
-                
+
         except Exception as e:
             if self.logger:
                 self.logger.error(
                     message="Crawl failed: {error}",
                     tag="CRAWL",
-                    params={"error": str(e), "url": url}
+                    params={"error": str(e), "url": url},
                 )
             raise
diff --git a/crawl4ai/async_dispatcher.py b/crawl4ai/async_dispatcher.py
index bd44557c7..5c447fdd7 100644
--- a/crawl4ai/async_dispatcher.py
+++ b/crawl4ai/async_dispatcher.py
@@ -85,7 +85,6 @@ def update_delay(self, url: str, status_code: int) -> bool:
         return True
 
 
-
 class BaseDispatcher(ABC):
     def __init__(
         self,
@@ -98,29 +97,31 @@ def __init__(
         self.rate_limiter = rate_limiter
         self.monitor = monitor
 
-    def select_config(self, url: str, configs: Union[CrawlerRunConfig, List[CrawlerRunConfig]]) -> Optional[CrawlerRunConfig]:
+    def select_config(
+        self, url: str, configs: Union[CrawlerRunConfig, List[CrawlerRunConfig]]
+    ) -> Optional[CrawlerRunConfig]:
         """Select the appropriate config for a given URL.
-        
+
         Args:
             url: The URL to match against
             configs: Single config or list of configs to choose from
-            
+
         Returns:
             The matching config, or None if no match found
         """
         # Single config - return as is
         if isinstance(configs, CrawlerRunConfig):
             return configs
-        
+
         # Empty list - return None
         if not configs:
             return None
-        
+
         # Find first matching config
         for config in configs:
             if config.is_match(url):
                 return config
-        
+
         # No match found - return None to indicate URL should be skipped
         return None
 
@@ -167,11 +168,15 @@ def __init__(
         self.fairness_timeout = fairness_timeout
         self.memory_wait_timeout = memory_wait_timeout
         self.result_queue = asyncio.Queue()
-        self.task_queue = asyncio.PriorityQueue()  # Priority queue for better management
-        self.memory_pressure_mode = False  # Flag to indicate when we're in memory pressure mode
+        self.task_queue = (
+            asyncio.PriorityQueue()
+        )  # Priority queue for better management
+        self.memory_pressure_mode = (
+            False  # Flag to indicate when we're in memory pressure mode
+        )
         self.current_memory_percent = 0.0  # Track current memory usage
         self._high_memory_start_time: Optional[float] = None
-        
+
     async def _memory_monitor_task(self):
         """Background task to continuously monitor memory usage and update state"""
         while True:
@@ -190,7 +195,8 @@ async def _memory_monitor_task(self):
                     if (
                         self.memory_wait_timeout is not None
                         and self._high_memory_start_time is not None
-                        and time.time() - self._high_memory_start_time >= self.memory_wait_timeout
+                        and time.time() - self._high_memory_start_time
+                        >= self.memory_wait_timeout
                     ):
                         raise MemoryError(
                             "Memory usage exceeded threshold for"
@@ -198,22 +204,25 @@ async def _memory_monitor_task(self):
                         )
 
             # Exit memory pressure mode if we go below recovery threshold
-            elif self.memory_pressure_mode and self.current_memory_percent <= self.recovery_threshold_percent:
+            elif (
+                self.memory_pressure_mode
+                and self.current_memory_percent <= self.recovery_threshold_percent
+            ):
                 self.memory_pressure_mode = False
                 self._high_memory_start_time = None
                 if self.monitor:
                     self.monitor.update_memory_status("NORMAL")
             elif self.current_memory_percent < self.memory_threshold_percent:
                 self._high_memory_start_time = None
-            
+
             # In critical mode, we might need to take more drastic action
             if self.current_memory_percent >= self.critical_threshold_percent:
                 if self.monitor:
                     self.monitor.update_memory_status("CRITICAL")
                 # We could implement additional memory-saving measures here
-                
+
             await asyncio.sleep(self.check_interval)
-    
+
     def _get_priority_score(self, wait_time: float, retry_count: int) -> float:
         """Calculate priority score (lower is higher priority)
         - URLs waiting longer than fairness_timeout get higher priority
@@ -224,7 +233,7 @@ def _get_priority_score(self, wait_time: float, retry_count: int) -> float:
             return -wait_time
         # Standard priority based on retries
         return retry_count
-    
+
     async def crawl_url(
         self,
         url: str,
@@ -235,109 +244,116 @@ async def crawl_url(
         start_time = time.time()
         error_message = ""
         memory_usage = peak_memory = 0.0
-        
+
         # Select appropriate config for this URL
         selected_config = self.select_config(url, config)
-        
+
         # If no config matches, return failed result
         if selected_config is None:
             error_message = f"No matching configuration found for URL: {url}"
             if self.monitor:
                 self.monitor.update_task(
-                    task_id, 
-                    status=CrawlStatus.FAILED,
-                    error_message=error_message
+                    task_id, status=CrawlStatus.FAILED, error_message=error_message
                 )
-            
+
             return CrawlerTaskResult(
                 task_id=task_id,
                 url=url,
                 result=CrawlResult(
-                    url=url, 
-                    html="", 
-                    metadata={"status": "no_config_match"}, 
-                    success=False, 
-                    error_message=error_message
+                    url=url,
+                    html="",
+                    metadata={"status": "no_config_match"},
+                    success=False,
+                    error_message=error_message,
                 ),
                 memory_usage=0,
                 peak_memory=0,
                 start_time=start_time,
                 end_time=time.time(),
                 error_message=error_message,
-                retry_count=retry_count
+                retry_count=retry_count,
             )
-        
+
         # Get starting memory for accurate measurement
         process = psutil.Process()
         start_memory = process.memory_info().rss / (1024 * 1024)
-        
+
         try:
             if self.monitor:
                 self.monitor.update_task(
-                    task_id, 
-                    status=CrawlStatus.IN_PROGRESS, 
+                    task_id,
+                    status=CrawlStatus.IN_PROGRESS,
                     start_time=start_time,
-                    retry_count=retry_count
+                    retry_count=retry_count,
                 )
-                
+
             self.concurrent_sessions += 1
-            
+
             if self.rate_limiter:
                 await self.rate_limiter.wait_if_needed(url)
-                
+
             # Check if we're in critical memory state
             if self.current_memory_percent >= self.critical_threshold_percent:
                 # Requeue this task with increased priority and retry count
                 enqueue_time = time.time()
-                priority = self._get_priority_score(enqueue_time - start_time, retry_count + 1)
-                await self.task_queue.put((priority, (url, task_id, retry_count + 1, enqueue_time)))
-                
+                priority = self._get_priority_score(
+                    enqueue_time - start_time, retry_count + 1
+                )
+                await self.task_queue.put(
+                    (priority, (url, task_id, retry_count + 1, enqueue_time))
+                )
+
                 # Update monitoring
                 if self.monitor:
                     self.monitor.update_task(
                         task_id,
                         status=CrawlStatus.QUEUED,
-                        error_message="Requeued due to critical memory pressure"
+                        error_message="Requeued due to critical memory pressure",
                     )
-                
+
                 # Return placeholder result with requeued status
                 return CrawlerTaskResult(
                     task_id=task_id,
                     url=url,
                     result=CrawlResult(
-                        url=url, html="", metadata={"status": "requeued"}, 
-                        success=False, error_message="Requeued due to critical memory pressure"
+                        url=url,
+                        html="",
+                        metadata={"status": "requeued"},
+                        success=False,
+                        error_message="Requeued due to critical memory pressure",
                     ),
                     memory_usage=0,
                     peak_memory=0,
                     start_time=start_time,
                     end_time=time.time(),
                     error_message="Requeued due to critical memory pressure",
-                    retry_count=retry_count + 1
+                    retry_count=retry_count + 1,
                 )
-            
+
             # Execute the crawl with selected config
-            result = await self.crawler.arun(url, config=selected_config, session_id=task_id)
-            
+            result = await self.crawler.arun(
+                url, config=selected_config, session_id=task_id
+            )
+
             # Measure memory usage
             end_memory = process.memory_info().rss / (1024 * 1024)
             memory_usage = peak_memory = end_memory - start_memory
-            
+
             # Handle rate limiting
             if self.rate_limiter and result.status_code:
                 if not self.rate_limiter.update_delay(url, result.status_code):
                     error_message = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}"
                     if self.monitor:
                         self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-                        
-            # Update status based on result
-            if not result.success:
-                error_message = result.error_message
-                if self.monitor:
-                    self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-            elif self.monitor:
-                self.monitor.update_task(task_id, status=CrawlStatus.COMPLETED)
-                
+
+                # Update status based on result
+                if not result.success:
+                    error_message = result.error_message
+                    if self.monitor:
+                        self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+                elif self.monitor:
+                    self.monitor.update_task(task_id, status=CrawlStatus.COMPLETED)
+
         except Exception as e:
             error_message = str(e)
             if self.monitor:
@@ -345,7 +361,7 @@ async def crawl_url(
             result = CrawlResult(
                 url=url, html="", metadata={}, success=False, error_message=str(e)
             )
-            
+
         finally:
             end_time = time.time()
             if self.monitor:
@@ -355,10 +371,10 @@ async def crawl_url(
                     memory_usage=memory_usage,
                     peak_memory=peak_memory,
                     error_message=error_message,
-                    retry_count=retry_count
+                    retry_count=retry_count,
                 )
             self.concurrent_sessions -= 1
-            
+
         return CrawlerTaskResult(
             task_id=task_id,
             url=url,
@@ -368,9 +384,9 @@ async def crawl_url(
             start_time=start_time,
             end_time=end_time,
             error_message=error_message,
-            retry_count=retry_count
+            retry_count=retry_count,
         )
-        
+
     async def run_urls(
         self,
         urls: List[str],
@@ -378,13 +394,13 @@ async def run_urls(
         config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
     ) -> List[CrawlerTaskResult]:
         self.crawler = crawler
-        
+
         # Start the memory monitor task
         memory_monitor = asyncio.create_task(self._memory_monitor_task())
-        
+
         if self.monitor:
             self.monitor.start()
-            
+
         results = []
 
         try:
@@ -413,119 +429,130 @@ async def run_urls(
                     while slots > 0:
                         try:
                             # Use get_nowait() to immediately get tasks without blocking
-                            priority, (url, task_id, retry_count, enqueue_time) = self.task_queue.get_nowait()
-                            
+                            priority, (url, task_id, retry_count, enqueue_time) = (
+                                self.task_queue.get_nowait()
+                            )
+
                             # Create and start the task
                             task = asyncio.create_task(
                                 self.crawl_url(url, config, task_id, retry_count)
                             )
                             active_tasks.append(task)
-                            
+
                             # Update waiting time in monitor
                             if self.monitor:
                                 wait_time = time.time() - enqueue_time
                                 self.monitor.update_task(
-                                    task_id, 
+                                    task_id,
                                     wait_time=wait_time,
-                                    status=CrawlStatus.IN_PROGRESS
+                                    status=CrawlStatus.IN_PROGRESS,
                                 )
-                            
+
                             slots -= 1
-                            
+
                         except asyncio.QueueEmpty:
                             # No more tasks in queue, exit the loop
                             break
-                        
+
                 # Wait for completion even if queue is starved
                 if active_tasks:
                     done, pending = await asyncio.wait(
                         active_tasks, timeout=0.1, return_when=asyncio.FIRST_COMPLETED
                     )
-                    
+
                     # Process completed tasks
                     for completed_task in done:
                         result = await completed_task
                         results.append(result)
-                        
+
                     # Update active tasks list
                     active_tasks = list(pending)
                 else:
                     # If no active tasks but still waiting, sleep briefly
                     await asyncio.sleep(self.check_interval / 2)
-                    
+
                 # Update priorities for waiting tasks if needed
                 await self._update_queue_priorities()
 
         except Exception as e:
             if self.monitor:
-                self.monitor.update_memory_status(f"QUEUE_ERROR: {str(e)}")                
-        
+                self.monitor.update_memory_status(f"QUEUE_ERROR: {str(e)}")
+
         finally:
             # Clean up
             memory_monitor.cancel()
             if self.monitor:
                 self.monitor.stop()
             return results
-                
+
     async def _update_queue_priorities(self):
         """Periodically update priorities of items in the queue to prevent starvation"""
         # Skip if queue is empty
         if self.task_queue.empty():
             return
-            
+
         # Use a drain-and-refill approach to update all priorities
         temp_items = []
-        
+
         # Drain the queue (with a safety timeout to prevent blocking)
         try:
             drain_start = time.time()
-            while not self.task_queue.empty() and time.time() - drain_start < 5.0:  # 5 second safety timeout
+            while (
+                not self.task_queue.empty() and time.time() - drain_start < 5.0
+            ):  # 5 second safety timeout
                 try:
                     # Get item from queue with timeout
-                    priority, (url, task_id, retry_count, enqueue_time) = await asyncio.wait_for(
-                        self.task_queue.get(), timeout=0.1
-                    )
-                    
+                    (
+                        priority,
+                        (url, task_id, retry_count, enqueue_time),
+                    ) = await asyncio.wait_for(self.task_queue.get(), timeout=0.1)
+
                     # Calculate new priority based on current wait time
                     current_time = time.time()
                     wait_time = current_time - enqueue_time
                     new_priority = self._get_priority_score(wait_time, retry_count)
-                    
+
                     # Store with updated priority
-                    temp_items.append((new_priority, (url, task_id, retry_count, enqueue_time)))
-                    
+                    temp_items.append(
+                        (new_priority, (url, task_id, retry_count, enqueue_time))
+                    )
+
                     # Update monitoring stats for this task
                     if self.monitor and task_id in self.monitor.stats:
                         self.monitor.update_task(task_id, wait_time=wait_time)
-                        
+
                 except asyncio.TimeoutError:
                     # Queue might be empty or very slow
                     break
         except Exception as e:
             # If anything goes wrong, make sure we refill the queue with what we've got
             self.monitor.update_memory_status(f"QUEUE_ERROR: {str(e)}")
-        
+
         # Calculate queue statistics
         if temp_items and self.monitor:
             total_queued = len(temp_items)
             wait_times = [item[1][3] for item in temp_items]
             highest_wait_time = time.time() - min(wait_times) if wait_times else 0
-            avg_wait_time = sum(time.time() - t for t in wait_times) / len(wait_times) if wait_times else 0
-            
+            avg_wait_time = (
+                sum(time.time() - t for t in wait_times) / len(wait_times)
+                if wait_times
+                else 0
+            )
+
             # Update queue statistics in monitor
             self.monitor.update_queue_statistics(
                 total_queued=total_queued,
                 highest_wait_time=highest_wait_time,
-                avg_wait_time=avg_wait_time
+                avg_wait_time=avg_wait_time,
             )
-        
+
         # Sort by priority (lowest number = highest priority)
         temp_items.sort(key=lambda x: x[0])
-        
+
         # Refill the queue with updated priorities
         for item in temp_items:
             await self.task_queue.put(item)
-                
+
     async def run_urls_stream(
         self,
         urls: List[str],
@@ -533,13 +560,13 @@ async def run_urls_stream(
         config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
     ) -> AsyncGenerator[CrawlerTaskResult, None]:
         self.crawler = crawler
-        
+
         # Start the memory monitor task
         memory_monitor = asyncio.create_task(self._memory_monitor_task())
-        
+
         if self.monitor:
             self.monitor.start()
-            
+
         try:
             # Initialize task queue
             for url in urls:
@@ -548,7 +575,7 @@ async def run_urls_stream(
                     self.monitor.add_task(task_id, url)
                 # Add to queue with initial priority 0, retry count 0, and current time
                 await self.task_queue.put((0, (url, task_id, 0, time.time())))
-                
+
             active_tasks = []
             completed_count = 0
             total_urls = len(urls)
@@ -566,58 +593,60 @@ async def run_urls_stream(
                     while slots > 0:
                         try:
                             # Use get_nowait() to immediately get tasks without blocking
-                            priority, (url, task_id, retry_count, enqueue_time) = self.task_queue.get_nowait()
-                            
+                            priority, (url, task_id, retry_count, enqueue_time) = (
+                                self.task_queue.get_nowait()
+                            )
+
                             # Create and start the task
                             task = asyncio.create_task(
                                 self.crawl_url(url, config, task_id, retry_count)
                             )
                             active_tasks.append(task)
-                            
+
                             # Update waiting time in monitor
                             if self.monitor:
                                 wait_time = time.time() - enqueue_time
                                 self.monitor.update_task(
-                                    task_id, 
+                                    task_id,
                                     wait_time=wait_time,
-                                    status=CrawlStatus.IN_PROGRESS
+                                    status=CrawlStatus.IN_PROGRESS,
                                 )
-                            
+
                             slots -= 1
-                            
+
                         except asyncio.QueueEmpty:
                             # No more tasks in queue, exit the loop
                             break
-                        
+
                 # Process completed tasks and yield results
                 if active_tasks:
                     done, pending = await asyncio.wait(
                         active_tasks, timeout=0.1, return_when=asyncio.FIRST_COMPLETED
                     )
-                    
+
                     for completed_task in done:
                         result = await completed_task
-                        
+
                         # Only count as completed if it wasn't requeued
                         if "requeued" not in result.error_message:
                             completed_count += 1
                             yield result
-                        
+
                     # Update active tasks list
                     active_tasks = list(pending)
                 else:
                     # If no active tasks but still waiting, sleep briefly
                     await asyncio.sleep(self.check_interval / 2)
-                
+
                 # Update priorities for waiting tasks if needed
                 await self._update_queue_priorities()
-                
+
         finally:
             # Clean up
             memory_monitor.cancel()
             if self.monitor:
                 self.monitor.stop()
-                
+
 
 class SemaphoreDispatcher(BaseDispatcher):
     def __init__(
@@ -644,32 +673,30 @@ async def crawl_url(
 
         # Select appropriate config for this URL
         selected_config = self.select_config(url, config)
-        
+
         # If no config matches, return failed result
         if selected_config is None:
             error_message = f"No matching configuration found for URL: {url}"
             if self.monitor:
                 self.monitor.update_task(
-                    task_id, 
-                    status=CrawlStatus.FAILED,
-                    error_message=error_message
+                    task_id, status=CrawlStatus.FAILED, error_message=error_message
                 )
-            
+
             return CrawlerTaskResult(
                 task_id=task_id,
                 url=url,
                 result=CrawlResult(
-                    url=url, 
-                    html="", 
-                    metadata={"status": "no_config_match"}, 
-                    success=False, 
-                    error_message=error_message
+                    url=url,
+                    html="",
+                    metadata={"status": "no_config_match"},
+                    success=False,
+                    error_message=error_message,
                 ),
                 memory_usage=0,
                 peak_memory=0,
                 start_time=start_time,
                 end_time=time.time(),
-                error_message=error_message
+                error_message=error_message,
             )
 
         try:
@@ -684,7 +711,9 @@ async def crawl_url(
             async with semaphore:
                 process = psutil.Process()
                 start_memory = process.memory_info().rss / (1024 * 1024)
-                result = await self.crawler.arun(url, config=selected_config, session_id=task_id)
+                result = await self.crawler.arun(
+                    url, config=selected_config, session_id=task_id
+                )
                 end_memory = process.memory_info().rss / (1024 * 1024)
 
                 memory_usage = peak_memory = end_memory - start_memory
@@ -768,4 +797,4 @@ async def run_urls(
             return await asyncio.gather(*tasks, return_exceptions=True)
         finally:
             if self.monitor:
-                self.monitor.stop()
\ No newline at end of file
+                self.monitor.stop()
diff --git a/crawl4ai/async_webcrawler.py b/crawl4ai/async_webcrawler.py
index 1b571b50a..abf538b7a 100644
--- a/crawl4ai/async_webcrawler.py
+++ b/crawl4ai/async_webcrawler.py
@@ -15,7 +15,7 @@
     DispatchResult,
     ScrapingResult,
     CrawlResultContainer,
-    RunManyReturn
+    RunManyReturn,
 )
 from .async_database import async_db_manager
 from .chunking_strategy import *  # noqa: F403
@@ -111,8 +111,7 @@ def __init__(
         self,
         crawler_strategy: AsyncCrawlerStrategy = None,
         config: BrowserConfig = None,
-        base_directory: str = str(
-            os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home())),
+        base_directory: str = str(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home())),
         thread_safe: bool = False,
         logger: AsyncLoggerBase = None,
         **kwargs,
@@ -140,8 +139,7 @@ def __init__(
         )
 
         # Initialize crawler strategy
-        params = {k: v for k, v in kwargs.items() if k in [
-            "browser_config", "logger"]}
+        params = {k: v for k, v in kwargs.items() if k in ["browser_config", "logger"]}
         self.crawler_strategy = crawler_strategy or AsyncPlaywrightCrawlerStrategy(
             browser_config=browser_config,
             logger=self.logger,
@@ -164,7 +162,7 @@ def __init__(
         # Decorate arun method with deep crawling capabilities
         self._deep_handler = DeepCrawlDecorator(self)
         self.arun = self._deep_handler(self.arun)
-        
+
         self.url_seeder: Optional[AsyncUrlSeeder] = None
 
     async def start(self):
@@ -241,8 +239,7 @@ async def arun(
 
         config = config or CrawlerRunConfig()
         if not isinstance(url, str) or not url:
-            raise ValueError(
-                "Invalid URL, make sure the URL is a non-empty string")
+            raise ValueError("Invalid URL, make sure the URL is a non-empty string")
 
         async with self._lock or self.nullcontext():
             try:
@@ -296,12 +293,14 @@ async def arun(
 
                 # Update proxy configuration from rotation strategy if available
                 if config and config.proxy_rotation_strategy:
-                    next_proxy: ProxyConfig = await config.proxy_rotation_strategy.get_next_proxy()
+                    next_proxy: ProxyConfig = (
+                        await config.proxy_rotation_strategy.get_next_proxy()
+                    )
                     if next_proxy:
                         self.logger.info(
                             message="Switch proxy: {proxy}",
                             tag="PROXY",
-                            params={"proxy": next_proxy.server}
+                            params={"proxy": next_proxy.server},
                         )
                         config.proxy_config = next_proxy
                         # config = config.clone(proxy_config=next_proxy)
@@ -311,8 +310,7 @@ async def arun(
                     t1 = time.perf_counter()
 
                     if config.user_agent:
-                        self.crawler_strategy.update_user_agent(
-                            config.user_agent)
+                        self.crawler_strategy.update_user_agent(config.user_agent)
 
                     # Check robots.txt if enabled
                     if config and config.check_robots_txt:
@@ -355,6 +353,7 @@ async def arun(
                     # Process the HTML content, Call CrawlerStrategy.process_html #
                     ###############################################################
                     from urllib.parse import urlparse
+
                     crawl_result: CrawlResult = await self.aprocess_html(
                         url=url,
                         html=html,
@@ -381,8 +380,7 @@ async def arun(
                     crawl_result.console_messages = async_response.console_messages
 
                     crawl_result.success = bool(html)
-                    crawl_result.session_id = getattr(
-                        config, "session_id", None)
+                    crawl_result.session_id = getattr(config, "session_id", None)
 
                     self.logger.url_status(
                         url=cache_context.display_url,
@@ -402,11 +400,10 @@ async def arun(
                         url=cache_context.display_url,
                         success=True,
                         timing=time.perf_counter() - start_time,
-                        tag="COMPLETE"
+                        tag="COMPLETE",
                     )
                     cached_result.success = bool(html)
-                    cached_result.session_id = getattr(
-                        config, "session_id", None)
+                    cached_result.session_id = getattr(config, "session_id", None)
                     cached_result.redirected_url = cached_result.redirected_url or url
                     return CrawlResultContainer(cached_result)
 
@@ -473,14 +470,12 @@ async def aprocess_html(
             params = config.__dict__.copy()
             params.pop("url", None)
             # add keys from kwargs to params that doesn't exist in params
-            params.update({k: v for k, v in kwargs.items()
-                          if k not in params.keys()})
+            params.update({k: v for k, v in kwargs.items() if k not in params.keys()})
 
             ################################
             # Scraping Strategy Execution  #
             ################################
-            result: ScrapingResult = scraping_strategy.scrap(
-                url, html, **params)
+            result: ScrapingResult = scraping_strategy.scrap(url, html, **params)
 
             if result is None:
                 raise ValueError(
@@ -496,8 +491,7 @@ async def aprocess_html(
 
         # Extract results - handle both dict and ScrapingResult
         if isinstance(result, dict):
-            cleaned_html = sanitize_input_encode(
-                result.get("cleaned_html", ""))
+            cleaned_html = sanitize_input_encode(result.get("cleaned_html", ""))
             media = result.get("media", {})
             tables = media.pop("tables", []) if isinstance(media, dict) else []
             links = result.get("links", {})
@@ -507,12 +501,22 @@ async def aprocess_html(
             # media = result.media.model_dump()
             # tables = media.pop("tables", [])
             # links = result.links.model_dump()
-            media = result.media.model_dump() if hasattr(result.media, 'model_dump') else result.media
+            media = (
+                result.media.model_dump()
+                if hasattr(result.media, "model_dump")
+                else result.media
+            )
             tables = media.pop("tables", []) if isinstance(media, dict) else []
-            links = result.links.model_dump() if hasattr(result.links, 'model_dump') else result.links
+            links = (
+                result.links.model_dump()
+                if hasattr(result.links, "model_dump")
+                else result.links
+            )
             metadata = result.metadata
 
-        fit_html = preprocess_html_for_schema(html_content=html, text_threshold= 500, max_size= 300_000)
+        fit_html = preprocess_html_for_schema(
+            html_content=html, text_threshold=500, max_size=300_000
+        )
 
         ################################
         # Generate Markdown            #
@@ -523,7 +527,9 @@ async def aprocess_html(
 
         # --- SELECT HTML SOURCE BASED ON CONTENT_SOURCE ---
         # Get the desired source from the generator config, default to 'cleaned_html'
-        selected_html_source = getattr(markdown_generator, 'content_source', 'cleaned_html')
+        selected_html_source = getattr(
+            markdown_generator, "content_source", "cleaned_html"
+        )
 
         # Define the source selection logic using dict dispatch
         html_source_selector = {
@@ -536,7 +542,9 @@ async def aprocess_html(
 
         try:
             # Get the appropriate lambda function, default to returning cleaned_html if key not found
-            source_lambda = html_source_selector.get(selected_html_source, lambda: cleaned_html)
+            source_lambda = html_source_selector.get(
+                selected_html_source, lambda: cleaned_html
+            )
             # Execute the lambda to get the selected HTML
             markdown_input_html = source_lambda()
 
@@ -550,7 +558,7 @@ async def aprocess_html(
             if self.logger:
                 self.logger.warning(
                     f"Error getting/processing '{selected_html_source}' for markdown source: {e}. Falling back to cleaned_html.",
-                    tag="MARKDOWN_SRC"
+                    tag="MARKDOWN_SRC",
                 )
             # Ensure markdown_input_html is still the default cleaned_html in case of error
             markdown_input_html = cleaned_html
@@ -563,7 +571,7 @@ async def aprocess_html(
         markdown_result: MarkdownGenerationResult = (
             markdown_generator.generate_markdown(
                 input_html=markdown_input_html,
-                base_url=params.get("redirected_url", url)
+                base_url=params.get("redirected_url", url),
                 # html2text_options=kwargs.get('html2text', {})
             )
         )
@@ -573,7 +581,7 @@ async def aprocess_html(
             url=_url,
             success=True,
             timing=int((time.perf_counter() - t1) * 1000) / 1000,
-            tag="SCRAPE"
+            tag="SCRAPE",
         )
         # self.logger.info(
         #     message="{url:.50}... | Time: {timing}s",
@@ -593,13 +601,12 @@ async def aprocess_html(
             # Choose content based on input_format
             content_format = config.extraction_strategy.input_format
             if content_format == "fit_markdown" and not markdown_result.fit_markdown:
-
                 self.logger.url_status(
-                        url=_url,
-                        success=bool(html),
-                        timing=time.perf_counter() - t1,
-                        tag="EXTRACT",
-                    )
+                    url=_url,
+                    success=bool(html),
+                    timing=time.perf_counter() - t1,
+                    tag="EXTRACT",
+                )
                 content_format = "markdown"
 
             content = {
@@ -620,7 +627,7 @@ async def aprocess_html(
             # extracted_content = config.extraction_strategy.run(url, sections)
 
             # Use async version if available for better parallelism
-            if hasattr(config.extraction_strategy, 'arun'):
+            if hasattr(config.extraction_strategy, "arun"):
                 extracted_content = await config.extraction_strategy.arun(url, sections)
             else:
                 # Fallback to sync version run in thread pool to avoid blocking
@@ -634,11 +641,11 @@ async def aprocess_html(
 
             # Log extraction completion
             self.logger.url_status(
-                        url=_url,
-                        success=bool(html),
-                        timing=time.perf_counter() - t1,
-                        tag="EXTRACT",
-                    )
+                url=_url,
+                success=bool(html),
+                timing=time.perf_counter() - t1,
+                tag="EXTRACT",
+            )
 
         # Apply HTML formatting if requested
         if config.prettiify:
@@ -652,7 +659,7 @@ async def aprocess_html(
             cleaned_html=cleaned_html,
             markdown=markdown_result,
             media=media,
-            tables=tables,                       # NEW
+            tables=tables,  # NEW
             links=links,
             metadata=metadata,
             screenshot=screenshot_data,
@@ -770,13 +777,35 @@ async def result_transformer():
             return result_transformer()
         else:
             _results = await dispatcher.run_urls(crawler=self, urls=urls, config=config)
-            return [transform_result(res) for res in _results]
+
+            data = []
+            from .models import CrawlerTaskResult
+
+            for task_result in _results:
+                for res in task_result.result:
+                    # all sub urls within parent url will have same dispatch_results
+                    tsk = CrawlerTaskResult(
+                        task_id=task_result.task_id,
+                        url=task_result.url,
+                        result=res,
+                        memory_usage=task_result.memory_usage,
+                        peak_memory=task_result.peak_memory,
+                        start_time=task_result.start_time,
+                        end_time=task_result.end_time,
+                        error_message=task_result.error_message,
+                        retry_count=task_result.retry_count,
+                        wait_time=task_result.wait_time,
+                    )
+                    data.append(tsk)
+            _results = data
+            results = [transform_result(res) for res in _results]
+            return results
 
     async def aseed_urls(
         self,
         domain_or_domains: Union[str, List[str]],
         config: Optional[SeedingConfig] = None,
-        **kwargs
+        **kwargs,
     ) -> Union[List[str], Dict[str, List[Union[str, Dict[str, Any]]]]]:
         """
         Discovers, filters, and optionally validates URLs for a given domain(s)
@@ -819,15 +848,18 @@ async def aseed_urls(
             # Pass the crawler's base_directory for seeder's cache management
             # Pass the crawler's logger for consistent logging
             self.url_seeder = AsyncUrlSeeder(
-                base_directory=self.crawl4ai_folder,
-                logger=self.logger
-            )                    
+                base_directory=self.crawl4ai_folder, logger=self.logger
+            )
 
         # Merge config object with direct kwargs, giving kwargs precedence
-        seeding_config = config.clone(**kwargs) if config else SeedingConfig.from_kwargs(kwargs)
-        
+        seeding_config = (
+            config.clone(**kwargs) if config else SeedingConfig.from_kwargs(kwargs)
+        )
+
         # Ensure base_directory is set for the seeder's cache
-        seeding_config.base_directory = seeding_config.base_directory or self.crawl4ai_folder        
+        seeding_config.base_directory = (
+            seeding_config.base_directory or self.crawl4ai_folder
+        )
         # Ensure the seeder uses the crawler's logger (if not already set)
         if not self.url_seeder.logger:
             self.url_seeder.logger = self.logger
@@ -835,30 +867,25 @@ async def aseed_urls(
         # Pass verbose setting if explicitly provided in SeedingConfig or kwargs
         if seeding_config.verbose is not None:
             self.url_seeder.logger.verbose = seeding_config.verbose
-        else: # Default to crawler's verbose setting
+        else:  # Default to crawler's verbose setting
             self.url_seeder.logger.verbose = self.logger.verbose
 
-
         if isinstance(domain_or_domains, str):
             self.logger.info(
                 message="Starting URL seeding for domain: {domain}",
                 tag="SEED",
-                params={"domain": domain_or_domains}
-            )
-            return await self.url_seeder.urls(
-                domain_or_domains,
-                seeding_config
+                params={"domain": domain_or_domains},
             )
+            return await self.url_seeder.urls(domain_or_domains, seeding_config)
         elif isinstance(domain_or_domains, (list, tuple)):
             self.logger.info(
                 message="Starting URL seeding for {count} domains",
                 tag="SEED",
-                params={"count": len(domain_or_domains)}
+                params={"count": len(domain_or_domains)},
             )
             # AsyncUrlSeeder.many_urls directly accepts a list of domains and individual params.
-            return await self.url_seeder.many_urls(
-                domain_or_domains,
-                seeding_config
-            )
+            return await self.url_seeder.many_urls(domain_or_domains, seeding_config)
         else:
-            raise ValueError("`domain_or_domains` must be a string or a list of strings.")
\ No newline at end of file
+            raise ValueError(
+                "`domain_or_domains` must be a string or a list of strings."
+            )
diff --git a/crawl4ai/deep_crawling/base_strategy.py b/crawl4ai/deep_crawling/base_strategy.py
index e1b3fe6bd..e42f89820 100644
--- a/crawl4ai/deep_crawling/base_strategy.py
+++ b/crawl4ai/deep_crawling/base_strategy.py
@@ -5,47 +5,80 @@
 from functools import wraps
 from contextvars import ContextVar
 from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
+from ..models import CrawlResultContainer
 
 
 class DeepCrawlDecorator:
     """Decorator that adds deep crawling capability to arun method."""
+
     deep_crawl_active = ContextVar("deep_crawl_active", default=False)
-    
-    def __init__(self, crawler: AsyncWebCrawler): 
+
+    def __init__(self, crawler: AsyncWebCrawler):
         self.crawler = crawler
 
     def __call__(self, original_arun):
         @wraps(original_arun)
         async def wrapped_arun(url: str, config: CrawlerRunConfig = None, **kwargs):
             # If deep crawling is already active, call the original method to avoid recursion.
-            if config and config.deep_crawl_strategy and not self.deep_crawl_active.get():
+            if (
+                config
+                and config.deep_crawl_strategy
+                and not self.deep_crawl_active.get()
+            ):
                 token = self.deep_crawl_active.set(True)
                 # Await the arun call to get the actual result object.
                 result_obj = await config.deep_crawl_strategy.arun(
-                    crawler=self.crawler,
-                    start_url=url,
-                    config=config
+                    crawler=self.crawler, start_url=url, config=config
                 )
                 if config.stream:
+
                     async def result_wrapper():
                         try:
                             async for result in result_obj:
                                 yield result
                         finally:
                             self.deep_crawl_active.reset(token)
+
                     return result_wrapper()
                 else:
                     try:
+                        """
+                        # before
+                        result_obj = [
+                            CrawlResultContainer([
+                                CrawlResult(...)
+                            ]),
+                            CrawlResultContainer([
+                                CrawlResult(...)
+                            ]),
+                        ]
+                        # after
+                        result_obj = CrawlResultContainer([
+                            CrawlResult(...),
+                            CrawlResult(...),
+                            ...
+                        ])
+                        """
+
+                        crwl_rslt = [
+                            data
+                            for crwl_cntr in result_obj
+                            for data in crwl_cntr._results
+                        ]
+                        result_obj = CrawlResultContainer(results=crwl_rslt)
                         return result_obj
+
                     finally:
                         self.deep_crawl_active.reset(token)
             return await original_arun(url, config=config, **kwargs)
+
         return wrapped_arun
 
+
 class DeepCrawlStrategy(ABC):
     """
     Abstract base class for deep crawling strategies.
-    
+
     Core functions:
       - arun: Main entry point that returns an async generator of CrawlResults.
       - shutdown: Clean up resources.
@@ -78,7 +111,7 @@ async def _arun_stream(
         Processes one BFS level at a time and yields results immediately as they arrive.
         """
         pass
-    
+
     async def arun(
         self,
         start_url: str,
@@ -87,12 +120,12 @@ async def arun(
     ) -> RunManyReturn:
         """
         Traverse the given URL using the specified crawler.
-        
+
         Args:
             start_url (str): The URL from which to start crawling.
             crawler (AsyncWebCrawler): The crawler instance to use.
             crawler_run_config (Optional[CrawlerRunConfig]): Crawler configuration.
-        
+
         Returns:
             Union[CrawlResultT, List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]
         """
@@ -104,7 +137,9 @@ async def arun(
         else:
             return await self._arun_batch(start_url, crawler, config)
 
-    def __call__(self, start_url: str, crawler: AsyncWebCrawler, config: CrawlerRunConfig):
+    def __call__(
+        self, start_url: str, crawler: AsyncWebCrawler, config: CrawlerRunConfig
+    ):
         return self.arun(start_url, crawler, config)
 
     @abstractmethod
@@ -118,11 +153,11 @@ async def shutdown(self) -> None:
     async def can_process_url(self, url: str, depth: int) -> bool:
         """
         Validate the URL format and apply custom filtering logic.
-        
+
         Args:
             url (str): The URL to validate.
             depth (int): The current depth in the crawl.
-        
+
         Returns:
             bool: True if the URL should be processed, False otherwise.
         """
@@ -140,13 +175,13 @@ async def link_discovery(
     ) -> None:
         """
         Extract and process links from the given crawl result.
-        
+
         This method should:
           - Validate each extracted URL using can_process_url.
           - Optionally score URLs.
           - Append valid URLs (and their parent references) to the next_level list.
           - Update the depths dictionary with the new depth for each URL.
-        
+
         Args:
             result (CrawlResult): The result from a crawl operation.
             source_url (str): The URL from which this result was obtained.
@@ -156,4 +191,3 @@ async def link_discovery(
             depths (Dict[str, int]): Mapping of URLs to their current depth.
         """
         pass
-
diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.py
index eb699f828..28432074b 100644
--- a/crawl4ai/deep_crawling/bfs_strategy.py
+++ b/crawl4ai/deep_crawling/bfs_strategy.py
@@ -8,25 +8,27 @@
 from ..models import TraversalStats
 from .filters import FilterChain
 from .scorers import URLScorer
-from . import DeepCrawlStrategy  
+from . import DeepCrawlStrategy
 from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult
 from ..utils import normalize_url_for_deep_crawl, efficient_normalize_url_for_deep_crawl
 from math import inf as infinity
 
+
 class BFSDeepCrawlStrategy(DeepCrawlStrategy):
     """
     Breadth-First Search deep crawling strategy.
-    
+
     Core functions:
       - arun: Main entry point; splits execution into batch or stream modes.
       - link_discovery: Extracts, filters, and (if needed) scores the outgoing URLs.
       - can_process_url: Validates URL format and applies the filter chain.
     """
+
     def __init__(
         self,
         max_depth: int,
         filter_chain: FilterChain = FilterChain(),
-        url_scorer: Optional[URLScorer] = None,        
+        url_scorer: Optional[URLScorer] = None,
         include_external: bool = False,
         score_threshold: float = -infinity,
         max_pages: int = infinity,
@@ -85,7 +87,7 @@ async def link_discovery(
         prepares the next level of URLs.
         Each valid URL is appended to next_level as a tuple (url, parent_url)
         and its depth is tracked.
-        """            
+        """
         next_depth = current_depth + 1
         if next_depth > self.max_depth:
             return
@@ -93,7 +95,9 @@ async def link_discovery(
         # If we've reached the max pages limit, don't discover new links
         remaining_capacity = self.max_pages - self._pages_crawled
         if remaining_capacity <= 0:
-            self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping link discovery")
+            self.logger.info(
+                f"Max pages limit ({self.max_pages}) reached, stopping link discovery"
+            )
             return
 
         # Get internal links and, if enabled, external links.
@@ -102,7 +106,7 @@ async def link_discovery(
             links += result.links.get("external", [])
 
         valid_links = []
-        
+
         # First collect all valid links
         for link in links:
             url = link.get("href")
@@ -117,16 +121,18 @@ async def link_discovery(
 
             # Score the URL if a scorer is provided
             score = self.url_scorer.score(base_url) if self.url_scorer else 0
-            
+
             # Skip URLs with scores below the threshold
             if score < self.score_threshold:
-                self.logger.debug(f"URL {url} skipped: score {score} below threshold {self.score_threshold}")
+                self.logger.debug(
+                    f"URL {url} skipped: score {score} below threshold {self.score_threshold}"
+                )
                 self.stats.urls_skipped += 1
                 continue
 
             visited.add(base_url)
             valid_links.append((base_url, score))
-        
+
         # If we have more valid links than capacity, sort by score and take the top ones
         if len(valid_links) > remaining_capacity:
             if self.url_scorer:
@@ -134,8 +140,10 @@ async def link_discovery(
                 valid_links.sort(key=lambda x: x[1], reverse=True)
             # Take only as many as we have capacity for
             valid_links = valid_links[:remaining_capacity]
-            self.logger.info(f"Limiting to {remaining_capacity} URLs due to max_pages limit")
-            
+            self.logger.info(
+                f"Limiting to {remaining_capacity} URLs due to max_pages limit"
+            )
+
         # Process the final selected links
         for url, score in valid_links:
             # attach the score to metadata if needed
@@ -165,33 +173,39 @@ async def _arun_batch(
         while current_level and not self._cancel_event.is_set():
             # Check if we've already reached max_pages before starting a new level
             if self._pages_crawled >= self.max_pages:
-                self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
+                self.logger.info(
+                    f"Max pages limit ({self.max_pages}) reached, stopping crawl"
+                )
                 break
-            
+
             next_level: List[Tuple[str, Optional[str]]] = []
             urls = [url for url, _ in current_level]
 
             # Clone the config to disable deep crawling recursion and enforce batch mode.
             batch_config = config.clone(deep_crawl_strategy=None, stream=False)
             batch_results = await crawler.arun_many(urls=urls, config=batch_config)
-            
+
             # Update pages crawled counter - count only successful crawls
             successful_results = [r for r in batch_results if r.success]
             self._pages_crawled += len(successful_results)
-            
+
             for result in batch_results:
                 url = result.url
                 depth = depths.get(url, 0)
                 result.metadata = result.metadata or {}
                 result.metadata["depth"] = depth
-                parent_url = next((parent for (u, parent) in current_level if u == url), None)
+                parent_url = next(
+                    (parent for (u, parent) in current_level if u == url), None
+                )
                 result.metadata["parent_url"] = parent_url
                 results.append(result)
-                
+
                 # Only discover links from successful crawls
                 if result.success:
                     # Link discovery will handle the max pages limit internally
-                    await self.link_discovery(result, url, depth, visited, next_level, depths)
+                    await self.link_discovery(
+                        result, url, depth, visited, next_level, depths
+                    )
 
             current_level = next_level
 
@@ -218,7 +232,7 @@ async def _arun_stream(
 
             stream_config = config.clone(deep_crawl_strategy=None, stream=True)
             stream_gen = await crawler.arun_many(urls=urls, config=stream_config)
-            
+
             # Keep track of processed results for this batch
             results_count = 0
             async for result in stream_gen:
@@ -226,30 +240,38 @@ async def _arun_stream(
                 depth = depths.get(url, 0)
                 result.metadata = result.metadata or {}
                 result.metadata["depth"] = depth
-                parent_url = next((parent for (u, parent) in current_level if u == url), None)
+                parent_url = next(
+                    (parent for (u, parent) in current_level if u == url), None
+                )
                 result.metadata["parent_url"] = parent_url
-                
+
                 # Count only successful crawls
                 if result.success:
                     self._pages_crawled += 1
                     # Check if we've reached the limit during batch processing
                     if self._pages_crawled >= self.max_pages:
-                        self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")
+                        self.logger.info(
+                            f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl"
+                        )
                         break  # Exit the generator
-                
+
                 results_count += 1
                 yield result
-                
+
                 # Only discover links from successful crawls
                 if result.success:
                     # Link discovery will handle the max pages limit internally
-                    await self.link_discovery(result, url, depth, visited, next_level, depths)
-            
+                    await self.link_discovery(
+                        result, url, depth, visited, next_level, depths
+                    )
+
             # If we didn't get results back (e.g. due to errors), avoid getting stuck in an infinite loop
             # by considering these URLs as visited but not counting them toward the max_pages limit
             if results_count == 0 and urls:
-                self.logger.warning(f"No results returned for {len(urls)} URLs, marking as visited")
-                
+                self.logger.warning(
+                    f"No results returned for {len(urls)} URLs, marking as visited"
+                )
+
             current_level = next_level
 
     async def shutdown(self) -> None:
diff --git a/crawl4ai/models.py b/crawl4ai/models.py
index 63e398855..777245ae8 100644
--- a/crawl4ai/models.py
+++ b/crawl4ai/models.py
@@ -31,17 +31,19 @@ class CrawlerTaskResult:
     error_message: str = ""
     retry_count: int = 0
     wait_time: float = 0.0
-    
+
     @property
     def success(self) -> bool:
         return self.result.success
 
+
 class CrawlStatus(Enum):
     QUEUED = "QUEUED"
     IN_PROGRESS = "IN_PROGRESS"
     COMPLETED = "COMPLETED"
     FAILED = "FAILED"
 
+
 @dataclass
 class CrawlStats:
     task_id: str
@@ -60,21 +62,22 @@ class CrawlStats:
     def duration(self) -> str:
         if not self.start_time:
             return "0:00"
-            
+
         # Convert start_time to datetime if it's a float
         start = self.start_time
         if isinstance(start, float):
             start = datetime.fromtimestamp(start)
-            
+
         # Get end time or use current time
         end = self.end_time or datetime.now()
         # Convert end_time to datetime if it's a float
         if isinstance(end, float):
             end = datetime.fromtimestamp(end)
-            
+
         duration = end - start
         return str(timedelta(seconds=int(duration.total_seconds())))
 
+
 class DisplayMode(Enum):
     DETAILED = "DETAILED"
     AGGREGATED = "AGGREGATED"
@@ -91,12 +94,12 @@ class TokenUsage:
     completion_tokens_details: Optional[dict] = None
     prompt_tokens_details: Optional[dict] = None
 
+
 class UrlModel(BaseModel):
     url: HttpUrl
     forced: bool = False
 
 
-
 @dataclass
 class TraversalStats:
     """Statistics for the traversal process"""
@@ -108,6 +111,7 @@ class TraversalStats:
     total_depth_reached: int = 0
     current_depth: int = 0
 
+
 class DispatchResult(BaseModel):
     task_id: str
     memory_usage: float
@@ -116,6 +120,7 @@ class DispatchResult(BaseModel):
     end_time: Union[datetime, float]
     error_message: str = ""
 
+
 class MarkdownGenerationResult(BaseModel):
     raw_markdown: str
     markdown_with_citations: str
@@ -125,7 +130,8 @@ class MarkdownGenerationResult(BaseModel):
 
     def __str__(self):
         return self.raw_markdown
-    
+
+
 class CrawlResult(BaseModel):
     url: str
     html: str
@@ -151,25 +157,27 @@ class CrawlResult(BaseModel):
     redirected_url: Optional[str] = None
     network_requests: Optional[List[Dict[str, Any]]] = None
     console_messages: Optional[List[Dict[str, Any]]] = None
-    tables: List[Dict] = Field(default_factory=list)  # NEW – [{headers,rows,caption,summary}]
+    tables: List[Dict] = Field(
+        default_factory=list
+    )  # NEW – [{headers,rows,caption,summary}]
 
     class Config:
         arbitrary_types_allowed = True
 
-# NOTE: The StringCompatibleMarkdown class, custom __init__ method, property getters/setters,
-# and model_dump override all exist to support a smooth transition from markdown as a string
-# to markdown as a MarkdownGenerationResult object, while maintaining backward compatibility.
-# 
-# This allows code that expects markdown to be a string to continue working, while also
-# providing access to the full MarkdownGenerationResult object's properties.
-# 
-# The markdown_v2 property is deprecated and raises an error directing users to use markdown.
-# 
-# When backward compatibility is no longer needed in future versions, this entire mechanism
-# can be simplified to a standard field with no custom accessors or serialization logic.
-    
+    # NOTE: The StringCompatibleMarkdown class, custom __init__ method, property getters/setters,
+    # and model_dump override all exist to support a smooth transition from markdown as a string
+    # to markdown as a MarkdownGenerationResult object, while maintaining backward compatibility.
+    #
+    # This allows code that expects markdown to be a string to continue working, while also
+    # providing access to the full MarkdownGenerationResult object's properties.
+    #
+    # The markdown_v2 property is deprecated and raises an error directing users to use markdown.
+    #
+    # When backward compatibility is no longer needed in future versions, this entire mechanism
+    # can be simplified to a standard field with no custom accessors or serialization logic.
+
     def __init__(self, **data):
-        markdown_result = data.pop('markdown', None)
+        markdown_result = data.pop("markdown", None)
         super().__init__(**data)
         if markdown_result is not None:
             self._markdown = (
@@ -177,27 +185,27 @@ def __init__(self, **data):
                 if isinstance(markdown_result, dict)
                 else markdown_result
             )
-    
+
     @property
     def markdown(self):
         """
         Property that returns a StringCompatibleMarkdown object that behaves like
         a string but also provides access to MarkdownGenerationResult attributes.
-        
+
         This approach allows backward compatibility with code that expects 'markdown'
         to be a string, while providing access to the full MarkdownGenerationResult.
         """
         if self._markdown is None:
             return None
         return StringCompatibleMarkdown(self._markdown)
-    
+
     @markdown.setter
     def markdown(self, value):
         """
         Setter for the markdown property.
         """
         self._markdown = value
-    
+
     @property
     def markdown_v2(self):
         """
@@ -216,7 +224,7 @@ def markdown_v2(self):
             - fit_markdown: The markdown string with fit text
             """
         )
-    
+
     @property
     def fit_markdown(self):
         """
@@ -226,7 +234,7 @@ def fit_markdown(self):
             "The 'fit_markdown' attribute is deprecated and has been removed. "
             "Please use 'markdown.fit_markdown' instead."
         )
-    
+
     @property
     def fit_html(self):
         """
@@ -240,45 +248,49 @@ def fit_html(self):
     def model_dump(self, *args, **kwargs):
         """
         Override model_dump to include the _markdown private attribute in serialization.
-        
+
         This override is necessary because:
         1. PrivateAttr fields are excluded from serialization by default
         2. We need to maintain backward compatibility by including the 'markdown' field
            in the serialized output
         3. We're transitioning from 'markdown_v2' to enhancing 'markdown' to hold
            the same type of data
-        
+
         Future developers: This method ensures that the markdown content is properly
         serialized despite being stored in a private attribute. If the serialization
         requirements change, this is where you would update the logic.
         """
         result = super().model_dump(*args, **kwargs)
-        
+
         # Remove any property descriptors that might have been included
         # These deprecated properties should not be in the serialized output
-        for key in ['fit_html', 'fit_markdown', 'markdown_v2']:
+        for key in ["fit_html", "fit_markdown", "markdown_v2"]:
             if key in result and isinstance(result[key], property):
                 # del result[key]
                 # Nasrin: I decided to convert it to string instead of removing it.
                 result[key] = str(result[key])
-        
+
         # Add the markdown field properly
         if self._markdown is not None:
-            result["markdown"] = self._markdown.model_dump() 
+            result["markdown"] = self._markdown.model_dump()
         return result
 
+
 class StringCompatibleMarkdown(str):
     """A string subclass that also provides access to MarkdownGenerationResult attributes"""
+
     def __new__(cls, markdown_result):
         return super().__new__(cls, markdown_result.raw_markdown)
-    
+
     def __init__(self, markdown_result):
         self._markdown_result = markdown_result
-    
+
     def __getattr__(self, name):
         return getattr(self._markdown_result, name)
 
-CrawlResultT = TypeVar('CrawlResultT', bound=CrawlResult)
+
+CrawlResultT = TypeVar("CrawlResultT", bound=CrawlResult)
+
 
 class CrawlResultContainer(Generic[CrawlResultT]):
     def __init__(self, results: Union[CrawlResultT, List[CrawlResultT]]):
@@ -301,14 +313,16 @@ def __getattr__(self, attr):
         # Delegate attribute access to the first element.
         if self._results:
             return getattr(self._results[0], attr)
-        raise AttributeError(f"{self.__class__.__name__} object has no attribute '{attr}'")
+        raise AttributeError(
+            f"{self.__class__.__name__} object has no attribute '{attr}'"
+        )
 
     def __repr__(self):
         return f"{self.__class__.__name__}({self._results!r})"
 
+
 RunManyReturn = Union[
-    CrawlResultContainer[CrawlResultT],
-    AsyncGenerator[CrawlResultT, None]
+    CrawlResultContainer[CrawlResultT], AsyncGenerator[CrawlResultT, None]
 ]
 
 
@@ -317,6 +331,7 @@ def __repr__(self):
 # 1. Replace the private attribute and property with a standard field
 # 2. Update any serialization logic that might depend on the current behavior
 
+
 class AsyncCrawlResponse(BaseModel):
     html: str
     response_headers: Dict[str, str]
@@ -335,6 +350,7 @@ class AsyncCrawlResponse(BaseModel):
     class Config:
         arbitrary_types_allowed = True
 
+
 ###############################
 # Scraping Models
 ###############################
@@ -355,12 +371,20 @@ class Link(BaseModel):
     text: Optional[str] = ""
     title: Optional[str] = ""
     base_domain: Optional[str] = ""
-    head_data: Optional[Dict[str, Any]] = None  # Head metadata extracted from link target
+    head_data: Optional[Dict[str, Any]] = (
+        None  # Head metadata extracted from link target
+    )
     head_extraction_status: Optional[str] = None  # "success", "failed", "skipped"
     head_extraction_error: Optional[str] = None  # Error message if extraction failed
-    intrinsic_score: Optional[float] = None  # Quality score based on URL structure, text, and context
-    contextual_score: Optional[float] = None  # BM25 relevance score based on query and head content
-    total_score: Optional[float] = None  # Combined score from intrinsic and contextual scores
+    intrinsic_score: Optional[float] = (
+        None  # Quality score based on URL structure, text, and context
+    )
+    contextual_score: Optional[float] = (
+        None  # BM25 relevance score based on query and head content
+    )
+    total_score: Optional[float] = (
+        None  # Combined score from intrinsic and contextual scores
+    )
 
 
 class Media(BaseModel):