import requests from bs4 import BeautifulSoup from urllib.parse import urljoin, urlparse import os import time from pathlib import Path import re import sys class WebpageDownloader: def __init__(self, base_url, output_dir='downloaded_pages', ignore_navigation=True): self.base_url = base_url self.domain = urlparse(base_url).netloc self.output_dir = output_dir self.assets_dir = os.path.join(output_dir, 'assets') self.css_dir = os.path.join(self.assets_dir, 'css') self.downloaded_urls = set() self.downloaded_assets = {} # Maps original URLs to local paths self.ignore_navigation = ignore_navigation self.session = requests.Session() self.session.headers.update({ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' }) # Common navigation elements to ignore self.nav_classes = { 'nav', 'navbar', 'navigation', 'menu', 'sidebar', 'header', 'footer', 'topbar', 'top-bar', 'site-nav', 'main-nav', 'primary-nav', 'secondary-nav', 'breadcrumb', 'pagination' } self.nav_ids = self.nav_classes # Create necessary directories os.makedirs(self.css_dir, exist_ok=True) def clean_filename(self, url): """Convert URL to a valid filename.""" # Extract the path and query components parsed = urlparse(url) path = parsed.path if not path or path == '/': path = 'index' else: path = path.strip('/') # Handle query parameters by appending them as a hash if parsed.query: query_hash = hashlib.md5(parsed.query.encode()).hexdigest()[:8] path = f"{path}_{query_hash}" # Clean the filename filename = re.sub(r'[<>:"/\\|?*]', '_', path) return filename def download_resource(self, url): """Download a resource (webpage or asset) and return its content.""" try: response = self.session.get(url, timeout=10) response.raise_for_status() return response.content except requests.RequestException as e: print(f"Error downloading {url}: {e}") return None def download_and_save_css(self, url, base_url): """Download a CSS file and save it locally.""" if url in self.downloaded_assets: return self.downloaded_assets[url] # Handle data URLs if url.startswith('data:'): return url # Convert relative URLs to absolute absolute_url = urljoin(base_url, url) # Only download from same domain if urlparse(absolute_url).netloc != self.domain: return url content = self.download_resource(absolute_url) if not content: return url # Generate filename for CSS css_filename = self.clean_filename(absolute_url) if not css_filename.endswith('.css'): css_filename += '.css' css_path = os.path.join(self.css_dir, css_filename) # Save the CSS file with open(css_path, 'wb') as f: f.write(content) # Store the relative path from HTML to CSS relative_path = os.path.relpath(css_path, self.output_dir) self.downloaded_assets[url] = relative_path return relative_path def process_css_urls(self, css_content, base_url): """Process and update URLs within CSS content.""" def replace_url(match): url = match.group(1).strip('"\'') if url.startswith(('data:', 'http:', 'https:')): return f"url({url})" absolute_url = urljoin(base_url, url) return f"url({absolute_url})" # Replace URLs in CSS return re.sub(r'url\((.*?)\)', replace_url, css_content.decode('utf-8')) def save_page(self, content, url): """Save the webpage content to a file and process its CSS.""" if not content: return soup = BeautifulSoup(content, 'html.parser') # Process external stylesheets for link in soup.find_all('link', rel='stylesheet'): if 'href' in link.attrs: css_path = self.download_and_save_css(link['href'], url) link['href'] = css_path # Process inline styles for style in soup.find_all('style'): if style.string: style.string = self.process_css_urls(style.string, url) # Save the processed HTML filename = self.clean_filename(url) if not filename.endswith('.html'): filename += '.html' filepath = os.path.join(self.output_dir, filename) os.makedirs(os.path.dirname(filepath), exist_ok=True) with open(filepath, 'w', encoding='utf-8') as f: f.write(str(soup)) print(f"Saved: {filepath}") def is_navigation_element(self, element): """Check if an element is likely part of navigation.""" if not element: return False current = element while current: if current.get('class'): if any(nav_class in ' '.join(current['class']).lower() for nav_class in self.nav_classes): return True if current.get('id'): if current['id'].lower() in self.nav_ids: return True if current.get('role'): if current['role'].lower() in {'navigation', 'menu', 'menubar'}: return True if current.name in {'nav', 'header', 'footer'}: return True current = current.parent return False def extract_links(self, content, current_url): """Extract all same-domain links from the page.""" soup = BeautifulSoup(content, 'html.parser') links = set() main_content = soup.find(['main', 'article']) or soup.find(class_='content') or soup.find(id='content') for a in (main_content or soup).find_all('a', href=True): if self.ignore_navigation and self.is_navigation_element(a): continue href = a['href'] if href.startswith('#'): continue absolute_url = urljoin(current_url, href) if urlparse(absolute_url).netloc == self.domain: links.add(absolute_url) return links def download_site(self, max_pages=50): """Download the website and its linked pages.""" urls_to_process = {self.base_url} pages_downloaded = 0 while urls_to_process and pages_downloaded < max_pages: current_url = urls_to_process.pop() if current_url in self.downloaded_urls: continue print(f"Downloading: {current_url}") content = self.download_resource(current_url) if content: self.save_page(content, current_url) self.downloaded_urls.add(current_url) pages_downloaded += 1 new_links = self.extract_links(content, current_url) urls_to_process.update(new_links - self.downloaded_urls) time.sleep(1) print(f"\nDownload complete! Downloaded {pages_downloaded} pages to {self.output_dir}") if __name__ == "__main__": # Example usage url = "https://www.fractalpress.com" # Replace with your target URL if len(sys.argv) > 1: url = sys.argv[1] # Create downloader with navigation filtering enabled downloader = WebpageDownloader( url, output_dir='downloaded_pages', ignore_navigation=True # Set to False to include navigation links ) downloader.download_site(max_pages=5)