path: root/py/webget.py

import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import os
import time
from pathlib import Path
import re
import sys

class WebpageDownloader:
    def __init__(self, base_url, output_dir='downloaded_pages', ignore_navigation=True):
        self.base_url = base_url
        self.domain = urlparse(base_url).netloc
        self.output_dir = output_dir
        self.assets_dir = os.path.join(output_dir, 'assets')
        self.css_dir = os.path.join(self.assets_dir, 'css')
        self.downloaded_urls = set()
        self.downloaded_assets = {}  # Maps original URLs to local paths
        self.ignore_navigation = ignore_navigation
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        })

        # Common navigation elements to ignore
        self.nav_classes = {
            'nav', 'navbar', 'navigation', 'menu', 'sidebar', 'header', 'footer',
            'topbar', 'top-bar', 'site-nav', 'main-nav', 'primary-nav',
            'secondary-nav', 'breadcrumb', 'pagination'
        }
        self.nav_ids = self.nav_classes

        # Create necessary directories
        os.makedirs(self.css_dir, exist_ok=True)

    def clean_filename(self, url):
        """Convert URL to a valid filename."""
        # Extract the path and query components
        parsed = urlparse(url)
        path = parsed.path
        
        if not path or path == '/':
            path = 'index'
        else:
            path = path.strip('/')
            
        # Handle query parameters by appending them as a hash
        if parsed.query:
            query_hash = hashlib.md5(parsed.query.encode()).hexdigest()[:8]
            path = f"{path}_{query_hash}"
        
        # Clean the filename
        filename = re.sub(r'[<>:"/\\|?*]', '_', path)
        
        return filename

    def download_resource(self, url):
        """Download a resource (webpage or asset) and return its content."""
        try:
            response = self.session.get(url, timeout=10)
            response.raise_for_status()
            return response.content
        except requests.RequestException as e:
            print(f"Error downloading {url}: {e}")
            return None

    def download_and_save_css(self, url, base_url):
        """Download a CSS file and save it locally."""
        if url in self.downloaded_assets:
            return self.downloaded_assets[url]

        # Handle data URLs
        if url.startswith('data:'):
            return url

        # Convert relative URLs to absolute
        absolute_url = urljoin(base_url, url)
        
        # Only download from same domain
        if urlparse(absolute_url).netloc != self.domain:
            return url

        content = self.download_resource(absolute_url)
        if not content:
            return url

        # Generate filename for CSS
        css_filename = self.clean_filename(absolute_url)
        if not css_filename.endswith('.css'):
            css_filename += '.css'

        css_path = os.path.join(self.css_dir, css_filename)
        
        # Save the CSS file
        with open(css_path, 'wb') as f:
            f.write(content)

        # Store the relative path from HTML to CSS
        relative_path = os.path.relpath(css_path, self.output_dir)
        self.downloaded_assets[url] = relative_path
        return relative_path

    def process_css_urls(self, css_content, base_url):
        """Process and update URLs within CSS content."""
        def replace_url(match):
            url = match.group(1).strip('"\'')
            if url.startswith(('data:', 'http:', 'https:')):
                return f"url({url})"
            absolute_url = urljoin(base_url, url)
            return f"url({absolute_url})"

        # Replace URLs in CSS
        return re.sub(r'url\((.*?)\)', replace_url, css_content.decode('utf-8'))

    def save_page(self, content, url):
        """Save the webpage content to a file and process its CSS."""
        if not content:
            return

        soup = BeautifulSoup(content, 'html.parser')

        # Process external stylesheets
        for link in soup.find_all('link', rel='stylesheet'):
            if 'href' in link.attrs:
                css_path = self.download_and_save_css(link['href'], url)
                link['href'] = css_path

        # Process inline styles
        for style in soup.find_all('style'):
            if style.string:
                style.string = self.process_css_urls(style.string, url)

        # Save the processed HTML
        filename = self.clean_filename(url)
        if not filename.endswith('.html'):
            filename += '.html'
            
        filepath = os.path.join(self.output_dir, filename)
        
        os.makedirs(os.path.dirname(filepath), exist_ok=True)
        
        with open(filepath, 'w', encoding='utf-8') as f:
            f.write(str(soup))
        print(f"Saved: {filepath}")

    def is_navigation_element(self, element):
        """Check if an element is likely part of navigation."""
        if not element:
            return False

        current = element
        while current:
            if current.get('class'):
                if any(nav_class in ' '.join(current['class']).lower() 
                      for nav_class in self.nav_classes):
                    return True

            if current.get('id'):
                if current['id'].lower() in self.nav_ids:
                    return True

            if current.get('role'):
                if current['role'].lower() in {'navigation', 'menu', 'menubar'}:
                    return True

            if current.name in {'nav', 'header', 'footer'}:
                return True

            current = current.parent

        return False

    def extract_links(self, content, current_url):
        """Extract all same-domain links from the page."""
        soup = BeautifulSoup(content, 'html.parser')
        links = set()
        
        main_content = soup.find(['main', 'article']) or soup.find(class_='content') or soup.find(id='content')
        
        for a in (main_content or soup).find_all('a', href=True):
            if self.ignore_navigation and self.is_navigation_element(a):
                continue
                
            href = a['href']
            if href.startswith('#'):
                continue
                
            absolute_url = urljoin(current_url, href)
            
            if urlparse(absolute_url).netloc == self.domain:
                links.add(absolute_url)
                
        return links

    def download_site(self, max_pages=50):
        """Download the website and its linked pages."""
        urls_to_process = {self.base_url}
        pages_downloaded = 0
        
        while urls_to_process and pages_downloaded < max_pages:
            current_url = urls_to_process.pop()
            
            if current_url in self.downloaded_urls:
                continue
                
            print(f"Downloading: {current_url}")
            content = self.download_resource(current_url)
            
            if content:
                self.save_page(content, current_url)
                self.downloaded_urls.add(current_url)
                pages_downloaded += 1
                
                new_links = self.extract_links(content, current_url)
                urls_to_process.update(new_links - self.downloaded_urls)
                
            time.sleep(1)
            
        print(f"\nDownload complete! Downloaded {pages_downloaded} pages to {self.output_dir}")

if __name__ == "__main__":
    
    # Example usage
    url = "https://www.fractalpress.com"  # Replace with your target URL
    if len(sys.argv) > 1:
        url = sys.argv[1]

    # Create downloader with navigation filtering enabled
    downloader = WebpageDownloader(
        url,
        output_dir='downloaded_pages',
        ignore_navigation=True  # Set to False to include navigation links
    )
    
    downloader.download_site(max_pages=5)