python stuffHEADmain

3 files changed, 456 insertions, 0 deletions

diff --git a/py/util.py b/py/util.py
new file mode 100755
index 0000000..ad5fa62
--- /dev/null
+++ b/py/util.py
@@ -0,0 +1,41 @@
+from datetime import datetime
+import os
+import subprocess
+from pathlib import Path
+
+#TODO bring code over from complete module
+
+datadir=Path.home() / '.local' / 'share' / 'pyutil'
+logdir=datadir / 'log'
+
+
+def cmd(cmdstr,v=False):
+    '''run cmdstr as a command in the shell, return the output'''
+    cmdarray = cmdstr.strip().split(' ')
+    log(f'runcmd: {cmdstr}')
+    #TODO handling pipe not yet working
+    proc = subprocess.run(cmdarray, stdout=subprocess.PIPE)
+    if proc.returncode == 0:
+        res = proc.stdout
+    else:
+        log(f'returncode = {proc.returncode}')
+        res = proc.stderr
+    return res
+
+def log(msg):
+    d = datetime.now().strftime('%Y%m%d')
+    fn = logdir / f'log-{d}.txt'
+    if not logdir.exists():
+        os.makedirs(logdir, exist_ok=True)
+    with open(fn, 'a') as lf:
+        lf.write(f'{d}: {msg}\n')
+        lf.close()
+
+def tnow():
+    '''
+    return: current time, formatted as %Y%m%d-%H%M%S
+    ''' 
+    return datetime.now().strftime('%Y%m%d-%H%M%S')
+
+
+############################
\ No newline at end of file
diff --git a/py/webget.py b/py/webget.py
new file mode 100755
index 0000000..b71d87a
--- /dev/null
+++ b/py/webget.py
@@ -0,0 +1,235 @@
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+import os
+import time
+from pathlib import Path
+import re
+import sys
+
+class WebpageDownloader:
+    def __init__(self, base_url, output_dir='downloaded_pages', ignore_navigation=True):
+        self.base_url = base_url
+        self.domain = urlparse(base_url).netloc
+        self.output_dir = output_dir
+        self.assets_dir = os.path.join(output_dir, 'assets')
+        self.css_dir = os.path.join(self.assets_dir, 'css')
+        self.downloaded_urls = set()
+        self.downloaded_assets = {}  # Maps original URLs to local paths
+        self.ignore_navigation = ignore_navigation
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
+        })
+
+        # Common navigation elements to ignore
+        self.nav_classes = {
+            'nav', 'navbar', 'navigation', 'menu', 'sidebar', 'header', 'footer',
+            'topbar', 'top-bar', 'site-nav', 'main-nav', 'primary-nav',
+            'secondary-nav', 'breadcrumb', 'pagination'
+        }
+        self.nav_ids = self.nav_classes
+
+        # Create necessary directories
+        os.makedirs(self.css_dir, exist_ok=True)
+
+    def clean_filename(self, url):
+        """Convert URL to a valid filename."""
+        # Extract the path and query components
+        parsed = urlparse(url)
+        path = parsed.path
+        
+        if not path or path == '/':
+            path = 'index'
+        else:
+            path = path.strip('/')
+            
+        # Handle query parameters by appending them as a hash
+        if parsed.query:
+            query_hash = hashlib.md5(parsed.query.encode()).hexdigest()[:8]
+            path = f"{path}_{query_hash}"
+        
+        # Clean the filename
+        filename = re.sub(r'[<>:"/\\|?*]', '_', path)
+        
+        return filename
+
+    def download_resource(self, url):
+        """Download a resource (webpage or asset) and return its content."""
+        try:
+            response = self.session.get(url, timeout=10)
+            response.raise_for_status()
+            return response.content
+        except requests.RequestException as e:
+            print(f"Error downloading {url}: {e}")
+            return None
+
+    def download_and_save_css(self, url, base_url):
+        """Download a CSS file and save it locally."""
+        if url in self.downloaded_assets:
+            return self.downloaded_assets[url]
+
+        # Handle data URLs
+        if url.startswith('data:'):
+            return url
+
+        # Convert relative URLs to absolute
+        absolute_url = urljoin(base_url, url)
+        
+        # Only download from same domain
+        if urlparse(absolute_url).netloc != self.domain:
+            return url
+
+        content = self.download_resource(absolute_url)
+        if not content:
+            return url
+
+        # Generate filename for CSS
+        css_filename = self.clean_filename(absolute_url)
+        if not css_filename.endswith('.css'):
+            css_filename += '.css'
+
+        css_path = os.path.join(self.css_dir, css_filename)
+        
+        # Save the CSS file
+        with open(css_path, 'wb') as f:
+            f.write(content)
+
+        # Store the relative path from HTML to CSS
+        relative_path = os.path.relpath(css_path, self.output_dir)
+        self.downloaded_assets[url] = relative_path
+        return relative_path
+
+    def process_css_urls(self, css_content, base_url):
+        """Process and update URLs within CSS content."""
+        def replace_url(match):
+            url = match.group(1).strip('"\'')
+            if url.startswith(('data:', 'http:', 'https:')):
+                return f"url({url})"
+            absolute_url = urljoin(base_url, url)
+            return f"url({absolute_url})"
+
+        # Replace URLs in CSS
+        return re.sub(r'url\((.*?)\)', replace_url, css_content.decode('utf-8'))
+
+    def save_page(self, content, url):
+        """Save the webpage content to a file and process its CSS."""
+        if not content:
+            return
+
+        soup = BeautifulSoup(content, 'html.parser')
+
+        # Process external stylesheets
+        for link in soup.find_all('link', rel='stylesheet'):
+            if 'href' in link.attrs:
+                css_path = self.download_and_save_css(link['href'], url)
+                link['href'] = css_path
+
+        # Process inline styles
+        for style in soup.find_all('style'):
+            if style.string:
+                style.string = self.process_css_urls(style.string, url)
+
+        # Save the processed HTML
+        filename = self.clean_filename(url)
+        if not filename.endswith('.html'):
+            filename += '.html'
+            
+        filepath = os.path.join(self.output_dir, filename)
+        
+        os.makedirs(os.path.dirname(filepath), exist_ok=True)
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            f.write(str(soup))
+        print(f"Saved: {filepath}")
+
+    def is_navigation_element(self, element):
+        """Check if an element is likely part of navigation."""
+        if not element:
+            return False
+
+        current = element
+        while current:
+            if current.get('class'):
+                if any(nav_class in ' '.join(current['class']).lower() 
+                      for nav_class in self.nav_classes):
+                    return True
+
+            if current.get('id'):
+                if current['id'].lower() in self.nav_ids:
+                    return True
+
+            if current.get('role'):
+                if current['role'].lower() in {'navigation', 'menu', 'menubar'}:
+                    return True
+
+            if current.name in {'nav', 'header', 'footer'}:
+                return True
+
+            current = current.parent
+
+        return False
+
+    def extract_links(self, content, current_url):
+        """Extract all same-domain links from the page."""
+        soup = BeautifulSoup(content, 'html.parser')
+        links = set()
+        
+        main_content = soup.find(['main', 'article']) or soup.find(class_='content') or soup.find(id='content')
+        
+        for a in (main_content or soup).find_all('a', href=True):
+            if self.ignore_navigation and self.is_navigation_element(a):
+                continue
+                
+            href = a['href']
+            if href.startswith('#'):
+                continue
+                
+            absolute_url = urljoin(current_url, href)
+            
+            if urlparse(absolute_url).netloc == self.domain:
+                links.add(absolute_url)
+                
+        return links
+
+    def download_site(self, max_pages=50):
+        """Download the website and its linked pages."""
+        urls_to_process = {self.base_url}
+        pages_downloaded = 0
+        
+        while urls_to_process and pages_downloaded < max_pages:
+            current_url = urls_to_process.pop()
+            
+            if current_url in self.downloaded_urls:
+                continue
+                
+            print(f"Downloading: {current_url}")
+            content = self.download_resource(current_url)
+            
+            if content:
+                self.save_page(content, current_url)
+                self.downloaded_urls.add(current_url)
+                pages_downloaded += 1
+                
+                new_links = self.extract_links(content, current_url)
+                urls_to_process.update(new_links - self.downloaded_urls)
+                
+            time.sleep(1)
+            
+        print(f"\nDownload complete! Downloaded {pages_downloaded} pages to {self.output_dir}")
+
+if __name__ == "__main__":
+    
+    # Example usage
+    url = "https://www.fractalpress.com"  # Replace with your target URL
+    if len(sys.argv) > 1:
+        url = sys.argv[1]
+
+    # Create downloader with navigation filtering enabled
+    downloader = WebpageDownloader(
+        url,
+        output_dir='downloaded_pages',
+        ignore_navigation=True  # Set to False to include navigation links
+    )
+    
+    downloader.download_site(max_pages=5)
\ No newline at end of file
diff --git a/py/webutils.py b/py/webutils.py
new file mode 100755
index 0000000..8e28460
--- /dev/null
+++ b/py/webutils.py
@@ -0,0 +1,180 @@
+#!/bin/python
+
+#some useful web-related scripts, such as crawling, searching for, downloading certain data
+
+import os
+
+import requests
+from bs4 import BeautifulSoup
+import re
+import sys
+import time
+from urllib.parse import urlsplit, urljoin
+from util import *
+
+#returns an array of img src urls from a 4chan thread
+def pull4chImgs(url):
+    result = []
+    resp = requests.get(url)
+    html = BeautifulSoup(resp.text, 'html.parser')
+    for a in html.find_all('a'):
+        if 'class' in a.attrs and 'fileThumb' in a.attrs['class']:
+            url = a.get('href')
+            if url[0:2] == '//':
+                url = 'https:' + url
+            result.append(url)
+    return result
+
+def pullVids(url):
+    result = []
+    resp = requests.get(url)
+    html = BeautifulSoup(resp.text, 'html.parser')
+    for a in html.find_all('a'):
+        if '.webm' in a.get('href'):
+            result.append(a.get('href'))
+    return result
+
+def pullImgs(url):
+    result = []
+    resp = requests.get(url)
+    html = BeautifulSoup(resp.text, 'html.parser')
+    for img in html.find_all('img'):
+        srcURL = img.get('src')
+        result.append(srcURL)
+    return result
+
+
+#i used this for downloading a bunch of pdfs from some prepper website
+def pullPDFs(url, depth=0, alreadycrawled=None, max_depth=5, delay=5.0, timeout=20.0, user_agent=None):
+    """Return a list of discovered PDF URLs by crawling from `url`.
+
+    Notes:
+    - Avoids mutable default args.
+    - `max_depth` and `delay` make it safer to run from a CLI.
+    """
+    if alreadycrawled is None:
+        alreadycrawled = []
+
+    if depth > max_depth or url == '' or url is None or url in alreadycrawled:
+        return []
+
+    baseurl = url[0:url.find('/', 8) + 1]
+    result = []
+    print(url)
+
+    headers = {}
+    if user_agent:
+        headers["User-Agent"] = user_agent
+
+    resp = requests.get(url, headers=headers, timeout=timeout)
+    html = BeautifulSoup(resp.text, 'html.parser')
+    alreadycrawled.append(url)
+
+    for a in html.find_all('a'):
+        url = a.get('href')
+        if url is None or url == '' or url == '/':
+            continue
+        if baseurl not in url and 'http' in url[0:4]:  # external
+            continue
+        print('found ' + url)
+        if 'http' not in url:
+            url = baseurl + url
+        if url.find('.pdf') > 0 and os.path.isfile(url):
+            result.append(url)
+        else:
+            time.sleep(delay)
+            result = result + pullPDFs(
+                url,
+                depth + 1,
+                alreadycrawled,
+                max_depth=max_depth,
+                delay=delay,
+                timeout=timeout,
+                user_agent=user_agent,
+            )
+    return result
+
+#return a list of strings of all the links on the given webpage (<a> elements) whose href contains the given search string
+def getLinksContainingStr(url, s):
+    result = []
+    resp = requests.get(url)
+    html = BeautifulSoup(resp.text, 'html.parser')
+    for link in html.find_all('a'):
+        if link == None:
+            print('non')
+            continue
+        h = link.get('href')
+        if h and s in h:
+            result.append(url + '/' + h)
+    return result
+
+def downloadFromBandcamp(url, path):
+    res = getAnchorWithPattern(url, ['track/','album/'])
+    log(f'found {len(res)} tracks/albums for {str(url)}')
+    if len(res) > 0:
+        for page in res:
+            print()
+            cmd(f'yt-dlp -x --audio-format flac --audio-quality 0 -P {path} {str(page)}')
+
+def downloadFromYoutubeMusic(url, path):
+    res = getAnchorWithPattern(url, ['browse/.*_.+'])
+    log(f'found {len(res)} tracks for {str(url)}')
+    if len(res) > 0:
+        for page in res:
+            cmd(f'ytdla -P {path} {str(page)}')
+
+#write each element of list l to file of path p
+def writeListToFile(l, path, append=False):
+    f = open(path, 'a' if append else 'w')
+    for e in l:
+        f.write(str(e))
+        f.write('\n')
+    f.close()
+
+def getAnchorWithPattern(url, pattern, content=None):
+    result = []
+    patterns = pattern if isinstance(pattern, (list, tuple, set)) else [pattern]
+    if content is None:
+        resp = requests.get(url)
+        html = BeautifulSoup(resp.text, 'html.parser')
+    else:
+        html = BeautifulSoup(content, 'html.parser')
+
+    parts = urlsplit(url)
+    base_url = f'{parts.scheme}://{parts.netloc}/'
+
+    for link in html.find_all('a'):
+        if link == None:
+            print('non')
+            continue
+        h = link.get('href')
+
+        if h and any(re.search(single_pattern, h) for single_pattern in patterns):
+            if h[0:4] != 'http':
+                h = urljoin(base_url, h.lstrip('/'))
+            result.append(h)
+        else:
+            print(h)
+    return result 
+
+def writeListToFile(l, path, append=False):
+    f = open(path, 'a' if append else 'w')
+    for e in l:
+        f.write(str(e))
+        f.write('\n')
+    f.close()
+
+def ytdlaFromFile(path):
+    '''downloads audio from each url in the file at path'''
+    with open(path, 'r') as f:
+        for line in f:
+            url = line.strip()
+            if url != '':
+                cmd('echo ' + url)
+
+#if len(sys.argv) < 2:
+#    sys.exit(0)
+
+#for url in pull4chImgs(sys.argv[1]):
+#    print(url)
+