1 files changed, 180 insertions, 0 deletions
diff --git a/py/webutils.py b/py/webutils.py
new file mode 100755
index 0000000..8e28460
--- /dev/null
+++ b/py/webutils.py
@@ -0,0 +1,180 @@
+#!/bin/python
+
+#some useful web-related scripts, such as crawling, searching for, downloading certain data
+
+import os
+
+import requests
+from bs4 import BeautifulSoup
+import re
+import sys
+import time
+from urllib.parse import urlsplit, urljoin
+from util import *
+
+#returns an array of img src urls from a 4chan thread
+def pull4chImgs(url):
+    result = []
+    resp = requests.get(url)
+    html = BeautifulSoup(resp.text, 'html.parser')
+    for a in html.find_all('a'):
+        if 'class' in a.attrs and 'fileThumb' in a.attrs['class']:
+            url = a.get('href')
+            if url[0:2] == '//':
+                url = 'https:' + url
+            result.append(url)
+    return result
+
+def pullVids(url):
+    result = []
+    resp = requests.get(url)
+    html = BeautifulSoup(resp.text, 'html.parser')
+    for a in html.find_all('a'):
+        if '.webm' in a.get('href'):
+            result.append(a.get('href'))
+    return result
+
+def pullImgs(url):
+    result = []
+    resp = requests.get(url)
+    html = BeautifulSoup(resp.text, 'html.parser')
+    for img in html.find_all('img'):
+        srcURL = img.get('src')
+        result.append(srcURL)
+    return result
+
+
+#i used this for downloading a bunch of pdfs from some prepper website
+def pullPDFs(url, depth=0, alreadycrawled=None, max_depth=5, delay=5.0, timeout=20.0, user_agent=None):
+    """Return a list of discovered PDF URLs by crawling from `url`.
+
+    Notes:
+    - Avoids mutable default args.
+    - `max_depth` and `delay` make it safer to run from a CLI.
+    """
+    if alreadycrawled is None:
+        alreadycrawled = []
+
+    if depth > max_depth or url == '' or url is None or url in alreadycrawled:
+        return []
+
+    baseurl = url[0:url.find('/', 8) + 1]
+    result = []
+    print(url)
+
+    headers = {}
+    if user_agent:
+        headers["User-Agent"] = user_agent
+
+    resp = requests.get(url, headers=headers, timeout=timeout)
+    html = BeautifulSoup(resp.text, 'html.parser')
+    alreadycrawled.append(url)
+
+    for a in html.find_all('a'):
+        url = a.get('href')
+        if url is None or url == '' or url == '/':
+            continue
+        if baseurl not in url and 'http' in url[0:4]:  # external
+            continue
+        print('found ' + url)
+        if 'http' not in url:
+            url = baseurl + url
+        if url.find('.pdf') > 0 and os.path.isfile(url):
+            result.append(url)
+        else:
+            time.sleep(delay)
+            result = result + pullPDFs(
+                url,
+                depth + 1,
+                alreadycrawled,
+                max_depth=max_depth,
+                delay=delay,
+                timeout=timeout,
+                user_agent=user_agent,
+            )
+    return result
+
+#return a list of strings of all the links on the given webpage (<a> elements) whose href contains the given search string
+def getLinksContainingStr(url, s):
+    result = []
+    resp = requests.get(url)
+    html = BeautifulSoup(resp.text, 'html.parser')
+    for link in html.find_all('a'):
+        if link == None:
+            print('non')
+            continue
+        h = link.get('href')
+        if h and s in h:
+            result.append(url + '/' + h)
+    return result
+
+def downloadFromBandcamp(url, path):
+    res = getAnchorWithPattern(url, ['track/','album/'])
+    log(f'found {len(res)} tracks/albums for {str(url)}')
+    if len(res) > 0:
+        for page in res:
+            print()
+            cmd(f'yt-dlp -x --audio-format flac --audio-quality 0 -P {path} {str(page)}')
+
+def downloadFromYoutubeMusic(url, path):
+    res = getAnchorWithPattern(url, ['browse/.*_.+'])
+    log(f'found {len(res)} tracks for {str(url)}')
+    if len(res) > 0:
+        for page in res:
+            cmd(f'ytdla -P {path} {str(page)}')
+
+#write each element of list l to file of path p
+def writeListToFile(l, path, append=False):
+    f = open(path, 'a' if append else 'w')
+    for e in l:
+        f.write(str(e))
+        f.write('\n')
+    f.close()
+
+def getAnchorWithPattern(url, pattern, content=None):
+    result = []
+    patterns = pattern if isinstance(pattern, (list, tuple, set)) else [pattern]
+    if content is None:
+        resp = requests.get(url)
+        html = BeautifulSoup(resp.text, 'html.parser')
+    else:
+        html = BeautifulSoup(content, 'html.parser')
+
+    parts = urlsplit(url)
+    base_url = f'{parts.scheme}://{parts.netloc}/'
+
+    for link in html.find_all('a'):
+        if link == None:
+            print('non')
+            continue
+        h = link.get('href')
+
+        if h and any(re.search(single_pattern, h) for single_pattern in patterns):
+            if h[0:4] != 'http':
+                h = urljoin(base_url, h.lstrip('/'))
+            result.append(h)
+        else:
+            print(h)
+    return result 
+
+def writeListToFile(l, path, append=False):
+    f = open(path, 'a' if append else 'w')
+    for e in l:
+        f.write(str(e))
+        f.write('\n')
+    f.close()
+
+def ytdlaFromFile(path):
+    '''downloads audio from each url in the file at path'''
+    with open(path, 'r') as f:
+        for line in f:
+            url = line.strip()
+            if url != '':
+                cmd('echo ' + url)
+
+#if len(sys.argv) < 2:
+#    sys.exit(0)
+
+#for url in pull4chImgs(sys.argv[1]):
+#    print(url)
+