web scraping utils

1 files changed, 98 insertions, 0 deletions

diff --git a/webutils.py b/webutils.py
new file mode 100755
index 0000000..bd7fb30
--- /dev/null
+++ b/webutils.py
@@ -0,0 +1,98 @@
+#!/bin/python
+
+#some useful web-related scripts, such as crawling, searching for, downloading certain data
+
+import requests
+from bs4 import BeautifulSoup
+import re
+import sys
+import time
+
+#returns an array of img src urls from a 4chan thread
+def pull4chImgs(url):
+    result = []
+    resp = requests.get(url)
+    html = BeautifulSoup(resp.text, 'html.parser')
+    for a in html.find_all('a'):
+        if 'class' in a.attrs and 'fileThumb' in a.attrs['class']:
+            url = a.get('href')
+            if url[0:2] == '//':
+                url = 'https:' + url
+            result.append(url)
+    return result
+
+def pullVids(url):
+    result = []
+    resp = requests.get(url)
+    html = BeautifulSoup(resp.text, 'html.parser')
+    for a in html.find_all('a'):
+        if '.webm' in a.get('href'):
+            result.append(a.get('href'))
+    return result
+
+def pullImgs(url):
+    result = []
+    resp = requests.get(url)
+    html = BeautifulSoup(resp.text, 'html.parser')
+    for img in html.find_all('img'):
+        srcURL = img.get('src')
+        result.append(srcURL)
+    return result
+
+
+#i used this for downloading a bunch of pdfs from some prepper website
+def pullPDFs(url, depth=0, alreadycrawled=[]):
+    if depth > 5 or url == '' or url is None or url in alreadycrawled: 
+        return [] 
+    baseurl=url[0:url.find('/', 8)+1]
+    result = []
+    print(url)
+    resp = requests.get(url)
+    html = BeautifulSoup(resp.text, 'html.parser')
+    alreadycrawled.append(url)
+    for a in html.find_all('a'):
+        url = a.get('href')
+        if url is None or url == '' or url == '/':
+            continue
+        if baseurl not in url and 'http' in url[0:4]: #this means that the url is pointing to external site
+            continue
+        print('found ' + url)
+        if 'http' not in url:
+            url = baseurl+url
+        if url.find('.pdf')>0 and os.path.isfile(url):
+            result.append(url)
+        else:
+            time.sleep(5)
+            result = result + pullPDFs(url, depth+1, alreadycrawled)
+    return result
+
+#return a list of strings of all the links on the given webpage (<a> elements) whose href contains the given search string
+def getLinksContainingStr(url, s):
+    result = []
+    resp = requests.get(url)
+    html = BeautifulSoup(resp.text, 'html.parser')
+    for link in html.find_all('a'):
+        if link == None:
+            print('non')
+            continue
+        h = link.get('href')
+        if h and s in h:
+            result.append(url + '/' + h)
+    return result
+
+#write each element of list l to file of path p
+def writeListToFile(l, path):
+    f = open(path, 'w')
+    for e in l:
+        f.write(str(e))
+        f.write('\n')
+    f.close()
+    
+
+
+#if len(sys.argv) < 2:
+#    sys.exit(0)
+
+#for url in pull4chImgs(sys.argv[1]):
+#    print(url)
+