#!/bin/python

#some useful web-related scripts, such as crawling, searching for, downloading certain data

import os

import requests
from bs4 import BeautifulSoup
import re
import sys
import time
from urllib.parse import urlsplit, urljoin
from util import *

#returns an array of img src urls from a 4chan thread
def pull4chImgs(url):
    result = []
    resp = requests.get(url)
    html = BeautifulSoup(resp.text, 'html.parser')
    for a in html.find_all('a'):
        if 'class' in a.attrs and 'fileThumb' in a.attrs['class']:
            url = a.get('href')
            if url[0:2] == '//':
                url = 'https:' + url
            result.append(url)
    return result

def pullVids(url):
    result = []
    resp = requests.get(url)
    html = BeautifulSoup(resp.text, 'html.parser')
    for a in html.find_all('a'):
        if '.webm' in a.get('href'):
            result.append(a.get('href'))
    return result

def pullImgs(url):
    result = []
    resp = requests.get(url)
    html = BeautifulSoup(resp.text, 'html.parser')
    for img in html.find_all('img'):
        srcURL = img.get('src')
        result.append(srcURL)
    return result


#i used this for downloading a bunch of pdfs from some prepper website
def pullPDFs(url, depth=0, alreadycrawled=None, max_depth=5, delay=5.0, timeout=20.0, user_agent=None):
    """Return a list of discovered PDF URLs by crawling from `url`.

    Notes:
    - Avoids mutable default args.
    - `max_depth` and `delay` make it safer to run from a CLI.
    """
    if alreadycrawled is None:
        alreadycrawled = []

    if depth > max_depth or url == '' or url is None or url in alreadycrawled:
        return []

    baseurl = url[0:url.find('/', 8) + 1]
    result = []
    print(url)

    headers = {}
    if user_agent:
        headers["User-Agent"] = user_agent

    resp = requests.get(url, headers=headers, timeout=timeout)
    html = BeautifulSoup(resp.text, 'html.parser')
    alreadycrawled.append(url)

    for a in html.find_all('a'):
        url = a.get('href')
        if url is None or url == '' or url == '/':
            continue
        if baseurl not in url and 'http' in url[0:4]:  # external
            continue
        print('found ' + url)
        if 'http' not in url:
            url = baseurl + url
        if url.find('.pdf') > 0 and os.path.isfile(url):
            result.append(url)
        else:
            time.sleep(delay)
            result = result + pullPDFs(
                url,
                depth + 1,
                alreadycrawled,
                max_depth=max_depth,
                delay=delay,
                timeout=timeout,
                user_agent=user_agent,
            )
    return result

#return a list of strings of all the links on the given webpage (<a> elements) whose href contains the given search string
def getLinksContainingStr(url, s):
    result = []
    resp = requests.get(url)
    html = BeautifulSoup(resp.text, 'html.parser')
    for link in html.find_all('a'):
        if link == None:
            print('non')
            continue
        h = link.get('href')
        if h and s in h:
            result.append(url + '/' + h)
    return result

def downloadFromBandcamp(url, path):
    res = getAnchorWithPattern(url, ['track/','album/'])
    log(f'found {len(res)} tracks/albums for {str(url)}')
    if len(res) > 0:
        for page in res:
            print()
            cmd(f'yt-dlp -x --audio-format flac --audio-quality 0 -P {path} {str(page)}')

def downloadFromYoutubeMusic(url, path):
    res = getAnchorWithPattern(url, ['browse/.*_.+'])
    log(f'found {len(res)} tracks for {str(url)}')
    if len(res) > 0:
        for page in res:
            cmd(f'ytdla -P {path} {str(page)}')

#write each element of list l to file of path p
def writeListToFile(l, path, append=False):
    f = open(path, 'a' if append else 'w')
    for e in l:
        f.write(str(e))
        f.write('\n')
    f.close()

def getAnchorWithPattern(url, pattern, content=None):
    result = []
    patterns = pattern if isinstance(pattern, (list, tuple, set)) else [pattern]
    if content is None:
        resp = requests.get(url)
        html = BeautifulSoup(resp.text, 'html.parser')
    else:
        html = BeautifulSoup(content, 'html.parser')

    parts = urlsplit(url)
    base_url = f'{parts.scheme}://{parts.netloc}/'

    for link in html.find_all('a'):
        if link == None:
            print('non')
            continue
        h = link.get('href')

        if h and any(re.search(single_pattern, h) for single_pattern in patterns):
            if h[0:4] != 'http':
                h = urljoin(base_url, h.lstrip('/'))
            result.append(h)
        else:
            print(h)
    return result 

def writeListToFile(l, path, append=False):
    f = open(path, 'a' if append else 'w')
    for e in l:
        f.write(str(e))
        f.write('\n')
    f.close()

def ytdlaFromFile(path):
    '''downloads audio from each url in the file at path'''
    with open(path, 'r') as f:
        for line in f:
            url = line.strip()
            if url != '':
                cmd('echo ' + url)

#if len(sys.argv) < 2:
#    sys.exit(0)

#for url in pull4chImgs(sys.argv[1]):
#    print(url)