some updates. yes some stuff from claude code

7 files changed, 403 insertions, 121 deletions

diff --git a/README.md b/README.md
index d4978e6..4362560 100755
--- a/README.md
+++ b/README.md
@@ -1 +1,5 @@
 this repo contains some of my scripts that i use on my laptop. some of them are used in conjunction with systemd services. most of them won't be of any use to anyone else. some of them may need to be modified to work with other systems. 
+
+
+TODO:
+- organize into utilities and random, etc.
diff --git a/betterbackup.sh b/betterbackup.sh
deleted file mode 100755
index 36232e4..0000000
--- a/betterbackup.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-
-#a simple backup script that doesn't require to write to network address
-
-if [[ $# -lt 2 ]]; then
-    echo 'provide source and destination'
-fi
-
-
-LOGF=/var/log/betterbackup.log
-src_path=${1}
-dest_path=${2}
-if [[ ${3} ]]; then
-    rsargs="--exclude-from=${HOME}/.config/rsync-exclude.txt ${3}"
-else
-    rsargs="--exclude-from=${HOME}/.config/rsync-exclude.txt "
-fi
-
-CMD="rsync -av ${rsargs} --progress --update --protect-args -e ssh --log-file=${LOGF} ${src_path} ${dest_path}"
-echo "running '${CMD}' in 3 seconds"
-sleep 1
-echo "2 seconds"
-sleep 1
-echo "1 second"
-sleep 1
-
-d=`date +%Y%m%d_%H%M%S`
-echo "starting backup at ${d}" >> ${LOGF}
-${CMD}
-d=`date +%Y%m%d_%H%M%S`
-echo "backup complete at ${d}"
diff --git a/cpu_limit_monitor.sh b/cpu_limit_monitor.sh
deleted file mode 100755
index 4b0bda2..0000000
--- a/cpu_limit_monitor.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-MAX_CPU_USAGE=30 #for limiting processes matching the query
-MAX_CPU_USAGE2=60 #for reporting (and optionally limiting) any other process
-
-# Specify the log file location
-LOG_FILE="/var/log/cpulimit.log"
-
-query='[v]scode-server'
-if [[ $1 ]]; then
-    query=${1}
-fi
-
-while true; do
-    #get the processes of interest and see if they are using too much cpu
-    PIDS=$(ps aux | grep ${query} | awk '{print $2}')
-    if [ -z "$PIDS" ]; then
-        echo "[$(date)] No ${query} processes found." | tee -a $LOG_FILE
-    else
-        # Apply the CPU limit to each offending process
-        for PID in $PIDS
-        do
-            cpulimit -p $PID -l $MAX_CPU_USAGE &
-            echo "[$(date)] CPU limit of $MAX_CPU_USAGE% applied to process with PID: $PID" | tee -a $LOG_FILE
-        done
-    fi
-
-    #see if there are any other processes using too much cpu
-    #procs=$(ps ax -o pid,%cpu,%mem,args)
-    PIDS=$(ps aux | grep -v PID | awk '{print $2}')
-    for pid in ${PIDS}; do
-        res=$(ps -p $pid -o %cpu,args | grep -v %CPU)
-        cpuu=$(echo ${res} | awk '{print $1}')
-        args=$(echo ${res} | awk '{print $2}')
-        if [[ -z ${cpuu} ]]; then continue; fi
-        if (( $(echo "${cpuu} > ${MAX_CPU_USAGE2}" | bc -l) )); then
-            echo "cpuu of ${pid} is ${cpuu}"
-        fi
-    done
-    # Sleep for a specified amount of time before checking again
-    sleep 5
-done
\ No newline at end of file
diff --git a/disk_health_report.sh b/disk_health_report.sh
new file mode 100755
index 0000000..0841945
--- /dev/null
+++ b/disk_health_report.sh
@@ -0,0 +1,239 @@
+#!/bin/bash
+
+# Disk Health and Performance Report Generator
+# Requires: smartmontools, hdparm, sysstat (for iostat)
+
+set -e
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+BOLD='\033[1m'
+
+# Check if running as root
+if [[ $EUID -ne 0 ]]; then
+   echo -e "${RED}This script must be run as root (use sudo)${NC}"
+   exit 1
+fi
+
+# Check for required commands
+MISSING_DEPS=()
+command -v smartctl >/dev/null 2>&1 || MISSING_DEPS+=("smartmontools")
+command -v hdparm >/dev/null 2>&1 || MISSING_DEPS+=("hdparm")
+command -v iostat >/dev/null 2>&1 || MISSING_DEPS+=("sysstat")
+
+if [ ${#MISSING_DEPS[@]} -ne 0 ]; then
+    echo -e "${YELLOW}Warning: Missing dependencies: ${MISSING_DEPS[*]}${NC}"
+    echo "Install with: sudo apt install ${MISSING_DEPS[*]}"
+    echo "Continuing with limited functionality..."
+    echo
+fi
+
+# Output file
+REPORT_FILE="disk_report_$(date +%Y%m%d_%H%M%S).txt"
+
+# Function to print section header
+print_header() {
+    echo -e "${BOLD}${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+    echo -e "${BOLD}${BLUE}$1${NC}"
+    echo -e "${BOLD}${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+}
+
+# Function to get disk type
+get_disk_type() {
+    local disk=$1
+    if [[ $disk == nvme* ]]; then
+        echo "NVMe"
+    elif cat /sys/block/$disk/queue/rotational 2>/dev/null | grep -q "0"; then
+        echo "SSD"
+    elif cat /sys/block/$disk/queue/rotational 2>/dev/null | grep -q "1"; then
+        echo "HDD"
+    else
+        echo "Unknown"
+    fi
+}
+
+# Function to format bytes
+format_bytes() {
+    local bytes=$1
+    if [ $bytes -ge 1099511627776 ]; then
+        echo "$(awk "BEGIN {printf \"%.2f TB\", $bytes/1099511627776}")"
+    elif [ $bytes -ge 1073741824 ]; then
+        echo "$(awk "BEGIN {printf \"%.2f GB\", $bytes/1073741824}")"
+    else
+        echo "$(awk "BEGIN {printf \"%.2f MB\", $bytes/1048576}")"
+    fi
+}
+
+# Function to check health status
+check_health() {
+    local disk=$1
+    local dev_path="/dev/$disk"
+
+    echo -e "\n${BOLD}SMART Health Status:${NC}"
+
+    if ! command -v smartctl >/dev/null 2>&1; then
+        echo -e "${YELLOW}  smartctl not available - install smartmontools${NC}"
+        return
+    fi
+
+    # Check if SMART is supported
+    if ! smartctl -i $dev_path >/dev/null 2>&1; then
+        echo -e "${YELLOW}  SMART not supported or accessible for this device${NC}"
+        return
+    fi
+
+    # Get health status
+    local health_status=$(smartctl -H $dev_path 2>/dev/null | grep "SMART overall-health" || echo "Unknown")
+    if echo "$health_status" | grep -q "PASSED"; then
+        echo -e "  ${GREEN}✓ PASSED${NC}"
+    elif echo "$health_status" | grep -q "FAILED"; then
+        echo -e "  ${RED}✗ FAILED - IMMEDIATE BACKUP RECOMMENDED${NC}"
+    else
+        echo -e "  ${YELLOW}? Unknown${NC}"
+    fi
+
+    # Get temperature
+    local temp=$(smartctl -A $dev_path 2>/dev/null | grep -i "temperature" | head -1 | awk '{print $10}')
+    if [ ! -z "$temp" ]; then
+        echo "  Temperature: ${temp}°C"
+    fi
+
+    # Get power on hours
+    local hours=$(smartctl -A $dev_path 2>/dev/null | grep "Power_On_Hours" | awk '{print $10}')
+    if [ ! -z "$hours" ]; then
+        local days=$((hours / 24))
+        echo "  Power On Time: $hours hours ($days days)"
+    fi
+
+    # Get reallocated sectors (bad sectors)
+    local reallocated=$(smartctl -A $dev_path 2>/dev/null | grep "Reallocated_Sector" | awk '{print $10}')
+    if [ ! -z "$reallocated" ]; then
+        if [ $reallocated -eq 0 ]; then
+            echo -e "  Reallocated Sectors: ${GREEN}$reallocated${NC}"
+        else
+            echo -e "  Reallocated Sectors: ${YELLOW}$reallocated (warning)${NC}"
+        fi
+    fi
+
+    # Get wear leveling for SSDs
+    local wear=$(smartctl -A $dev_path 2>/dev/null | grep -i "Wear_Leveling_Count\|Media_Wearout_Indicator" | awk '{print $4, $10}')
+    if [ ! -z "$wear" ]; then
+        echo "  SSD Wear: $wear"
+    fi
+}
+
+# Function to check performance
+check_performance() {
+    local disk=$1
+    local dev_path="/dev/$disk"
+
+    echo -e "\n${BOLD}Performance Metrics:${NC}"
+
+    # Current I/O stats with iostat
+    if command -v iostat >/dev/null 2>&1; then
+        echo -e "\n  Current I/O Statistics:"
+        iostat -dx $dev_path 1 2 | tail -1 | awk '{
+            printf "    Read: %.2f MB/s  Write: %.2f MB/s\n", $6/1024, $7/1024
+            printf "    IOPS: %.0f reads/s, %.0f writes/s\n", $4, $5
+            printf "    Utilization: %.1f%%\n", $14
+        }'
+    fi
+
+    # Read speed test with hdparm
+    if command -v hdparm >/dev/null 2>&1; then
+        echo -e "\n  Sequential Read Speed Test (hdparm):"
+        echo "    Testing... (this may take a few seconds)"
+        local hdparm_result=$(hdparm -t $dev_path 2>&1 | grep "Timing")
+        echo "    $hdparm_result"
+
+        # Cached read test
+        local cached_result=$(hdparm -T $dev_path 2>&1 | grep "Timing")
+        echo "    $cached_result"
+    fi
+}
+
+# Function to get disk info
+get_disk_info() {
+    local disk=$1
+    local dev_path="/dev/$disk"
+
+    echo -e "\n${BOLD}Disk Information:${NC}"
+
+    # Get disk model
+    local model=$(smartctl -i $dev_path 2>/dev/null | grep "Device Model\|Model Number" | cut -d: -f2 | xargs || echo "Unknown")
+    echo "  Model: $model"
+
+    # Get serial number
+    local serial=$(smartctl -i $dev_path 2>/dev/null | grep "Serial Number" | cut -d: -f2 | xargs || echo "Unknown")
+    echo "  Serial: $serial"
+
+    # Get firmware version
+    local firmware=$(smartctl -i $dev_path 2>/dev/null | grep "Firmware Version" | cut -d: -f2 | xargs || echo "Unknown")
+    echo "  Firmware: $firmware"
+
+    # Get capacity
+    local size_bytes=$(cat /sys/block/$disk/size 2>/dev/null)
+    if [ ! -z "$size_bytes" ]; then
+        local size_human=$(format_bytes $((size_bytes * 512)))
+        echo "  Capacity: $size_human"
+    fi
+
+    # Get disk type
+    local dtype=$(get_disk_type $disk)
+    echo "  Type: $dtype"
+}
+
+# Main execution
+echo -e "${BOLD}${GREEN}Disk Health and Performance Report${NC}"
+echo "Generated: $(date)"
+echo
+
+# Discover all physical disks (excluding loop, ram, and other virtual devices)
+DISKS=$(lsblk -dn -o NAME,TYPE | grep "disk" | awk '{print $1}' | grep -v "loop\|ram\|sr")
+
+if [ -z "$DISKS" ]; then
+    echo -e "${RED}No physical disks found!${NC}"
+    exit 1
+fi
+
+echo "Found disks: $DISKS"
+echo
+
+# Process each disk
+for disk in $DISKS; do
+    print_header "Disk: /dev/$disk"
+
+    get_disk_info $disk
+    check_health $disk
+    check_performance $disk
+
+    echo
+done
+
+# Summary
+print_header "Summary"
+echo -e "\n${BOLD}Total Disks Analyzed: $(echo $DISKS | wc -w)${NC}\n"
+
+# Save to file
+echo "Report saved to: $REPORT_FILE"
+{
+    echo "Disk Health and Performance Report"
+    echo "Generated: $(date)"
+    echo
+
+    for disk in $DISKS; do
+        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+        echo "Disk: /dev/$disk"
+        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+        get_disk_info $disk
+        check_health $disk
+        check_performance $disk
+        echo
+    done
+} > "$REPORT_FILE"
+
+echo -e "${GREEN}Done!${NC}"
diff --git a/extractcontentfromhtml.py b/extractcontentfromhtml.py
new file mode 100644
index 0000000..5d2d2c7
--- /dev/null
+++ b/extractcontentfromhtml.py
@@ -0,0 +1,77 @@
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+import os
+
+def extract_article(url, output_filename="extracted_article.html"):
+    """
+    Extracts the main article content and images from a webpage and saves it to a simplified HTML file.
+
+    Args:
+        url: The URL of the article or blog post.
+        output_filename: The name of the HTML file to save the extracted content to.
+    """
+    try:
+        response = requests.get(url)
+        response.raise_for_status()  # Raise HTTPError for bad responses (4xx or 5xx)
+        soup = BeautifulSoup(response.content, 'html.parser')
+
+        # --- Identify Main Content Area ---
+        # This is the trickiest part. You'll likely need to adjust this based on the specific website structure.
+        # Common strategies:
+        # 1. Look for specific IDs or classes on container elements.
+        # 2. Look for the element containing the most text.
+        # 3. Manually inspect the HTML source to find the right element.
+
+        # Example: Let's assume the article content is within a <div class="article-content">
+        main_content = soup.find('div', class_='article-content')
+        if not main_content:
+            # Alternative strategy: find the tag containing the most text (crude but sometimes works)
+            main_content = max(soup.find_all('div'), key=lambda tag: len(tag.text)) #This will likely require manual adjustment for each site!
+            if not main_content:
+                print(f"Warning: Could not automatically identify the main content area.  You may need to adjust the selection logic in the script.")
+                main_content = soup.body #Use the whole body as last resort
+
+        # --- Extract Images and Update Source URLs ---
+        images = main_content.find_all('img')
+        for img in images:
+            src = img.get('src')
+            if src:
+                # Make URLs absolute if they are relative
+                absolute_url = urljoin(url, src)
+                img['src'] = absolute_url  # Update the image source
+
+        # --- Create Stripped-Down HTML ---
+        html_content = f"""<!DOCTYPE html>
+        <html>
+        <head>
+            <meta charset="UTF-8">
+            <title>Extracted Article</title>
+            <style>
+                body {{
+                    font-family: sans-serif;
+                    margin: 20px;
+                    line-height: 1.6;
+                }}
+                img {{
+                    max-width: 100%;
+                    height: auto;
+                }}
+            </style>
+        </head>
+        <body>
+            {main_content.prettify()}
+        </body>
+        </html>"""
+
+
+        # --- Save to File ---
+        with open(output_filename, 'w', encoding='utf-8') as f:
+            f.write(html_content)
+
+        print(f"Article extracted and saved to {output_filename}")
+
+    except requests.exceptions.RequestException as e:
+        print(f"Error fetching URL: {e}")
+    except Exception as e:
+        print(f"An error occurred: {e}")
diff --git a/procwatch.sh b/procwatch.sh
index 93e1c40..ae5ad63 100755
--- a/procwatch.sh
+++ b/procwatch.sh
@@ -1,68 +1,100 @@
 #!/bin/bash
 
-# Set the logging configuration
-log_file="process_monitor.log"
+# Configuration
+LOG_FILE="process_monitor.log"
+CPU_THRESHOLD=397        # Percentage
+MEM_THRESHOLD=15360      # Megabytes
+KILL_DELAY=10            # Seconds to wait before killing
+CHECK_INTERVAL=5         # Seconds between process checks
+MAX_KILL_ATTEMPTS=4      # Maximum normal kill attempts before force kill
+MIN_PROCESS_AGE=2        # Minimum process age in seconds to consider
+WARN_CPU_DIVISOR=4       # Show warning when CPU exceeds threshold/divisor
+WARN_MEM_DIVISOR=4       # Show warning when memory exceeds threshold/divisor
 
-# Set the CPU and memory usage thresholds
-cpu_threshold=396  # Percentage
-mem_threshold=15360  # Megabytes
-offender=-1
-kill_attempts=0
+# Critical processes that should never be killed
+CRITICAL_PROCESSES="^(systemd|init|sshd|bash|loginctl|dbus-daemon|systemd-logind)$"
+
+# Track multiple offending processes
+declare -A offender_attempts
 
 while true; do
     # Get the list of running processes
     processes=$(ps -eo pid,pcpu,rss,comm,etimes --sort=-pcpu | grep -v "PID")
-#    echo $processes
 
     # Loop through the processes
     while read -r process_info; do
-	# Extract the process information
-        pid=$(echo "$process_info" | awk '{print $1}')
-        name=$(echo "$process_info" | awk '{print $4}')
-        dur=$(echo "$process_info" | awk '{print $5}')
-        cpu_percent=$(echo "$process_info" | awk '{print $2}')
-        mem_usage=$(echo "$process_info" | awk '{print $3}')
-        mem_usage=$((mem_usage / 1024))  # Convert to Megabytes
-
-        #if (( ${dur} < 2 )); then continue; fi
-        if (( dur < 2 )); then continue; fi
-        
-        # this is just checking if the process exceeds 1/4th of threshold
-        if (( $(echo "$cpu_percent > $((cpu_threshold/4))" | bc -l) )) || (( mem_usage > $((mem_threshold/4)) )); then
+        # Extract all process information in a single awk call
+        read -r pid cpu_percent mem_kb name dur < <(echo "$process_info" | awk '{print $1, $2, $3, $4, $5}')
+
+        # Skip if any required field is empty or non-numeric
+        if [[ -z "$pid" || -z "$dur" || -z "$cpu_percent" || -z "$mem_kb" ]]; then
+            continue
+        fi
+
+        # Skip if dur is not a number (e.g., defunct processes)
+        if ! [[ "$dur" =~ ^[0-9]+$ ]]; then
+            continue
+        fi
+
+        # Convert memory to MB
+        mem_usage=$((mem_kb / 1024))
+
+        # Skip processes younger than minimum age
+        if (( dur < MIN_PROCESS_AGE )); then
+            continue
+        fi
+
+        # Skip critical processes
+        if [[ "$name" =~ $CRITICAL_PROCESSES ]]; then
+            continue
+        fi
+
+        # Warning for processes exceeding 1/4th of threshold
+        if (( $(echo "$cpu_percent > $((CPU_THRESHOLD/WARN_CPU_DIVISOR))" | bc -l) )) || (( mem_usage > $((MEM_THRESHOLD/WARN_MEM_DIVISOR)) )); then
             echo "PROCESS! ${pid}:${name} -- ${cpu_percent}%  ${mem_usage}MB  ${dur}s"
-        fi 
+        fi
 
         # Check if the process exceeds the CPU or memory usage threshold
-        if (( $(echo "$cpu_percent > $cpu_threshold" | bc -l) )) || (( mem_usage > mem_threshold )); then
-            msg_warn="$(date) - Process $name (PID: $pid) has exceeded the threshold: etime: ${dur}, CPU usage: ${cpu_percent}%, Memory usage: ${mem_usage} MB. Killing in 10 seconds" # >> "$log_file"
-            echo ${msg_warn}
+        if (( $(echo "$cpu_percent > $CPU_THRESHOLD" | bc -l) )) || (( mem_usage > MEM_THRESHOLD )); then
+            msg_warn="$(date) - Process $name (PID: $pid) has exceeded the threshold: etime: ${dur}, CPU usage: ${cpu_percent}%, Memory usage: ${mem_usage} MB. Killing in ${KILL_DELAY} seconds"
+
+            echo "$msg_warn"
+            echo "$msg_warn" >> "$LOG_FILE"
             notify-send -t 8000 "$msg_warn"
-            sleep 10
-            if [[ $pid != $offender ]]; then
-                offender=$pid
-                #TODO deal with multiple offending processes later
+
+            sleep "$KILL_DELAY"
+
+            # Check if process still exists after sleep
+            if ! kill -0 "$pid" 2>/dev/null; then
+                echo "Process $pid ($name) has already exited"
+                unset "offender_attempts[$pid]"
+                continue
+            fi
+
+            # Track kill attempts for this specific PID
+            if [[ -n "${offender_attempts[$pid]}" ]]; then
+                offender_attempts[$pid]=$((offender_attempts[$pid] + 1))
+            else
+                offender_attempts[$pid]=1
             fi
 
             # ACTUAL PROCESS KILL IS HERE IF YOU NEED TO MODIFY
-            if [[ ${kill_attempts} -gt 4 ]]; then       
-                echo "force kill ${pid}"
-                sleep 1
-                kill_attempts=$((kill_attempts+1))
-                kill -9 ${pid}
+            if [[ ${offender_attempts[$pid]} -gt $MAX_KILL_ATTEMPTS ]]; then
+                echo "Force kill $pid ($name) - attempt ${offender_attempts[$pid]}"
+                kill -9 "$pid" 2>/dev/null
             else
-                echo "kill ${pid}. attempt ${kill_attempts}"
-                sleep 1
-                kill_attempts=$((kill_attempts+1))
-                kill ${pid}
+                echo "Kill $pid ($name) - attempt ${offender_attempts[$pid]}"
+                kill "$pid" 2>/dev/null
             fi
-            sleep 1
-            if [[ $(! kill -0 ${pid} 2>/dev/null) ]]; then
-                offender=-1
-                kill_attempts=0
+
+            # Clean up if process is gone
+            if ! kill -0 "$pid" 2>/dev/null; then
+                echo "Successfully killed process $pid ($name)"
+                unset "offender_attempts[$pid]"
             fi
         fi
     done <<< "$processes"
 
-    # Wait for 5 seconds before checking again
-    sleep 5
+    # Wait before checking again
+    sleep "$CHECK_INTERVAL"
 done
diff --git a/wow3.py b/wow3.py
index e4cdd82..a1d3768 100755
--- a/wow3.py
+++ b/wow3.py
@@ -100,15 +100,18 @@ def getSampleFiles(paths, exclude_patterns = []):
 def getRandomFileTextContent(samplefiles):
     """pick random file until we get an acceptable one
         @return tuple(filename, textcontent)"""
-    fi = random.randint(0, len(samplefiles))
+    if len(samplefiles) == 0:
+        return (None, None)
+
+    fi = random.randint(0, len(samplefiles) - 1)
     if v: print('candidate file: {}'.format(samplefiles[fi]))
     t = attemptReadSampleFile(samplefiles[fi]) #check if valid file
     while t == None: #this file was invalid, try again
         del samplefiles[fi]
-        fi = random.randint(0, len(samplefiles))
-        t = attemptReadSampleFile(samplefiles[fi])
         if len(samplefiles) == 0: #no files are valid
             return (None, None)
+        fi = random.randint(0, len(samplefiles) - 1)
+        t = attemptReadSampleFile(samplefiles[fi])
     
     mt = time.ctime(os.path.getmtime(samplefiles[fi]))
     if v: print('{} ;\n       last modified {} :\n '.format(samplefiles[fi], mt))
@@ -136,7 +139,7 @@ def parseArgs():
     
     parser.add_argument('-p', '--paths', type=str, required=False, help='a path to scan', action='append', default=[])
     parser.add_argument('-o', '--output', type=str, required=False, help='output file')
-    parser.add_argument('-x', '--exclude', type=str, required=False, help='exclude files with pattern', action='append')
+    parser.add_argument('-x', '--exclude', type=str, required=False, help='exclude files with pattern', action='append', default=[])
     args = parser.parse_args()
 
     if (args.verbose): v = True