From 6d858fb18e7f40da7754a2add86c68a6a8a0d794 Mon Sep 17 00:00:00 2001 From: Brian Flowers Date: Wed, 1 Apr 2020 11:59:13 -0400 Subject: [PATCH 1/1] Rebuilding server --- TODO | 16 ++ bin/firewall-lists | 612 +++++++++++++++++++++++++++++++++++++++++++++++++++++ bin/th.sh | 15 ++ 3 files changed, 643 insertions(+) create mode 100644 TODO create mode 100755 bin/firewall-lists create mode 100755 bin/th.sh diff --git a/TODO b/TODO new file mode 100644 index 0000000..353dd91 --- /dev/null +++ b/TODO @@ -0,0 +1,16 @@ +TODO: ++ 1) Confirm if lookup logic is sound (DONE) ++ 2) Multithread lookup logic ++ Cleanup on kill ++ variable max thread count +? 3) Skip comments / validate on nameserver / domain import ++ 4) writeLists + 5) cleanNameservers ++ 6) Refactor input args + 7) Check if lookup thread insert gives error and retry + 8) Allow parallel inserts on the DB? + 9) Move printLists + 10) writeLists add ipv4/ipv6 support ++ 11) troubleshoot -- enter an IP and lookup the domain/list info + +CONCEPT: Split lookups into n files, check each in a thread, write output to a file and dump that to sqlite sequentially/in bulk (DONE?) diff --git a/bin/firewall-lists b/bin/firewall-lists new file mode 100755 index 0000000..1e1c731 --- /dev/null +++ b/bin/firewall-lists @@ -0,0 +1,612 @@ +#!/bin/bash +############################################################################### +# +# firewall-lists +# This script takes a list of domain names, queries each against a list of +# nameservers, and generates a list of corresponding IP addresses suitable +# for use with various firewall and network software systems. +# +# Return Codes: +# 1: User terminated +# 2: Invalid argument +# 3: Permission issue +# 4: Sanity failure +# 255: Unknown Signal Received +# +############################################################################### + +# CONFIGURATION +############################################################################### + +#config_path -- the path of firewall-lists configuration (usually under etc) +#onfig_path="/home/USERNAME/etc/firewall-lists" +config_path="/home/urza9814/etc/firewall-lists-3" + +# lookup_threats -- maximum number of live nslookup connections +lookup_threads=50 + +# lookup_delay -- delay between launching new threads (for rate limiting) +lookup_delay=5 + +# Default output logfile -- leave null for standard output +# This should usually be configured with the --logfile option +logfile="" + +# Debug mode -- non-zero values enable debug output +# To enable for a single execution you can use the -d or --debug flags +debug=0 + +# Trap UNCAUGHT ERRORS +############################################################################### + +trap failure SIGFPE SIGHUP SIGABRT SIGALRM SIGQUIT +trap term SIGTERM SIGINT + +term() +{ + echo + echo + error "TERMINATED." + info "Cleaning up child processes..." + ls "$config_path/pids" | while read pid; do + kill "$pid" + if [ `ps -p $pid | wc -l` -le 1 ]; then + rm "$config_path/pids/$pid" + fi + done + info "Exiting." + exit 1 +} + +failure() +{ + echo + echo + error "Something has gone SERIOUSLY wrong! (Received signal $?)" + exit 255 +} + +# UTILITY FUNCTIONS +############################################################################### +# Output functions +error() +{ + printf "\e[41m" 1>&2 + printf "`date` | ERROR: $1" 1>&2 + printf "\e[0m\n" 1>&2 +} + +info() +{ + printf "\e[33m" + printf "`date` | INFO: $1" + printf "\e[0m\n" +} + +warn() +{ + printf "\e[30;43m" 1>&2 + printf "`date` | WARNING: $1" 1>&2 + printf "\e[0m\n" 1>&2 +} + +debug() +{ + if [ $debug -ne 0 ]; then + return + fi + + length="`echo "${1}" | wc -l`" + if [ $length -lt 2 ]; then + printf "DEBUG: ${1}\n" + else + echo "${1}" | sed 's/^/\t/g' + fi +} + +progress() +{ + if [ ! -z "${logfile}" ]; then + return + fi + + status="${1}" + total="${2}" + details="${3}" + + percent=`expr $status \* 100 / $total` + if [ ! -z "${details}" ]; then + printf " $percent%% completed | $details\r" + else + printf " $percent%% completed ($status / $total)\r" + fi +} + + + +lookupDomain() +{ + touch "${config_path}/pids/$BASHPID" + domain="${1}" + nameserver="${2}" + outfile="${3}" + + retval=1 + counter=0 + while [ $retval -ne 0 ]; do + nslookup -timeout=5 -retry=2 "${domain}" "${nameserver}" | sed '2d' | grep "^Address" | \ + sed "s|Address: |INSERT INTO lookups(domain, ip) VALUES((SELECT id FROM domains WHERE url='${domain}'), '|g" |\ + sed "s|$|');|g" >> "${outfile}" + retval=$? + counter=`expr $counter + 1` + if [ $counter -gt 10 ]; then + warn "Failed to insert after ten attempts; giving up!" + break; + fi + done + + rm "${config_path}/pids/$BASHPID" +} + +runCmd() +{ + case "${1}" in + importNameservers) + importNameservers "${2}" + ;; + importList) + importList "`echo "${2}" | cut -d'=' -f1`" "`echo "${2}" | cut -d'=' -f2-`" + ;; + lookupList) + lookupLists "${2}" + ;; + lookupNext) + lookupLists "`echo "${2}" | cut -d'=' -f1`" "`echo "${2}" | cut -d'=' -f2-`" + ;; + writeList) + writeList "${2}" + ;; + writeListv4) + writeList "${2}" "v4" + ;; + addDomain) + addDomain "`echo "${2}" | cut -d'=' -f1`" "`echo "${2}" | cut -d'=' -f2-`" + ;; + removeDomain) + removeDomain "`echo "${2}" | cut -d'=' -f1`" "`echo "${2}" | cut -d'=' -f2-`" + ;; + purge) + purge "${2}" + ;; + why) + why "${2}" + ;; + interactive) + interactive + ;; + printLists) + echo "List name | Domains Included" + sqlite3 "$config_path/firewall-lists.sqlite" </dev/null 2>&1 + if [ $? -eq 1 ]; then + sqlite3 "$config_path/firewall-lists.sqlite"</dev/null 2>&1 + if [ $? -eq 1 ]; then + sqlite3 "$config_path/firewall-lists.sqlite" </dev/null 2>&1 + if [ $? -eq 1 ]; then + sqlite3 "$config_path/firewall-lists.sqlite" </dev/null 2>&1 + if [ $? -eq 1 ]; then + sqlite3 "$config_path/firewall-lists.sqlite" </dev/null 2>&1 + if [ $? -eq 1 ]; then + sqlite3 "$config_path/firewall-lists.sqlite" < "$config_path/.list-$name" + listId=`sqlite3 "$config_path/firewall-lists.sqlite" "SELECT id FROM lists WHERE name='${name}'"` + grep -v "^[ ]*#" "${file}" | while read domain; do + cat <> "$config_path/.list-$name" +REPLACE INTO domains(url) VALUES('${domain}'); +REPLACE INTO domainMap(list, domain) VALUES( + (SELECT id FROM lists WHERE name='${name}'), + (SELECT id FROM domains WHERE url='${domain}')); +EOF + done + + cat "$config_path/.list-$name" | sqlite3 "$config_path/firewall-lists.sqlite" + +# cat "${file}" | +# sed "s/\(.*\)/REPLACE INTO domains(url) VALUES('\1');/g" | \ +# sqlite3 "$config_path/firewall-lists.sqlite" +} + +lookupLists() +{ + list="${1}" + if [ ! -z "`echo "${list}" | grep '*'`" ]; then + list2="`echo "${list}" | sed 's/*/%/g'`" + list="`sqlite3 "$config_path/firewall-lists.sqlite" < "$config_path/.lookup-$$" +SELECT url FROM +(SELECT DISTINCT url FROM +(SELECT DISTINCT url, lookupDate FROM domains, domainMap, lists, lookups +WHERE domainMap.list = lists.id +AND domainMap.domain = domains.id +AND lists.name IN +('${list}') +AND lookups.domain = domainMap.domain +UNION ALL +SELECT DISTINCT url, null FROM domains, domainMap, lists +WHERE domainMap.list = lists.id +AND domainMap.domain = domains.id +AND lists.name IN +('${list}') +AND domainMap.domain NOT IN (SELECT domain FROM lookups)) +ORDER BY lookupDate ASC) +${limit}; +EOF + + info "Refreshing `cat "$config_path/.lookup-$$" | wc -l` records..." + + sqlite3 "$config_path/firewall-lists.sqlite" < "$config_path/.lookup-ns" +SELECT hostname FROM nameservers; +EOF + + echo > "$config_path/.lookups.tmp" + while read nameserver; do + startCnt="`cat "$config_path/.lookups.tmp" | wc -l`" + while read domain; do + while [ `ls -ltr "${config_path}/pids/" 2>/dev/null | wc -l` -gt "$lookup_threads" ]; do + sleep 1 + done + lookupDomain "${domain}" "${nameserver}" "$config_path/.lookups.tmp" & +# nslookup -timeout=5 -retry=2 "${domain}" "${nameserver}" | sed '2d' | grep "^Address" | \ +# sed "s|Address: |INSERT INTO lookups(domain, ip) VALUES((SELECT id FROM domains WHERE url='${domain}'), '|g" |\ +# sed "s|$|');|g" | sqlite3 "$config_path/firewall-lists.sqlite" + done < "$config_path/.lookup-$$" + + endCnt="`cat "$config_path/.lookups.tmp" | wc -l`" + if [ "$endCnt" -gt "$startCnt" ]; then + sqlite3 "$config_path/firewall-lists.sqlite" "UPDATE nameservers SET lastAccess = date('now') WHERE hostname='${nameserver}';" + fi + + done < "$config_path/.lookup-ns" + + cat "$config_path/.lookups.tmp" | sort -u | sqlite3 "$config_path/firewall-lists.sqlite" + + info "Complete" +} + +writeList() +{ + list="${1}" + type="${2}" + + if [ "${type}" == "v4" ]; then + condition="AND lookups.ip NOT LIKE '%:%'" + fi + + if [ ! -z "`echo "${list}" | grep '*'`" ]; then + list2="`echo "${list}" | sed 's/*/%/g'`" + list="`sqlite3 "$config_path/firewall-lists.sqlite" <