--- /dev/null
+#!/bin/bash
+###############################################################################
+#
+# firewall-lists
+# This script takes a list of domain names, queries each against a list of
+# nameservers, and generates a list of corresponding IP addresses suitable
+# for use with various firewall and network software systems.
+#
+# Return Codes:
+# 1: User terminated
+# 2: Invalid argument
+# 3: Permission issue
+# 4: Sanity failure
+# 255: Unknown Signal Received
+#
+###############################################################################
+
+# CONFIGURATION
+###############################################################################
+
+#config_path -- the path of firewall-lists configuration (usually under etc)
+#onfig_path="/home/USERNAME/etc/firewall-lists"
+config_path="/home/urza9814/etc/firewall-lists-3"
+
+# lookup_threats -- maximum number of live nslookup connections
+lookup_threads=50
+
+# lookup_delay -- delay between launching new threads (for rate limiting)
+lookup_delay=5
+
+# Default output logfile -- leave null for standard output
+# This should usually be configured with the --logfile option
+logfile=""
+
+# Debug mode -- non-zero values enable debug output
+# To enable for a single execution you can use the -d or --debug flags
+debug=0
+
+# Trap UNCAUGHT ERRORS
+###############################################################################
+
+trap failure SIGFPE SIGHUP SIGABRT SIGALRM SIGQUIT
+trap term SIGTERM SIGINT
+
+term()
+{
+ echo
+ echo
+ error "TERMINATED."
+ info "Cleaning up child processes..."
+ ls "$config_path/pids" | while read pid; do
+ kill "$pid"
+ if [ `ps -p $pid | wc -l` -le 1 ]; then
+ rm "$config_path/pids/$pid"
+ fi
+ done
+ info "Exiting."
+ exit 1
+}
+
+failure()
+{
+ echo
+ echo
+ error "Something has gone SERIOUSLY wrong! (Received signal $?)"
+ exit 255
+}
+
+# UTILITY FUNCTIONS
+###############################################################################
+# Output functions
+error()
+{
+ printf "\e[41m" 1>&2
+ printf "`date` | ERROR: $1" 1>&2
+ printf "\e[0m\n" 1>&2
+}
+
+info()
+{
+ printf "\e[33m"
+ printf "`date` | INFO: $1"
+ printf "\e[0m\n"
+}
+
+warn()
+{
+ printf "\e[30;43m" 1>&2
+ printf "`date` | WARNING: $1" 1>&2
+ printf "\e[0m\n" 1>&2
+}
+
+debug()
+{
+ if [ $debug -ne 0 ]; then
+ return
+ fi
+
+ length="`echo "${1}" | wc -l`"
+ if [ $length -lt 2 ]; then
+ printf "DEBUG: ${1}\n"
+ else
+ echo "${1}" | sed 's/^/\t/g'
+ fi
+}
+
+progress()
+{
+ if [ ! -z "${logfile}" ]; then
+ return
+ fi
+
+ status="${1}"
+ total="${2}"
+ details="${3}"
+
+ percent=`expr $status \* 100 / $total`
+ if [ ! -z "${details}" ]; then
+ printf " $percent%% completed | $details\r"
+ else
+ printf " $percent%% completed ($status / $total)\r"
+ fi
+}
+
+
+
+lookupDomain()
+{
+ touch "${config_path}/pids/$BASHPID"
+ domain="${1}"
+ nameserver="${2}"
+ outfile="${3}"
+
+ retval=1
+ counter=0
+ while [ $retval -ne 0 ]; do
+ nslookup -timeout=5 -retry=2 "${domain}" "${nameserver}" | sed '2d' | grep "^Address" | \
+ sed "s|Address: |INSERT INTO lookups(domain, ip) VALUES((SELECT id FROM domains WHERE url='${domain}'), '|g" |\
+ sed "s|$|');|g" >> "${outfile}"
+ retval=$?
+ counter=`expr $counter + 1`
+ if [ $counter -gt 10 ]; then
+ warn "Failed to insert after ten attempts; giving up!"
+ break;
+ fi
+ done
+
+ rm "${config_path}/pids/$BASHPID"
+}
+
+runCmd()
+{
+ case "${1}" in
+ importNameservers)
+ importNameservers "${2}"
+ ;;
+ importList)
+ importList "`echo "${2}" | cut -d'=' -f1`" "`echo "${2}" | cut -d'=' -f2-`"
+ ;;
+ lookupList)
+ lookupLists "${2}"
+ ;;
+ lookupNext)
+ lookupLists "`echo "${2}" | cut -d'=' -f1`" "`echo "${2}" | cut -d'=' -f2-`"
+ ;;
+ writeList)
+ writeList "${2}"
+ ;;
+ writeListv4)
+ writeList "${2}" "v4"
+ ;;
+ addDomain)
+ addDomain "`echo "${2}" | cut -d'=' -f1`" "`echo "${2}" | cut -d'=' -f2-`"
+ ;;
+ removeDomain)
+ removeDomain "`echo "${2}" | cut -d'=' -f1`" "`echo "${2}" | cut -d'=' -f2-`"
+ ;;
+ purge)
+ purge "${2}"
+ ;;
+ why)
+ why "${2}"
+ ;;
+ interactive)
+ interactive
+ ;;
+ printLists)
+ echo "List name | Domains Included"
+ sqlite3 "$config_path/firewall-lists.sqlite" <<EOF | tr '|' '\t'
+SELECT name, count(domains.id) FROM lists, domains, domainMap
+WHERE domainMap.domain = domains.id
+AND domainMap.list = lists.id
+GROUP BY lists.name
+EOF
+ ;;
+ esac
+}
+
+# MAIN SCRIPT FUNCTIONS
+###############################################################################
+# Configure the database
+setup()
+{
+ info "Creating tables (if necessary) at $config_path/firewall-lists.sqlite..."
+ sqlite3 "$config_path/firewall-lists.sqlite" "SELECT 1 FROM nameservers;" >/dev/null 2>&1
+ if [ $? -eq 1 ]; then
+ sqlite3 "$config_path/firewall-lists.sqlite"<<EOF
+CREATE TABLE nameservers(id INTEGER,
+ hostname VARCHAR(256) UNIQUE,
+ lastAccess DATETIME,
+ PRIMARY KEY(id));
+EOF
+ fi
+
+ sqlite3 "$config_path/firewall-lists.sqlite" "SELECT 1 FROM lists;" >/dev/null 2>&1
+ if [ $? -eq 1 ]; then
+ sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+CREATE TABLE lists(id INTEGER,
+ name VARCHAR(256),
+ expiry INTEGER default 2592000,
+ PRIMARY KEY(id),
+ UNIQUE(name));
+EOF
+ fi
+
+ sqlite3 "$config_path/firewall-lists.sqlite" "SELECT 1 FROM domains;" >/dev/null 2>&1
+ if [ $? -eq 1 ]; then
+ sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+CREATE TABLE domains(id INTEGER,
+ url VARCHAR(256),
+ PRIMARY KEY(id),
+ UNIQUE(url));
+EOF
+ fi
+
+ sqlite3 "$config_path/firewall-lists.sqlite" "SELECT 1 FROM domainMap;" >/dev/null 2>&1
+ if [ $? -eq 1 ]; then
+ sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+CREATE TABLE domainMap(id INTEGER,
+ list INTEGER,
+ domain INTEGER,
+ PRIMARY KEY(id),
+ UNIQUE(list,domain));
+EOF
+ fi
+
+ sqlite3 "$config_path/firewall-lists.sqlite" "SELECT 1 FROM lookups;" >/dev/null 2>&1
+ if [ $? -eq 1 ]; then
+ sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+CREATE TABLE lookups(id INTEGER,
+ domain INTEGER,
+ ip VARCHAR(128),
+ lookupDate DATETIME default CURRENT_TIMESTAMP,
+ PRIMARY KEY(id));
+EOF
+ fi
+}
+
+# Import nameservers from a file to the database
+importNameservers()
+{
+ file="${1}"
+ grep -v "^[ ]*#" "${file}" |
+ sed "s/\(.*\)/INSERT INTO nameservers(hostname) VALUES('\1');/g" |
+ sqlite3 "$config_path/firewall-lists.sqlite"
+}
+
+importList()
+{
+ name="${1}"
+ file="${2}"
+
+ sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+REPLACE INTO lists(name) VALUES('${name}');
+EOF
+
+ echo > "$config_path/.list-$name"
+ listId=`sqlite3 "$config_path/firewall-lists.sqlite" "SELECT id FROM lists WHERE name='${name}'"`
+ grep -v "^[ ]*#" "${file}" | while read domain; do
+ cat <<EOF >> "$config_path/.list-$name"
+REPLACE INTO domains(url) VALUES('${domain}');
+REPLACE INTO domainMap(list, domain) VALUES(
+ (SELECT id FROM lists WHERE name='${name}'),
+ (SELECT id FROM domains WHERE url='${domain}'));
+EOF
+ done
+
+ cat "$config_path/.list-$name" | sqlite3 "$config_path/firewall-lists.sqlite"
+
+# cat "${file}" |
+# sed "s/\(.*\)/REPLACE INTO domains(url) VALUES('\1');/g" | \
+# sqlite3 "$config_path/firewall-lists.sqlite"
+}
+
+lookupLists()
+{
+ list="${1}"
+ if [ ! -z "`echo "${list}" | grep '*'`" ]; then
+ list2="`echo "${list}" | sed 's/*/%/g'`"
+ list="`sqlite3 "$config_path/firewall-lists.sqlite" <<EOF | tr '\n' ',' | sed 's/,$//g' | sed "s/,/','/g"
+SELECT name FROM lists WHERE name LIKE '${list2}';
+EOF`"
+ fi
+
+ if [ "${2}" != "" ]; then
+ limit="LIMIT ${2}"
+ fi
+
+ sqlite3 "$config_path/firewall-lists.sqlite" <<EOF > "$config_path/.lookup-$$"
+SELECT url FROM
+(SELECT DISTINCT url FROM
+(SELECT DISTINCT url, lookupDate FROM domains, domainMap, lists, lookups
+WHERE domainMap.list = lists.id
+AND domainMap.domain = domains.id
+AND lists.name IN
+('${list}')
+AND lookups.domain = domainMap.domain
+UNION ALL
+SELECT DISTINCT url, null FROM domains, domainMap, lists
+WHERE domainMap.list = lists.id
+AND domainMap.domain = domains.id
+AND lists.name IN
+('${list}')
+AND domainMap.domain NOT IN (SELECT domain FROM lookups))
+ORDER BY lookupDate ASC)
+${limit};
+EOF
+
+ info "Refreshing `cat "$config_path/.lookup-$$" | wc -l` records..."
+
+ sqlite3 "$config_path/firewall-lists.sqlite" <<EOF > "$config_path/.lookup-ns"
+SELECT hostname FROM nameservers;
+EOF
+
+ echo > "$config_path/.lookups.tmp"
+ while read nameserver; do
+ startCnt="`cat "$config_path/.lookups.tmp" | wc -l`"
+ while read domain; do
+ while [ `ls -ltr "${config_path}/pids/" 2>/dev/null | wc -l` -gt "$lookup_threads" ]; do
+ sleep 1
+ done
+ lookupDomain "${domain}" "${nameserver}" "$config_path/.lookups.tmp" &
+# nslookup -timeout=5 -retry=2 "${domain}" "${nameserver}" | sed '2d' | grep "^Address" | \
+# sed "s|Address: |INSERT INTO lookups(domain, ip) VALUES((SELECT id FROM domains WHERE url='${domain}'), '|g" |\
+# sed "s|$|');|g" | sqlite3 "$config_path/firewall-lists.sqlite"
+ done < "$config_path/.lookup-$$"
+
+ endCnt="`cat "$config_path/.lookups.tmp" | wc -l`"
+ if [ "$endCnt" -gt "$startCnt" ]; then
+ sqlite3 "$config_path/firewall-lists.sqlite" "UPDATE nameservers SET lastAccess = date('now') WHERE hostname='${nameserver}';"
+ fi
+
+ done < "$config_path/.lookup-ns"
+
+ cat "$config_path/.lookups.tmp" | sort -u | sqlite3 "$config_path/firewall-lists.sqlite"
+
+ info "Complete"
+}
+
+writeList()
+{
+ list="${1}"
+ type="${2}"
+
+ if [ "${type}" == "v4" ]; then
+ condition="AND lookups.ip NOT LIKE '%:%'"
+ fi
+
+ if [ ! -z "`echo "${list}" | grep '*'`" ]; then
+ list2="`echo "${list}" | sed 's/*/%/g'`"
+ list="`sqlite3 "$config_path/firewall-lists.sqlite" <<EOF | tr '\n' ',' | sed 's/,$//g' | sed "s/,/','/g"
+SELECT name FROM lists WHERE name LIKE '${list2}';
+EOF`"
+ fi
+
+ sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+SELECT domains.url || ':' || lookups.ip FROM lookups, domains, domainMap, lists
+WHERE lookups.domain = domains.id
+AND domainMap.domain = domains.id
+AND domainMap.list = lists.id
+AND lists.name IN ('${list}')
+${condition};
+EOF
+
+}
+
+addDomain()
+{
+ list="${1}"
+ domain="${2}"
+
+ listId="`sqlite3 "$config_path/firewall-lists.sqlite" "SELECT id FROM lists WHERE name = '${list}'"`"
+ if [ -z "${listId}" ]; then
+ error "List not found: ${list}"
+ exit 2
+ fi
+
+ domainId="`sqlite3 "$config_path/firewall-lists.sqlite" "SELECT id FROM domains WHERE url = '${domain}'"`"
+ if [ -z "${domainId}" ]; then
+ sqlite3 "$config_path/firewall-lists.sqlite" "INSERT INTO domains(url) VALUES('${domain}')"
+ domainId="`sqlite3 "$config_path/firewall-lists.sqlite" "SELECT id FROM domains WHERE url = '${domain}'"`"
+ if [ -z "${domainId}" ]; then
+ error "Could not insert domain: ${domain}"
+ exit 4
+ fi
+ fi
+
+ sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+INSERT INTO domainMap(list, domain) VALUES (${listId}, ${domainId});
+EOF
+}
+
+removeDomain()
+{
+ list="${1}"
+ domain="${2}"
+
+ listId="`sqlite3 "$config_path/firewall-lists.sqlite" "SELECT id FROM lists WHERE name = '${list}'"`"
+ if [ -z "${listId}" ]; then
+ error "List not found: ${list}"
+ exit 2
+ fi
+
+ domainId="`sqlite3 "$config_path/firewall-lists.sqlite" "SELECT id FROM domains WHERE url = '${domain}'"`"
+ if [ -z "${domainId}" ]; then
+ error "Could not find domain: ${domain}"
+ exit 4
+ fi
+
+ sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+DELETE FROM domainMap WHERE list=${list} AND domain=${domainId};
+EOF
+}
+
+purge()
+{
+ days="${1}"
+
+ echo "Purging lookups older than ${days} days"
+ sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+DELETE FROM lookups WHERE lookupDate < DATETIME('now', '-${days} days');
+EOF
+
+ sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+DELETE FROM
+EOF
+}
+
+why()
+{
+ domain="${1}"
+ ips="`echo $domain | grep -v [a-zA-Z]`"
+ if [ -z "${ips}" ]; then
+ results="`sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+SELECT domains.url, lists.name, lookups.ip, lookups.lookupDate
+FROM domains,domainMap,lists,lookups
+WHERE domains.id = domainMap.domain
+AND lists.id = domainMap.list
+AND lookups.domain = domains.id
+AND domains.url = '${domain}'
+EOF`"
+ ips="`echo ${results} | cut -d'|' -f3 | sed "s/^/'/g" | sed "s/\$/'/g" | tr '\n' ',' | sed 's/,$//g'`"
+ else
+ ips="'${ips}'"
+ fi
+ results="${results}
+`sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+SELECT domains.url, lists.name, lookups.ip, lookups.lookupDate
+FROM domains,domainMap,lists,lookups
+WHERE domains.id = domainMap.domain
+AND lists.id = domainMap.list
+AND lookups.domain = domains.id
+AND lookups.ip IN (${ips});
+EOF`"
+
+ echo "${results}" | sort -u
+}
+
+interactive()
+{
+ echo "STUB."
+}
+
+help()
+{
+ echo "Usage: $0 [operation] [argument(s)] where [operation] is one of:"
+ echo " -a | --addDomain | add"
+ echo " Add a domain to a list"
+ echo " Expects one or more arguments in the format [list name]=[domain]"
+ echo
+ echo " -c | --lookupNext | next"
+ echo " Lookups the next [n] domains, starting with those which were last scanned least recently."
+ echo " Expects one argmment in the format [list regex]=[number of domains to update]"
+ echo
+# echo " -d | --cleanNameservers | cleanNameservers"
+# echo " Accepts a signle numeric argument, [n], and deletes all nameservers with more than [n]% errors"
+# echo
+ echo " -i | --importLists | importLists"
+ echo " Import lists from a file"
+ echo " Expects one argmunt, in the format [list name]=[filename]"
+ echo " The file should be a list of domains to add to the list, with one per line"
+ echo
+ echo " -l | --lookupLists | lookupLists"
+ echo " Lookup the selected lists and add the IP addresses to the database"
+ echo " Expects arguments of the list name to lookup, with '*' as a wildcard"
+ echo
+ echo " -n | --importNameservers | importNameservers"
+ echo " Import a list of nameservers to use to lookup IP addresses from domain names"
+ echo " Expects a filename as the argument"
+ echo
+ echo " -p | --printLists | print"
+ echo " Print the domains included in the selected lists"
+ echo " Expects a list name as an argument, with '*' as a wildcard"
+ echo
+ echo " -r | --removeDomain | removeDomain"
+ echo " Remove a domain from a list"
+ echo " Expects arguments in the format [list name]=[domain]"
+ echo
+ echo " -s | --setup | setup"
+ echo " Create the sqlite database"
+ echo
+ echo " -w | --writeLists | writeLists"
+ echo " Write out the IP address list for the selected list"
+ echo " Expects a list name as an argument, with '*' as a wildcard"
+ echo
+ echo " -wv4 | --writeListsIPv4 | writev4"
+ echo " Same as '-w', but writes only IPv4 format IP addresses"
+ echo
+ echo " -u | --ui | ui | --interactive | interactive"
+ echo " NOT YET IMPLEMENTED"
+ echo
+ echo " -x | --purge | purge"
+ echo " Accepts one numeric argument, [n], and deletes all lookups older than n days"
+ echo " and all nameservers which haven't been accessed in n days"
+ echo
+ echo " -y | --why | why"
+ echo " Show the lists containing a given domain name or IP address"
+ echo " Expects either a domain name or an IP address as an argument"
+ echo
+}
+# SCRIPT START
+###############################################################################
+cmd=""
+while [ ! -z "${1}" ]; do
+ case ${1} in
+ -a|--addDomain|add)
+ cmd="addDomain"
+ shift
+ ;;
+ -c|--lookupNext|next)
+ cmd="lookupNext"
+ shift
+ ;;
+ -i|--importLists|importLists)
+ cmd="importList"
+ shift
+ ;;
+ -l|--lookupLists|lookupLists)
+ cmd="lookupList"
+ shift
+ ;;
+ -n|--importNameservers|importNameservers)
+ cmd="importNameservers"
+ shift
+ ;;
+ -p|--printLists|print)
+ runCmd "printLists"
+ shift
+ ;;
+ -r|--removeDomain|remove)
+ cmd="removeDomain"
+ shift
+ ;;
+ -s|--setup|setup)
+ setup
+ exit 0
+ ;;
+ -w|--writeLists|writeLists)
+ cmd="writeList"
+ shift
+ ;;
+ -wv4|--writeListsIPv4|writev4)
+ cmd="writeListv4"
+ shift
+ ;;
+ -u|--ui|ui|--interactive|interactive)
+ runCmd "interactive"
+ shift
+ ;;
+ -x|--purge|purge)
+ cmd="purge"
+ shift
+ ;;
+ -y|--why|why)
+ cmd="why"
+ shift
+ ;;
+ *)
+ if [ -z "${cmd}" ]; then
+ error "Command not found: ${1}"
+ exit 4
+ fi
+ runCmd "${cmd}" "${1}"
+ shift
+ ;;
+ esac
+done
+
+
+
+
+
git://git.slightlycyberpunk.com / firewall-lists-3.git / commitdiff