From: Brian Flowers <git-admn@bsflowers.net>
Date: Wed, 1 Apr 2020 15:59:13 +0000 (-0400)
Subject: Rebuilding server
X-Git-Url: http://git.slightlycyberpunk.com%2C%20git.slightlycyberpunk.com/git/?a=commitdiff_plain;h=6d858fb18e7f40da7754a2add86c68a6a8a0d794;p=firewall-lists-3.git

Rebuilding server
---

6d858fb18e7f40da7754a2add86c68a6a8a0d794
diff --git a/TODO b/TODO
new file mode 100644
index 0000000..353dd91
--- /dev/null
+++ b/TODO
@@ -0,0 +1,16 @@
+TODO:
++  1) Confirm if lookup logic is sound (DONE)
++  2) Multithread lookup logic
++    Cleanup on kill
++   variable max thread count
+?  3) Skip comments / validate on nameserver / domain import
++  4) writeLists
+  5) cleanNameservers
++  6) Refactor input args
+  7) Check if lookup thread insert gives error and retry
+  8) Allow parallel inserts on the DB?
+  9) Move printLists
+ 10) writeLists add ipv4/ipv6 support
++ 11) troubleshoot -- enter an IP and lookup the domain/list info
+
+CONCEPT: Split lookups into n files, check each in a thread, write output to a file and dump that to sqlite sequentially/in bulk (DONE?)
diff --git a/bin/firewall-lists b/bin/firewall-lists
new file mode 100755
index 0000000..1e1c731
--- /dev/null
+++ b/bin/firewall-lists
@@ -0,0 +1,612 @@
+#!/bin/bash
+###############################################################################
+#
+# firewall-lists
+# This script takes a list of domain names, queries each against a list of
+# nameservers, and generates a list of corresponding IP addresses suitable
+# for use with various firewall and network software systems.
+#
+# Return Codes:
+#  1: User terminated
+#  2: Invalid argument
+#  3: Permission issue
+#  4: Sanity failure
+#  255: Unknown Signal Received
+#
+###############################################################################
+
+# CONFIGURATION
+###############################################################################
+
+#config_path -- the path of firewall-lists configuration (usually under etc)
+#onfig_path="/home/USERNAME/etc/firewall-lists"
+config_path="/home/urza9814/etc/firewall-lists-3"
+
+# lookup_threats -- maximum number of live nslookup connections
+lookup_threads=50
+
+# lookup_delay -- delay between launching new threads (for rate limiting)
+lookup_delay=5
+
+# Default output logfile -- leave null for standard output
+# This should usually be configured with the --logfile option
+logfile=""
+
+# Debug mode -- non-zero values enable debug output
+# To enable for a single execution you can use the -d or --debug flags
+debug=0
+
+# Trap UNCAUGHT ERRORS
+###############################################################################
+
+trap failure SIGFPE SIGHUP SIGABRT SIGALRM SIGQUIT
+trap term SIGTERM SIGINT
+
+term()
+{
+  echo
+  echo
+  error "TERMINATED."
+  info "Cleaning up child processes..."
+  ls "$config_path/pids" | while read pid; do
+    kill "$pid"
+    if [ `ps -p $pid | wc -l` -le 1 ]; then
+      rm "$config_path/pids/$pid"
+    fi
+  done
+  info "Exiting."
+  exit 1
+}
+
+failure()
+{
+  echo
+  echo
+  error "Something has gone SERIOUSLY wrong! (Received signal $?)"
+  exit 255
+}
+
+# UTILITY FUNCTIONS
+###############################################################################
+# Output functions
+error()
+{
+  printf "\e[41m" 1>&2
+  printf "`date` | ERROR: $1" 1>&2
+  printf "\e[0m\n" 1>&2
+}
+
+info()
+{
+  printf "\e[33m"
+  printf "`date` | INFO: $1"
+  printf "\e[0m\n"
+}
+
+warn()
+{
+  printf "\e[30;43m" 1>&2
+  printf "`date` | WARNING: $1" 1>&2
+  printf "\e[0m\n" 1>&2
+}
+
+debug()
+{
+    if [ $debug -ne 0 ]; then
+        return
+    fi
+    
+    length="`echo "${1}" | wc -l`"
+    if [ $length -lt 2 ]; then
+        printf "DEBUG: ${1}\n"
+    else
+        echo "${1}" | sed 's/^/\t/g'
+    fi
+}
+
+progress()
+{
+    if [ ! -z "${logfile}" ]; then
+        return
+    fi
+    
+    status="${1}"
+    total="${2}"
+    details="${3}"
+    
+    percent=`expr $status \* 100 / $total`
+    if [ ! -z "${details}" ]; then
+        printf "  $percent%% completed | $details\r"
+    else
+        printf "  $percent%% completed ($status / $total)\r"    
+    fi
+}
+
+
+
+lookupDomain()
+{
+  touch "${config_path}/pids/$BASHPID"
+  domain="${1}"
+  nameserver="${2}"
+  outfile="${3}"
+  
+  retval=1
+  counter=0
+  while [ $retval -ne 0 ]; do
+    nslookup -timeout=5 -retry=2 "${domain}" "${nameserver}" | sed '2d' | grep "^Address" | \
+      sed "s|Address: |INSERT INTO lookups(domain, ip) VALUES((SELECT id FROM domains WHERE url='${domain}'), '|g" |\
+      sed "s|$|');|g" >> "${outfile}"
+    retval=$?
+    counter=`expr $counter + 1`
+    if [ $counter -gt 10 ]; then
+      warn "Failed to insert after ten attempts; giving up!"
+      break;
+    fi
+  done
+
+  rm "${config_path}/pids/$BASHPID"
+}
+
+runCmd()
+{
+  case "${1}" in
+    importNameservers)
+      importNameservers "${2}"
+    ;;
+    importList)
+      importList "`echo "${2}" | cut -d'=' -f1`" "`echo "${2}" | cut -d'=' -f2-`"
+    ;;
+    lookupList)
+      lookupLists "${2}"
+    ;;
+    lookupNext)
+      lookupLists "`echo "${2}" | cut -d'=' -f1`" "`echo "${2}" | cut -d'=' -f2-`"
+    ;;
+    writeList)
+      writeList "${2}"
+    ;;
+    writeListv4)
+      writeList "${2}" "v4"
+    ;;
+    addDomain)
+      addDomain "`echo "${2}" | cut -d'=' -f1`" "`echo "${2}" | cut -d'=' -f2-`"
+    ;;
+    removeDomain)
+      removeDomain "`echo "${2}" | cut -d'=' -f1`" "`echo "${2}" | cut -d'=' -f2-`"
+    ;;
+    purge)
+      purge "${2}"
+    ;;
+    why)
+      why "${2}"
+    ;;
+    interactive)
+      interactive
+    ;;
+    printLists)
+      echo "List name  |  Domains Included"
+      sqlite3 "$config_path/firewall-lists.sqlite" <<EOF | tr '|' '\t'
+SELECT name, count(domains.id) FROM lists, domains, domainMap
+WHERE domainMap.domain = domains.id
+AND domainMap.list = lists.id
+GROUP BY lists.name
+EOF
+    ;;
+  esac
+}
+
+# MAIN SCRIPT FUNCTIONS
+###############################################################################
+# Configure the database
+setup()
+{
+  info "Creating tables (if necessary) at $config_path/firewall-lists.sqlite..."
+  sqlite3 "$config_path/firewall-lists.sqlite" "SELECT 1 FROM nameservers;" >/dev/null 2>&1
+  if [ $? -eq 1 ]; then
+    sqlite3 "$config_path/firewall-lists.sqlite"<<EOF
+CREATE TABLE nameservers(id INTEGER,
+       	     		 hostname VARCHAR(256) UNIQUE,
+                         lastAccess DATETIME,
+			 PRIMARY KEY(id));  
+EOF
+  fi
+
+  sqlite3 "$config_path/firewall-lists.sqlite" "SELECT 1 FROM lists;" >/dev/null 2>&1
+  if [ $? -eq 1 ]; then
+    sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+CREATE TABLE lists(id INTEGER,
+       	           name VARCHAR(256),
+                   expiry INTEGER default 2592000,
+		   PRIMARY KEY(id),
+		   UNIQUE(name));
+EOF
+  fi
+
+  sqlite3 "$config_path/firewall-lists.sqlite" "SELECT 1 FROM domains;" >/dev/null 2>&1
+  if [ $? -eq 1 ]; then
+    sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+CREATE TABLE domains(id INTEGER,
+       	     	     url VARCHAR(256),
+		     PRIMARY KEY(id),
+		     UNIQUE(url));
+EOF
+  fi
+
+  sqlite3 "$config_path/firewall-lists.sqlite" "SELECT 1 FROM domainMap;" >/dev/null 2>&1
+  if [ $? -eq 1 ]; then
+    sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+CREATE TABLE domainMap(id INTEGER,
+       	     	       list INTEGER,
+		       domain INTEGER,
+		       PRIMARY KEY(id),
+		       UNIQUE(list,domain));
+EOF
+  fi
+
+  sqlite3 "$config_path/firewall-lists.sqlite" "SELECT 1 FROM lookups;" >/dev/null 2>&1
+  if [ $? -eq 1 ]; then
+    sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+CREATE TABLE lookups(id INTEGER,
+       	     	     domain INTEGER,
+		     ip VARCHAR(128),
+		     lookupDate DATETIME default CURRENT_TIMESTAMP,
+		     PRIMARY KEY(id));
+EOF
+  fi
+}    
+
+# Import nameservers from a file to the database
+importNameservers()
+{
+  file="${1}"
+  grep -v "^[ ]*#" "${file}" | 
+    sed "s/\(.*\)/INSERT INTO nameservers(hostname) VALUES('\1');/g" |
+    sqlite3 "$config_path/firewall-lists.sqlite"
+}
+
+importList()
+{
+  name="${1}"
+  file="${2}"
+
+  sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+REPLACE INTO lists(name) VALUES('${name}');
+EOF
+
+  echo > "$config_path/.list-$name"
+  listId=`sqlite3 "$config_path/firewall-lists.sqlite" "SELECT id FROM lists WHERE name='${name}'"`
+  grep -v "^[ ]*#" "${file}" | while read domain; do
+    cat <<EOF >> "$config_path/.list-$name"    
+REPLACE INTO domains(url) VALUES('${domain}');
+REPLACE INTO domainMap(list, domain) VALUES(
+  (SELECT id FROM lists WHERE name='${name}'),
+  (SELECT id FROM domains WHERE url='${domain}'));
+EOF
+  done
+
+  cat "$config_path/.list-$name" | sqlite3 "$config_path/firewall-lists.sqlite"
+
+#  cat "${file}" |
+#    sed "s/\(.*\)/REPLACE INTO domains(url) VALUES('\1');/g" | \
+#    sqlite3 "$config_path/firewall-lists.sqlite"
+}
+
+lookupLists()
+{
+  list="${1}"
+  if [ ! -z "`echo "${list}" | grep '*'`" ]; then
+    list2="`echo "${list}" | sed 's/*/%/g'`"
+    list="`sqlite3 "$config_path/firewall-lists.sqlite" <<EOF | tr '\n' ',' | sed 's/,$//g' | sed "s/,/','/g"
+SELECT name FROM lists WHERE name LIKE '${list2}';
+EOF`"
+  fi
+
+  if [ "${2}" != "" ]; then
+    limit="LIMIT ${2}"
+  fi
+
+  sqlite3 "$config_path/firewall-lists.sqlite" <<EOF > "$config_path/.lookup-$$"
+SELECT url FROM
+(SELECT DISTINCT url FROM 
+(SELECT DISTINCT url, lookupDate FROM domains, domainMap, lists, lookups 
+WHERE domainMap.list = lists.id
+AND domainMap.domain = domains.id
+AND lists.name IN 
+('${list}')
+AND lookups.domain = domainMap.domain
+UNION ALL
+SELECT DISTINCT url, null FROM domains, domainMap, lists
+WHERE domainMap.list = lists.id
+AND domainMap.domain = domains.id
+AND lists.name IN
+('${list}')
+AND domainMap.domain NOT IN (SELECT domain FROM lookups))
+ORDER BY lookupDate ASC)
+${limit};
+EOF
+
+  info "Refreshing `cat "$config_path/.lookup-$$" | wc -l` records..."
+
+  sqlite3 "$config_path/firewall-lists.sqlite" <<EOF > "$config_path/.lookup-ns"
+SELECT hostname FROM nameservers;
+EOF
+
+  echo > "$config_path/.lookups.tmp"
+  while read nameserver; do
+    startCnt="`cat "$config_path/.lookups.tmp" | wc -l`"
+    while read domain; do
+      while [ `ls -ltr "${config_path}/pids/" 2>/dev/null | wc -l` -gt "$lookup_threads" ]; do
+        sleep 1
+      done
+      lookupDomain "${domain}" "${nameserver}" "$config_path/.lookups.tmp" &
+#      nslookup -timeout=5 -retry=2 "${domain}" "${nameserver}" | sed '2d' | grep "^Address" | \
+#        sed "s|Address: |INSERT INTO lookups(domain, ip) VALUES((SELECT id FROM domains WHERE url='${domain}'), '|g" |\
+#        sed "s|$|');|g" | sqlite3 "$config_path/firewall-lists.sqlite"
+    done < "$config_path/.lookup-$$"
+
+    endCnt="`cat "$config_path/.lookups.tmp" | wc -l`"
+    if [ "$endCnt" -gt "$startCnt" ]; then
+      sqlite3 "$config_path/firewall-lists.sqlite" "UPDATE nameservers SET lastAccess = date('now') WHERE hostname='${nameserver}';"
+    fi
+
+  done < "$config_path/.lookup-ns" 
+
+  cat "$config_path/.lookups.tmp" | sort -u | sqlite3 "$config_path/firewall-lists.sqlite"
+
+  info "Complete"
+}
+
+writeList()
+{
+  list="${1}"
+  type="${2}"
+
+  if [ "${type}" == "v4" ]; then
+    condition="AND lookups.ip NOT LIKE '%:%'"
+  fi
+
+  if [ ! -z "`echo "${list}" | grep '*'`" ]; then
+    list2="`echo "${list}" | sed 's/*/%/g'`"
+    list="`sqlite3 "$config_path/firewall-lists.sqlite" <<EOF | tr '\n' ',' | sed 's/,$//g' | sed "s/,/','/g"
+SELECT name FROM lists WHERE name LIKE '${list2}';
+EOF`"
+  fi
+
+  sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+SELECT domains.url || ':' || lookups.ip FROM lookups, domains, domainMap, lists
+WHERE lookups.domain = domains.id
+AND domainMap.domain = domains.id
+AND domainMap.list   = lists.id
+AND lists.name IN ('${list}')
+${condition};
+EOF
+
+}
+
+addDomain()
+{
+  list="${1}"
+  domain="${2}"
+
+  listId="`sqlite3 "$config_path/firewall-lists.sqlite" "SELECT id FROM lists WHERE name = '${list}'"`"
+  if [ -z "${listId}" ]; then
+    error "List not found: ${list}"
+    exit 2
+  fi
+
+  domainId="`sqlite3 "$config_path/firewall-lists.sqlite" "SELECT id FROM domains WHERE url = '${domain}'"`"
+  if [ -z "${domainId}" ]; then
+    sqlite3 "$config_path/firewall-lists.sqlite" "INSERT INTO domains(url) VALUES('${domain}')"
+    domainId="`sqlite3 "$config_path/firewall-lists.sqlite" "SELECT id FROM domains WHERE url = '${domain}'"`"
+    if [ -z "${domainId}" ]; then
+      error "Could not insert domain: ${domain}"
+      exit 4
+    fi
+  fi
+
+  sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+INSERT INTO domainMap(list, domain) VALUES (${listId}, ${domainId});
+EOF
+}
+
+removeDomain()
+{
+  list="${1}"
+  domain="${2}"
+
+  listId="`sqlite3 "$config_path/firewall-lists.sqlite" "SELECT id FROM lists WHERE name = '${list}'"`"
+  if [ -z "${listId}" ]; then
+    error "List not found: ${list}"
+    exit 2
+  fi
+
+  domainId="`sqlite3 "$config_path/firewall-lists.sqlite" "SELECT id FROM domains WHERE url = '${domain}'"`"
+  if [ -z "${domainId}" ]; then
+    error "Could not find domain: ${domain}"
+    exit 4
+  fi
+
+  sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+DELETE FROM domainMap WHERE list=${list} AND domain=${domainId};
+EOF
+}
+
+purge()
+{
+  days="${1}"
+
+  echo "Purging lookups older than ${days} days"
+  sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+DELETE FROM lookups WHERE lookupDate < DATETIME('now', '-${days} days');
+EOF
+
+  sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+DELETE FROM 
+EOF
+}
+
+why()
+{
+  domain="${1}"
+  ips="`echo $domain | grep -v [a-zA-Z]`"
+  if [ -z "${ips}" ]; then
+    results="`sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+SELECT domains.url, lists.name, lookups.ip, lookups.lookupDate
+FROM domains,domainMap,lists,lookups
+WHERE domains.id = domainMap.domain
+AND lists.id = domainMap.list
+AND lookups.domain = domains.id
+AND domains.url = '${domain}'
+EOF`"
+    ips="`echo ${results} | cut -d'|' -f3 | sed "s/^/'/g" | sed "s/\$/'/g" | tr '\n' ',' | sed 's/,$//g'`"
+  else
+    ips="'${ips}'"
+  fi
+    results="${results}
+`sqlite3 "$config_path/firewall-lists.sqlite" <<EOF
+SELECT domains.url, lists.name, lookups.ip, lookups.lookupDate
+FROM domains,domainMap,lists,lookups
+WHERE domains.id = domainMap.domain
+AND lists.id = domainMap.list
+AND lookups.domain = domains.id
+AND lookups.ip IN (${ips});
+EOF`"
+
+  echo "${results}" | sort -u
+}
+
+interactive()
+{
+  echo "STUB."
+}
+
+help()
+{
+  echo "Usage: $0 [operation] [argument(s)] where [operation] is one of:"
+  echo "  -a | --addDomain | add"
+  echo "     Add a domain to a list"
+  echo "     Expects one or more arguments in the format [list name]=[domain]"
+  echo
+  echo " -c | --lookupNext | next"
+  echo "    Lookups the next [n] domains, starting with those which were last scanned least recently."
+  echo "    Expects one argmment in the format [list regex]=[number of domains to update]"
+  echo
+#  echo " -d | --cleanNameservers | cleanNameservers"
+#  echo "    Accepts a signle numeric argument, [n], and deletes all nameservers with more than [n]% errors"
+#  echo
+  echo " -i | --importLists | importLists"
+  echo "    Import lists from a file"
+  echo "    Expects one argmunt, in the format [list name]=[filename]"
+  echo "    The file should be a list of domains to add to the list, with one per line"
+  echo
+  echo " -l | --lookupLists | lookupLists"
+  echo "    Lookup the selected lists and add the IP addresses to the database"
+  echo "    Expects arguments of the list name to lookup, with '*' as a wildcard"
+  echo
+  echo " -n | --importNameservers | importNameservers"
+  echo "    Import a list of nameservers to use to lookup IP addresses from domain names"
+  echo "    Expects a filename as the argument"
+  echo
+  echo " -p | --printLists | print"
+  echo "    Print the domains included in the selected lists"
+  echo "    Expects a list name as an argument, with '*' as a wildcard"
+  echo
+  echo " -r | --removeDomain | removeDomain"
+  echo "    Remove a domain from a list"
+  echo "    Expects arguments in the format [list name]=[domain]"
+  echo
+  echo " -s | --setup | setup"
+  echo "    Create the sqlite database"
+  echo
+  echo " -w | --writeLists | writeLists"
+  echo "    Write out the IP address list for the selected list"
+  echo "    Expects a list name as an argument, with '*' as a wildcard"
+  echo
+  echo " -wv4 | --writeListsIPv4 | writev4"
+  echo "    Same as '-w', but writes only IPv4 format IP addresses"
+  echo
+  echo " -u | --ui | ui | --interactive | interactive"
+  echo "    NOT YET IMPLEMENTED"
+  echo
+  echo " -x | --purge | purge"
+  echo "    Accepts one numeric argument, [n], and deletes all lookups older than n days"
+  echo "     and all nameservers which haven't been accessed in n days"
+  echo
+  echo " -y | --why | why"
+  echo "    Show the lists containing a given domain name or IP address"
+  echo "    Expects either a domain name or an IP address as an argument"
+  echo
+}
+# SCRIPT START
+###############################################################################
+cmd=""
+while [ ! -z "${1}" ]; do
+  case ${1} in
+    -a|--addDomain|add)
+      cmd="addDomain"
+      shift
+    ;;
+    -c|--lookupNext|next)
+      cmd="lookupNext"
+      shift
+    ;;
+    -i|--importLists|importLists)
+      cmd="importList"
+      shift
+    ;;
+    -l|--lookupLists|lookupLists)
+      cmd="lookupList"
+      shift
+    ;;
+    -n|--importNameservers|importNameservers)
+      cmd="importNameservers"
+      shift
+    ;;
+    -p|--printLists|print)
+      runCmd "printLists"
+      shift
+    ;;
+    -r|--removeDomain|remove)
+      cmd="removeDomain"
+      shift
+    ;;
+    -s|--setup|setup)
+      setup
+      exit 0
+    ;;
+    -w|--writeLists|writeLists)
+      cmd="writeList"
+      shift
+    ;;
+    -wv4|--writeListsIPv4|writev4)
+      cmd="writeListv4"
+      shift
+    ;;
+    -u|--ui|ui|--interactive|interactive)
+      runCmd "interactive"
+      shift
+    ;;
+    -x|--purge|purge)
+      cmd="purge"
+      shift
+    ;;
+    -y|--why|why)
+      cmd="why"
+      shift
+    ;;
+    *)
+      if [ -z "${cmd}" ]; then
+        error "Command not found: ${1}"
+        exit 4
+      fi
+      runCmd "${cmd}" "${1}"
+      shift
+    ;;
+  esac
+done
+
+
+
+
+
diff --git a/bin/th.sh b/bin/th.sh
new file mode 100755
index 0000000..b36df07
--- /dev/null
+++ b/bin/th.sh
@@ -0,0 +1,15 @@
+rm ~/etc/firewall-lists-3/firewall-lists.sqlite
+echo "`date` | setup"
+./firewall-lists --setup
+echo "`date` | importNS"
+./firewall-lists --importNameservers ~/etc/firewall-lists/nameservers
+echo "`date` | importLists"
+ls ~/etc/firewall-lists/lists/ | while read list; do
+  echo "`date` | Importing $list..."
+  ./firewall-lists --importLists ${list}=/home/urza9814/etc/firewall-lists/lists/${list}
+done
+echo "`date` | lookup"
+./firewall-lists --lookupLists "`ls ~/etc/firewall-lists/lists/ | tr '\n' ','`"
+echo "`date` | done"
+
+#./firewall-lists --importLists google=~/etc/firewall-lists/lists/google-block