WebSVN – ALCASAR – Diff – /scripts/alcasar-bl-autoupdate.sh


#!/bin/bash
 
# $Id: alcasar-bl.sh 2688 2019-01-18 23:15:49Z lucas.echard $
 
# alcasar-autoupdate.sh
# by Sven RATH and Rexy
# This script is distributed under the Gnu General Public License (GPL)
 
# Gestion de la BL pour le filtrage de domaine (via unbound) et d'URL (via E2guardian)
# Manage the BL for DnsBlackHole (unbound) and URL filtering (E2guardian)
 
FILE_tmp="/tmp/filesfilter.txt"
FILE_ip_tmp="/tmp/filesipfilter.txt"
DIR_DG="/etc/e2guardian/lists"
DIR_DG_BL="$DIR_DG/blacklists"
DIR_SHARE="/usr/local/share"
DIR_DNS_BL="$DIR_SHARE/unbound-bl"					# all the BL in the Unbound format
DIR_DNS_WL="$DIR_SHARE/unbound-wl"					# all the WL	'	'	'
DIR_IP_BL="$DIR_SHARE/iptables-bl"					# all the IP addresses of the BL
DIR_IP_WL="$DIR_SHARE/iptables-wl"					# IP ossi disabled WL
CNC_BL_NAME="ossi-bl-candc"
CNC_URL="https://osint.bambenekconsulting.com/feeds/"
CNC_DNS=${CNC_URL}c2-dommasterlist-high.txt
CNC_IP=${CNC_URL}c2-ipmasterlist-high.txt
SED="/bin/sed -i"
CURL="/usr/bin/curl"
 
# cleaning file and split it ("domains" in $FILE_tmp & "IP" in $FILE_ip_tmp)
function clean_split (){
	$SED '/^#.*/d' $FILE_tmp # remove commented lines
	$SED '/^\s*$/d' $FILE_tmp # remove empty lines
	$SED '/[äâëêïîöôüû@,]/d' $FILE_tmp # remove line with "chelou" characters
	# extract ip addresses for iptables.
	awk '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/{print "add bl_ip_blocked " $0}' $FILE_tmp > $FILE_ip_tmp
	# extract domain names for unbound.
	$SED -n '/^\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/!p' $FILE_tmp
	# Retrieve max Top Level Domain for domain name synthax
	#MAX_TLD=$(curl http://data.iana.org/TLD/tlds-alpha-by-domain.txt | grep -v '-' | grep -v '#' | wc -L)
	#if [ $(echo $MAX_TLD | wc -c) -eq 0 ];then
	#	MAX_TLD=18
	#fi
	# search for correction	egrep "([a-zA-Z0-9_-.]+\.){1,2}[a-zA-Z]{2,$MAX_TLD}" $ossi_custom_dir/domains > $FILE_tmp
}
 
usage="Usage: alcasar-bl-autoupdate.sh { -update_cat or --update_cat | -update_ossi-bl-candc or --update_ossi-bl-candc }"
nb_args=$#
args=$1
if [ $nb_args -eq 0 ]
then
	args="-h"
fi
case $args in
	-\? | -h* | --h*)
		echo "$usage"
		exit 0
		;;
	# Update the categories of Toulouse BL listed in "/usr/local/etc/update_cat.conf" (via rsync). Cron runs this function every 12h
	-update_cat | --update_cat)
		if [ $(cat /usr/local/etc/update_cat.conf | wc -l) -ne 0 ]
		then
			echo -n "Updating categories in /usr/local/etc/update_cat.conf ..."
			cat /usr/local/etc/update_cat.conf | while read LIGNE_RSYNC
			do
				CATEGORIE=$(echo $LIGNE_RSYNC | cut -d' ' -f1)
				URL=$(echo $LIGNE_RSYNC | cut -d' ' -f2)
				PATH_FILE=$(find $DIR_DG_BL/ -type d -name $CATEGORIE) # retrieve directory name of the category
				rsync -rv $URL $(dirname $PATH_FILE ) #rsync inside of the blacklist directory
				# Creation of unbound and Iptables BL and WL
				DOMAIN=$(basename $PATH_FILE)
				cp $PATH_FILE/domains $FILE_tmp
				clean_split  # clean ossi custom files & split them for unbound and for iptables
				black=`grep black $PATH_FILE/usage |wc -l`
				if [ $black == "1" ]
				then
					# adapt to the unbound syntax for the blacklist
					$SED "s?.*?local-zone: & typetransparent\nlocal-zone-tag: & blacklist?g" $FILE_tmp
					mv $FILE_tmp $DIR_DNS_BL/$DOMAIN.conf
					mv $FILE_ip_tmp $DIR_IP_BL/$DOMAIN
				else
					# adapt to the unbound syntax for the whitelist
					$SED "s?.*?local-zone: & transparent?g" $FILE_tmp
					mv $FILE_tmp $DIR_DNS_WL/$DOMAIN.conf
					mv $FILE_ip_tmp $DIR_IP_WL/$DOMAIN
				fi
				rm -f $FILE_tmp $FILE_ip_tmp
			done
			/usr/local/bin/alcasar-bl.sh --reload
		else
			echo -n "/usr/local/etc/update_cat.conf is empty ..."
		fi
		echo
		;;
	# Update C&C-Server Blacklist (TODO : check that there is a difference between two downloads)
	-update_ossi-bl-candc | --update_ossi-bl-candc)
		# check availability of the lists
        echo "Downloading blacklists from ${CNC_URL}..."
        STATUS_DNS_BL=$(${CURL} --connect-timeout 5 --write-out %{http_code} --silent --output /dev/null ${CNC_DNS})
        STATUS_IP_BL=$(${CURL} --connect-timeout 5 --write-out %{http_code} --silent --output /dev/null ${CNC_IP})
        # if downloaded successfully
        if [ $STATUS_DNS_BL = 200 ] && [ $STATUS_IP_BL = 200 ]; then
            ## parse domain names and ips from feed (cut first 19 lines (comments) and extract first column)
            CNC_DOMAINS=$($CURL $CNC_DNS | tail -n +19 | awk -F, '{print $1}')
            CNC_IPS=$($CURL $CNC_IP | tail -n +19 | awk -F, '{print $1}')
            ## create files and adapt downloaded data to alcasar structure (add newlines after each ip/domain)
            BL_DIR=${DIR_DG_BL}/${CNC_BL_NAME}
            rm -rf ${BL_DIR}
            mkdir $BL_DIR
            echo $CNC_DOMAINS | tr " " "\n" > ${BL_DIR}/urls
            echo $CNC_IPS | tr " " "\n" > ${BL_DIR}/domains
            ## reload ossi-blacklists to add the created blacklist to ALCASAR
            echo "Download successfull."
            /usr/local/bin/alcasar-bl.sh --reload
            exit 0
        # if server responded with a code different than 200
        else
            ## 000 means that curl failed
            if [ $STATUS_DNS_BL = 000 ] || [ $STATUS_IP_BL = 000 ]; then
                echo "ERROR: curl could not access the internet to download blacklists."
                echo "This appears to be an error on your side: please check the connection to the internet."
            else
                echo "ERROR: could not donwload blacklists: Server returned non-200 codes:"
                echo "${CNC_DNS} returned ${STATUS_DNS_BL}"
                echo "${CNC_IP} returned ${STATUS_IP_BL}"
                echo "Check the availability of the sites. Maybe the server removed its content or changed its address."            
            fi
	        exit 1
        fi
    	;;
esac
 

Subversion Repositories ALCASAR

(root)/scripts/alcasar-bl-autoupdate.sh – Rev 2769 → 2770

Rev 2769		Rev 2770
1	`#!/bin/bash`	1	`#!/bin/bash`
2		2
3	`# $Id: alcasar-bl.sh 2688 2019-01-18 23:15:49Z lucas.echard $`	3	`# $Id: alcasar-bl.sh 2688 2019-01-18 23:15:49Z lucas.echard $`
4		4
5	`# alcasar-autoupdate.sh`	5	`# alcasar-autoupdate.sh`
6	`# by Sven RATH and Rexy`	6	`# by Sven RATH and Rexy`
7	`# This script is distributed under the Gnu General Public License (GPL)`	7	`# This script is distributed under the Gnu General Public License (GPL)`
8		8
9	`# Gestion de la BL pour le filtrage de domaine (via unbound) et d'URL (via E2guardian)`	9	`# Gestion de la BL pour le filtrage de domaine (via unbound) et d'URL (via E2guardian)`
10	`# Manage the BL for DnsBlackHole (unbound) and URL filtering (E2guardian)`	10	`# Manage the BL for DnsBlackHole (unbound) and URL filtering (E2guardian)`
11		11
12	`FILE_tmp="/tmp/filesfilter.txt"`	12	`FILE_tmp="/tmp/filesfilter.txt"`
13	`FILE_ip_tmp="/tmp/filesipfilter.txt"`	13	`FILE_ip_tmp="/tmp/filesipfilter.txt"`
14	`DIR_DG="/etc/e2guardian/lists"`	14	`DIR_DG="/etc/e2guardian/lists"`
15	`DIR_DG_BL="$DIR_DG/blacklists"`	15	`DIR_DG_BL="$DIR_DG/blacklists"`
16	`DIR_SHARE="/usr/local/share"`	16	`DIR_SHARE="/usr/local/share"`
17	`DIR_DNS_BL="$DIR_SHARE/unbound-bl" # all the BL in the Unbound format`	17	`DIR_DNS_BL="$DIR_SHARE/unbound-bl" # all the BL in the Unbound format`
18	`DIR_DNS_WL="$DIR_SHARE/unbound-wl" # all the WL ' ' '`	18	`DIR_DNS_WL="$DIR_SHARE/unbound-wl" # all the WL ' ' '`
19	`DIR_IP_BL="$DIR_SHARE/iptables-bl" # all the IP addresses of the BL`	19	`DIR_IP_BL="$DIR_SHARE/iptables-bl" # all the IP addresses of the BL`
20	`DIR_IP_WL="$DIR_SHARE/iptables-wl" # IP ossi disabled WL`	20	`DIR_IP_WL="$DIR_SHARE/iptables-wl" # IP ossi disabled WL`
21	`CNC_BL_NAME="ossi-bl-candc"`	21	`CNC_BL_NAME="ossi-bl-candc"`
22	`CNC_URL="https://osint.bambenekconsulting.com/feeds/"`	22	`CNC_URL="https://osint.bambenekconsulting.com/feeds/"`
23	`CNC_DNS_BL_URL=${CNC_URL}c2-dommasterlist-high.txt`	23	`CNC_DNS=${CNC_URL}c2-dommasterlist-high.txt`
24	`CNC_IP_BL_URL=${CNC_URL}c2-ipmasterlist-high.txt`	24	`CNC_IP=${CNC_URL}c2-ipmasterlist-high.txt`
25	`SED="/bin/sed -i"`	25	`SED="/bin/sed -i"`
26	`CURL="/usr/bin/curl"`	26	`CURL="/usr/bin/curl"`
27		27
28	`# cleaning file and split it ("domains" in $FILE_tmp & "IP" in $FILE_ip_tmp)`	28	`# cleaning file and split it ("domains" in $FILE_tmp & "IP" in $FILE_ip_tmp)`
29	`function clean_split (){`	29	`function clean_split (){`
30	`$SED '/^#.*/d' $FILE_tmp # remove commented lines`	30	`$SED '/^#.*/d' $FILE_tmp # remove commented lines`
31	`$SED '/^\s*$/d' $FILE_tmp # remove empty lines`	31	`$SED '/^\s*$/d' $FILE_tmp # remove empty lines`
32	`$SED '/[äâëêïîöôüû@,]/d' $FILE_tmp # remove line with "chelou" characters`	32	`$SED '/[äâëêïîöôüû@,]/d' $FILE_tmp # remove line with "chelou" characters`
33	`# extract ip addresses for iptables.`	33	`# extract ip addresses for iptables.`
34	`awk '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/{print "add bl_ip_blocked " $0}' $FILE_tmp > $FILE_ip_tmp`	34	`awk '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/{print "add bl_ip_blocked " $0}' $FILE_tmp > $FILE_ip_tmp`
35	`# extract domain names for unbound.`	35	`# extract domain names for unbound.`
36	`$SED -n '/^\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/!p' $FILE_tmp`	36	`$SED -n '/^\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/!p' $FILE_tmp`
37	`# Retrieve max Top Level Domain for domain name synthax`	37	`# Retrieve max Top Level Domain for domain name synthax`
38	`#MAX_TLD=$(curl http://data.iana.org/TLD/tlds-alpha-by-domain.txt \| grep -v '-' \| grep -v '#' \| wc -L)`	38	`#MAX_TLD=$(curl http://data.iana.org/TLD/tlds-alpha-by-domain.txt \| grep -v '-' \| grep -v '#' \| wc -L)`
39	`#if [ $(echo $MAX_TLD \| wc -c) -eq 0 ];then`	39	`#if [ $(echo $MAX_TLD \| wc -c) -eq 0 ];then`
40	`# MAX_TLD=18`	40	`# MAX_TLD=18`
41	`#fi`	41	`#fi`
42	`# search for correction egrep "([a-zA-Z0-9_-.]+\.){1,2}[a-zA-Z]{2,$MAX_TLD}" $ossi_custom_dir/domains > $FILE_tmp`	42	`# search for correction egrep "([a-zA-Z0-9_-.]+\.){1,2}[a-zA-Z]{2,$MAX_TLD}" $ossi_custom_dir/domains > $FILE_tmp`
43	`}`	43	`}`
44		44
45	`usage="Usage: alcasar-bl-autoupdate.sh { -update_cat or --update_cat \| -update_ossi-bl-candc or --update_ossi-bl-candc }"`	45	`usage="Usage: alcasar-bl-autoupdate.sh { -update_cat or --update_cat \| -update_ossi-bl-candc or --update_ossi-bl-candc }"`
46	`nb_args=$#`	46	`nb_args=$#`
47	`args=$1`	47	`args=$1`
48	`if [ $nb_args -eq 0 ]`	48	`if [ $nb_args -eq 0 ]`
49	`then`	49	`then`
50	`args="-h"`	50	`args="-h"`
51	`fi`	51	`fi`
52	`case $args in`	52	`case $args in`
53	`-\? \| -h* \| --h*)`	53	`-\? \| -h* \| --h*)`
54	`echo "$usage"`	54	`echo "$usage"`
55	`exit 0`	55	`exit 0`
56	`;;`	56	`;;`
57	`# Update the categories of Toulouse BL listed in "/usr/local/etc/update_cat.conf" (via rsync). Cron runs this function every 12h`	57	`# Update the categories of Toulouse BL listed in "/usr/local/etc/update_cat.conf" (via rsync). Cron runs this function every 12h`
58	`-update_cat \| --update_cat)`	58	`-update_cat \| --update_cat)`
59	`if [ $(cat /usr/local/etc/update_cat.conf \| wc -l) -ne 0 ]`	59	`if [ $(cat /usr/local/etc/update_cat.conf \| wc -l) -ne 0 ]`
60	`then`	60	`then`
61	`echo -n "Updating categories in /usr/local/etc/update_cat.conf ..."`	61	`echo -n "Updating categories in /usr/local/etc/update_cat.conf ..."`
62	`cat /usr/local/etc/update_cat.conf \| while read LIGNE_RSYNC`	62	`cat /usr/local/etc/update_cat.conf \| while read LIGNE_RSYNC`
63	`do`	63	`do`
64	`CATEGORIE=$(echo $LIGNE_RSYNC \| cut -d' ' -f1)`	64	`CATEGORIE=$(echo $LIGNE_RSYNC \| cut -d' ' -f1)`
65	`URL=$(echo $LIGNE_RSYNC \| cut -d' ' -f2)`	65	`URL=$(echo $LIGNE_RSYNC \| cut -d' ' -f2)`
66	`PATH_FILE=$(find $DIR_DG_BL/ -type d -name $CATEGORIE) # retrieve directory name of the category`	66	`PATH_FILE=$(find $DIR_DG_BL/ -type d -name $CATEGORIE) # retrieve directory name of the category`
67	`rsync -rv $URL $(dirname $PATH_FILE ) #rsync inside of the blacklist directory`	67	`rsync -rv $URL $(dirname $PATH_FILE ) #rsync inside of the blacklist directory`
68	`# Creation of unbound and Iptables BL and WL`	68	`# Creation of unbound and Iptables BL and WL`
69	`DOMAIN=$(basename $PATH_FILE)`	69	`DOMAIN=$(basename $PATH_FILE)`
70	`cp $PATH_FILE/domains $FILE_tmp`	70	`cp $PATH_FILE/domains $FILE_tmp`
71	`clean_split # clean ossi custom files & split them for unbound and for iptables`	71	`clean_split # clean ossi custom files & split them for unbound and for iptables`
72	black=`grep black $PATH_FILE/usage \|wc -l`	72	black=`grep black $PATH_FILE/usage \|wc -l`
73	`if [ $black == "1" ]`	73	`if [ $black == "1" ]`
74	`then`	74	`then`
75	`# adapt to the unbound syntax for the blacklist`	75	`# adapt to the unbound syntax for the blacklist`
76	`$SED "s?.*?local-zone: & typetransparent\nlocal-zone-tag: & blacklist?g" $FILE_tmp`	76	`$SED "s?.*?local-zone: & typetransparent\nlocal-zone-tag: & blacklist?g" $FILE_tmp`
77	`mv $FILE_tmp $DIR_DNS_BL/$DOMAIN.conf`	77	`mv $FILE_tmp $DIR_DNS_BL/$DOMAIN.conf`
78	`mv $FILE_ip_tmp $DIR_IP_BL/$DOMAIN`	78	`mv $FILE_ip_tmp $DIR_IP_BL/$DOMAIN`
79	`else`	79	`else`
80	`# adapt to the unbound syntax for the whitelist`	80	`# adapt to the unbound syntax for the whitelist`
81	`$SED "s?.*?local-zone: & transparent?g" $FILE_tmp`	81	`$SED "s?.*?local-zone: & transparent?g" $FILE_tmp`
82	`mv $FILE_tmp $DIR_DNS_WL/$DOMAIN.conf`	82	`mv $FILE_tmp $DIR_DNS_WL/$DOMAIN.conf`
83	`mv $FILE_ip_tmp $DIR_IP_WL/$DOMAIN`	83	`mv $FILE_ip_tmp $DIR_IP_WL/$DOMAIN`
84	`fi`	84	`fi`
85	`rm -f $FILE_tmp $FILE_ip_tmp`	85	`rm -f $FILE_tmp $FILE_ip_tmp`
86	`done`	86	`done`
87	`/usr/local/bin/alcasar-bl.sh --reload`	87	`/usr/local/bin/alcasar-bl.sh --reload`
88	`else`	88	`else`
89	`echo -n "/usr/local/etc/update_cat.conf is empty ..."`	89	`echo -n "/usr/local/etc/update_cat.conf is empty ..."`
90	`fi`	90	`fi`
91	`echo`	91	`echo`
92	`;;`	92	`;;`
93	`# Update C&C-Server Blacklist (TODO : check that there is a difference between two downloads)`	93	`# Update C&C-Server Blacklist (TODO : check that there is a difference between two downloads)`
94	`-update_ossi-bl-candc \| --update_ossi-bl-candc)`	94	`-update_ossi-bl-candc \| --update_ossi-bl-candc)`
95	`# check availability of the lists`	95	`# check availability of the lists`
96	`echo "Downloading blacklists from ${CNC_URL}..."`	96	`echo "Downloading blacklists from ${CNC_URL}..."`
97	`STATUS_URL_BL=$(${CURL} --connect-timeout 5 --write-out %{http_code} --silent --output /dev/null ${CNC_DNS_BL_URL})`	97	`STATUS_DNS_BL=$(${CURL} --connect-timeout 5 --write-out %{http_code} --silent --output /dev/null ${CNC_DNS})`
98	`STATUS_IP_BL=$(${CURL} --connect-timeout 5 --write-out %{http_code} --silent --output /dev/null ${CNC_IP_BL_URL})`	98	`STATUS_IP_BL=$(${CURL} --connect-timeout 5 --write-out %{http_code} --silent --output /dev/null ${CNC_IP})`
99	`# if downloaded successfully`	99	`# if downloaded successfully`
100	`if [ $STATUS_URL_BL = 200 ] && [ $STATUS_IP_BL = 200 ]; then`	100	`if [ $STATUS_DNS_BL = 200 ] && [ $STATUS_IP_BL = 200 ]; then`
101	`## parse domain names and ips from feed (cut first 19 lines (comments) and extract first column)`	101	`## parse domain names and ips from feed (cut first 19 lines (comments) and extract first column)`
102	`CNC_URLS=$($CURL $CNC_DNS_BL_URL \| tail -n +19 \| awk -F, '{print $1}')`	102	`CNC_DOMAINS=$($CURL $CNC_DNS \| tail -n +19 \| awk -F, '{print $1}')`
103	`CNC_IPS=$($CURL $CNC_IP_BL_URL \| tail -n +19 \| awk -F, '{print $1}')`	103	`CNC_IPS=$($CURL $CNC_IP \| tail -n +19 \| awk -F, '{print $1}')`
104	`## create files and adapt downloaded data to alcasar structure (add newlines after each ip/domain)`	104	`## create files and adapt downloaded data to alcasar structure (add newlines after each ip/domain)`
105	`BL_DIR=${DIR_DG_BL}/${CNC_BL_NAME}`	105	`BL_DIR=${DIR_DG_BL}/${CNC_BL_NAME}`
106	`rm -rf ${BL_DIR}`	106	`rm -rf ${BL_DIR}`
107	`mkdir $BL_DIR`	107	`mkdir $BL_DIR`
108	`echo $CNC_URLS \| tr " " "\n" > ${BL_DIR}/urls`	108	`echo $CNC_DOMAINS \| tr " " "\n" > ${BL_DIR}/urls`
109	`echo $CNC_IPS \| tr " " "\n" > ${BL_DIR}/domains`	109	`echo $CNC_IPS \| tr " " "\n" > ${BL_DIR}/domains`
110	`## reload ossi-blacklists to add the created blacklist to ALCASAR`	110	`## reload ossi-blacklists to add the created blacklist to ALCASAR`
111	`echo "Download successfull."`	111	`echo "Download successfull."`
112	`/usr/local/bin/alcasar-bl.sh --reload`	112	`/usr/local/bin/alcasar-bl.sh --reload`
113	`exit 0`	113	`exit 0`
114	`# if server responded with a code different than 200`	114	`# if server responded with a code different than 200`
115	`else`	115	`else`
116	`## 000 means that curl failed`	116	`## 000 means that curl failed`
117	`if [ $STATUS_URL_BL = 000 ] \|\| [ $STATUS_IP_BL = 000 ]; then`	117	`if [ $STATUS_DNS_BL = 000 ] \|\| [ $STATUS_IP_BL = 000 ]; then`
118	`echo "ERROR: curl could not access the internet to download blacklists."`	118	`echo "ERROR: curl could not access the internet to download blacklists."`
119	`echo "This appears to be an error on your side: please check the connection to the internet."`	119	`echo "This appears to be an error on your side: please check the connection to the internet."`
120	`else`	120	`else`
121	`echo "ERROR: could not donwload blacklists: Server returned non-200 codes:"`	121	`echo "ERROR: could not donwload blacklists: Server returned non-200 codes:"`
122	`echo "${CNC_DNS_BL_URL} returned ${STATUS_URL_BL}"`	122	`echo "${CNC_DNS} returned ${STATUS_DNS_BL}"`
123	`echo "${CNC_IP_BL_URL} returned ${STATUS_IP_BL}"`	123	`echo "${CNC_IP} returned ${STATUS_IP_BL}"`
124	`echo "Check the availability of the sites. Maybe the server removed its content or changed its address."`	124	`echo "Check the availability of the sites. Maybe the server removed its content or changed its address."`
125	`fi`	125	`fi`
126	`exit 1`	126	`exit 1`
127	`fi`	127	`fi`
128	`;;`	128	`;;`
129	`esac`	129	`esac`
130		130