WebSVN – ALCASAR – Diff – /scripts/alcasar-bl.sh

 #/bin/bash
-# $Id: alcasar-bl.sh 2187 2017-04-27 22:18:45Z richard $
+# $Id: alcasar-bl.sh 2192 2017-05-01 19:52:53Z richard $
 # alcasar-bl.sh
 # by Franck BOUIJOUX and Richard REY
 # This script is distributed under the Gnu General Public License (GPL)
 	mv $FILE_tmp $WL_CATEGORIES
 	chown root:apache $WL_CATEGORIES
 	chmod 660 $WL_CATEGORIES
+}
+# cleaning file and split it ("domains" in $FILE_tmp & "IP" in $FILE_ip_tmp)
+function clean_split (){
+	$SED '/^#.*/d' $FILE_tmp # remove commented lines
+	$SED '/^\s*$/d' $FILE_tmp # remove empty lines
+	$SED '/[äâëêïîöôüû@,]/d' $FILE_tmp # remove line with "chelou" characters
+	# extract ip addresses for iptables.
+	awk '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/{print "add bl_ip_blocked " $0}' $FILE_tmp > $FILE_ip_tmp
+	# extract domain names for dnsmasq.
+	$SED -n '/^\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/!p' $FILE_tmp
+	# Retrieve max Top Level Domain for domain name synthax
+	#MAX_TLD=$(curl http://data.iana.org/TLD/tlds-alpha-by-domain.txt | grep -v '-' | grep -v '#' | wc -L)
+	#if [ $(echo $MAX_TLD | wc -c) -eq 0 ];then
+	#	MAX_TLD=18
+	#fi
+	# search for correction	egrep "([a-zA-Z0-9_-.]+\.){1,2}[a-zA-Z]{2,$MAX_TLD}" $ossi_custom_dir/domains > $FILE_tmp
+}
 usage="Usage: alcasar-bl.sh { -cat_choice or --cat_choice } | { -download or --download } | { -adapt or --adapt } | { -reload or --reload } | { -update_cat or --update_cat }"
 nb_args=$#
 args=$1
 if [ $nb_args -eq 0 ]
 then
 				mv $DIR_DG_BL/$x $DIR_tmp
 			done
 			rm -rf $DIR_DG_BL $DIR_IP_BL
 			mkdir $DIR_DG_BL $DIR_IP_BL
 			tar zxf $DIR_tmp/blacklists.tar.gz --directory=$DIR_DG/
-			chown -R root:apache $DIR_DG
+			chown -R dansguardian:apache $DIR_DG
 			chmod -R 770 $DIR_DG
 			# Add the two local categories (ossi-bl & ossi-wl) to the usage file
 			# Add the custom categories (ossi-tor_nodes) to the usage file
 			cat << EOF >> $DIR_DG_BL/global_usage
 		  		if [ ! -f $PATH_FILE/urls ] # create 'urls' file if it doesn't exist
 				then
 					touch $PATH_FILE/urls
 					chown dansguardian:apache $PATH_FILE/urls
 				fi
-				$SED "/[äâëêïîöôüû@]/d" $FILE_tmp
-				$SED "/^#.*/d" $FILE_tmp
+				cp $PATH_FILE/domains  $FILE_tmp
-				$SED "s/\.\{2,10\}/\./g" $PATH_FILE/domains $PATH_FILE/urls # correct some syntax errors
-				# extract ip addresses for iptables
-				awk '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/{print "add bl_ip_blocked " $0}' $PATH_FILE/domains > $FILE_ip_tmp
-				# for dnsmask, remove IP addesses, accented characters and commented lines.
+				clean_split # clean ossi custom files & split them for dnsmasq and for iptables
-				egrep  -v "^([0-9]{1,3}\.){3}[0-9]{1,3}$" $PATH_FILE/domains > $FILE_tmp
 				if [ "$LIST" == "$BL_CATEGORIES" ]
 				then
 					# adapt to the dnsmasq syntax for the blacklist
 					$SED "s?.*?address=/&/$PRIVATE_IP?g" $FILE_tmp
 					mv $FILE_tmp $DIR_DNS_BL/$DOMAIN.conf
 	       	chmod 770 $DIR_DNS_BL $DIR_DNS_WL $DIR_IP_BL $DIR_IP_WL
 		chmod -f 660 $BL_CATEGORIES $WL_CATEGORIES $BL_CATEGORIES_ENABLED $WL_CATEGORIES_ENABLED $DIR_DNS_BL/* $DIR_DNS_WL/* $DIR_IP_BL/* $DIR_IP_WL/*
 		rm -f $FILE_tmp $FILE_ip_tmp
 		rm -rf $DIR_tmp
 		;;
-	# update the categories which are written in "/usr/local/etc/update_cat.conf" with rsync
+	# Rsync the categories which are listed in "/usr/local/etc/update_cat.conf". Cron run this function every 12h
 	-update_cat | --update_cat)
 		if [ $(cat /usr/local/etc/update_cat.conf | wc -l) -ne 0 ]
 		then
 			echo -n "Updating categories in /usr/local/etc/update_cat.conf ..."
 			cat /usr/local/etc/update_cat.conf | while read LIGNE_RSYNC
 				URL=$(echo $LIGNE_RSYNC | cut -d' ' -f2)
 				PATH_FILE=$(find $DIR_DG_BL/ -type d -name $CATEGORIE) # retrieve directory name of the category
 				rsync -rv $URL $(dirname $PATH_FILE ) #rsync inside of the blacklist directory
 				# Creation of DNSMASQ and Iptables BL and WL
 				DOMAIN=$(basename $PATH_FILE)
-				$SED "s/\.\{2,10\}/\./g" $PATH_FILE/domains $PATH_FILE/urls # correction 'coma' instead of 'dot'
-				$SED "/^$/d" $PATH_FILE/domains $PATH_FILE/urls # remove empty lines
+				cp $PATH_FILE/domains $FILE_tmp
-				# extract ip addresses for iptables
-				awk '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/{print "add bl_ip_blocked " $0}' $PATH_FILE/domains > $FILE_ip_tmp
-				# for dnsmask, remove IP addresses, accentuated characters and commented lines.
+				clean_split  # clean ossi custom files & split them for dnsmasq and for iptables
-				egrep  -v "^([0-9]{1,3}\.){3}[0-9]{1,3}$" $PATH_FILE/domains > $FILE_tmp
-				$SED "/[äâëêïîöôüû@]/d" $FILE_tmp
-				$SED "/^#.*/d" $FILE_tmp
 				black=`grep black $PATH_FILE/usage |wc -l`
 				if [ $black == "1" ]
 				then
 					# adapt to the dnsmasq syntax for the blacklist
 					$SED "s?.*?address=/&/$PRIVATE_IP?g" $FILE_tmp
 		do
 			rm -f $dir/ossi*
 		done
 		find $DIR_DG_BL/ -type f -name domains | grep ossi- > $FILE_tmp # retrieve ossi directories name where a domain file exist
 		$SED "s?\/domains??g" $FILE_tmp # remove "/domains" suffix
-		#retrieve max Top Level Domain for domain name synthax
-		MAX_TLD=$(curl http://data.iana.org/TLD/tlds-alpha-by-domain.txt | grep -v '-' | grep -v '#' | wc -L)
-		if [ $(echo $MAX_TLD | wc -c) -eq 0 ];then
-			MAX_TLD=18
-		fi
 		for ossi_custom_dir in `cat $FILE_tmp` # create the blacklist and the whitelist files
 		do
+			chown -R root:apache $ossi_custom_dir
+			chmod 770 $ossi_custom_dir
 			ossi_categorie=`echo $ossi_custom_dir|cut -d "/" -f6`
 			short_categorie=`echo "$ossi_categorie" | cut -d"-" -f2`
 			if [ $short_categorie == "bl" ]
 			then
 				categorie_type="black"
 			else
 				categorie_type="white"
 			fi
-			$SED "s/\.\{2,10\}/\./g" $ossi_custom_dir/domains $ossi_custom_dir/urls # correction 'coma' instead of 'dot'
-			$SED "/^$/d" $ossi_custom_dir/domains $ossi_custom_dir/urls # remove empty lines
-			$SED "s/\r//" $ossi_custom_dir/domains $ossi_custom_dir/urls # remove Windows <CR>
+			$SED "s/\r//" $ossi_custom_dir/domains $ossi_custom_dir/urls # remove Windows <CR> from custom file
-			# extract ip addresses for iptables
+			cp $ossi_custom_dir/domains $FILE_tmp
-			awk '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/{print "add bl_ip_blocked " $0}' $ossi_custom_dir/domains > $FILE_ip_tmp
- 			egrep  -v "^([0-9]{1,3}\.){3}[0-9]{1,3}$" $ossi_custom_dir/domains > $FILE_tmp
+			clean_split # clean ossi custom files & split them for dnsmasq and for iptables
-# search for correction	egrep "([a-zA-Z0-9_-.]+\.){1,2}[a-zA-Z]{2,$MAX_TLD}" $ossi_custom_dir/domains > $FILE_tmp
 			if [ $categorie_type == "white" ]
 			then
 			# adapt the file to the dnsmasq syntax and enable it if needed
 			# for the WL
 				$SED "s?.*?server=/&/$DNS1?g" $FILE_tmp

Subversion Repositories ALCASAR

(root)/scripts/alcasar-bl.sh @ 2474 – Rev 2187 → 2192