| ... | ... |
@@ -0,0 +1,68 @@ |
| 1 |
+#!/bin/bash |
|
| 2 |
+ |
|
| 3 |
+array=("https://adaway.org/hosts.txt"
|
|
| 4 |
+"https://hosts-file.net/ad_servers.txt" |
|
| 5 |
+"https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext" |
|
| 6 |
+"http://winhelp2002.mvps.org/hosts.txt" |
|
| 7 |
+"http://sysctl.org/cameleon/hosts" |
|
| 8 |
+"http://rlwpx.free.fr/WPFF/hosts.htm" |
|
| 9 |
+"http://someonewhocares.org/hosts/hosts" |
|
| 10 |
+"http://www.malwaredomainlist.com/hostslist/hosts.txt" |
|
| 11 |
+"http://www.hostsfile.org/Downloads/hosts.txt" |
|
| 12 |
+"http://securemecca.com/Downloads/hosts.txt" |
|
| 13 |
+"http://adblock.gjtech.net/?format=unix-hosts" |
|
| 14 |
+"https://jansal.googlecode.com/svn/trunk/adblock/hosts" |
|
| 15 |
+"https://sites.google.com/site/logroid/files/hosts.txt?attredirects=0" |
|
| 16 |
+"http://optimate.dl.sourceforge.net/project/adzhosts/HOSTS.txt" |
|
| 17 |
+"http://hosts.eladkarako.com/hosts.txt" |
|
| 18 |
+"https://github.com/StevenBlack/hosts/raw/master/hosts" |
|
| 19 |
+"https://github.com/yous/YousList/raw/master/hosts.txt" |
|
| 20 |
+) |
|
| 21 |
+ |
|
| 22 |
+REDIRECTION="127.0.0.1" |
|
| 23 |
+WGET="wget -q" |
|
| 24 |
+TEMPHTML="/tmp/host.txt" |
|
| 25 |
+TEMPHTML2="/tmp/host2.txt" |
|
| 26 |
+OUTPUT="/var/www/html/completehosts-uniq.txt" |
|
| 27 |
+ |
|
| 28 |
+function clean_temp_files() |
|
| 29 |
+{
|
|
| 30 |
+ rm -f $TEMPHTML $TEMPHTML2 |
|
| 31 |
+} |
|
| 32 |
+ |
|
| 33 |
+ |
|
| 34 |
+trap clean_temp_files EXIT |
|
| 35 |
+ |
|
| 36 |
+rm -f $TEMPHTML $TEMPHTML2 |
|
| 37 |
+for link in ${array[@]}
|
|
| 38 |
+do |
|
| 39 |
+ echo "+- Source [$link]" |
|
| 40 |
+ $WGET $link -O $TEMPHTML |
|
| 41 |
+ dos2unix $TEMPHTML &> /dev/null |
|
| 42 |
+ cat $TEMPHTML >> $TEMPHTML2 |
|
| 43 |
+done |
|
| 44 |
+ |
|
| 45 |
+echo "+- Cleaning" |
|
| 46 |
+grep -v -e '^#' -e '^$' $TEMPHTML2|sort -u|iconv -t ASCII//TRANSLIT -f UTF-8 > $TEMPHTML |
|
| 47 |
+rm -f $TEMPHTML2 |
|
| 48 |
+ |
|
| 49 |
+NUM=$(cat $TEMPHTML |wc -l) |
|
| 50 |
+echo "+- # Domains Before $NUM" |
|
| 51 |
+ |
|
| 52 |
+echo "+- Extracting domains" |
|
| 53 |
+awk '{print $2}' $TEMPHTML > $TEMPHTML2
|
|
| 54 |
+ |
|
| 55 |
+echo "+- Removing Dup" |
|
| 56 |
+cat $TEMPHTML2 |sort -u |uniq > $TEMPHTML |
|
| 57 |
+awk '{print redir " " $1}' redir=$REDIRECTION $TEMPHTML | grep -v -e "^127.0.0.1 0.0.0.0$" -e "^127.0.0.1 $" > $TEMPHTML2
|
|
| 58 |
+cat $TEMPHTML2 | sort -u | uniq > $TEMPHTML |
|
| 59 |
+ |
|
| 60 |
+ |
|
| 61 |
+echo "#hostfile created $(date) by merging and deduplicating :" > $OUTPUT |
|
| 62 |
+for link in ${array[@]}
|
|
| 63 |
+do |
|
| 64 |
+ echo "# - $link" >> $OUTPUT |
|
| 65 |
+done |
|
| 66 |
+cat $TEMPHTML >> $OUTPUT |
|
| 67 |
+NUM=$(cat $TEMPHTML |wc -l) |
|
| 68 |
+echo "+- # Domains $NUM" |