@@ -17,6 +17,33 @@

 #"https://github.com/yous/YousList/raw/master/hosts.txt"

 #)

+#array=("https://adaway.org/hosts.txt"

+#"https://kawi.fr/android-hosts.txt"

+#"http://winhelp2002.mvps.org/hosts.txt"

+#"https://hosts-file.net/ad_servers.txt"

+#"https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext"

+#"http://sysctl.org/cameleon/hosts"

+#"http://rlwpx.free.fr/WPFF/hosts.htm"

+#"http://www.malwaredomainlist.com/hostslist/hosts.txt"

+#"https://sites.google.com/site/logroid/files/hosts.txt?attredirects=0"

+#"https://github.com/yous/YousList/raw/master/hosts.txt"

+#"https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-blocklist.txt"  

+#"https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt"

+#"https://zerodot1.gitlab.io/CoinBlockerLists/list.txt"

+#"https://zerodot1.gitlab.io/CoinBlockerLists/list_browser.txt"

+#"https://raw.githubusercontent.com/mitchellkrogza/Badd-Boyz-Hosts/master/hosts"

+#"https://raw.githubusercontent.com/r-a-y/mobile-hosts/master/AdguardMobileAds.txt"

+#"https://raw.githubusercontent.com/r-a-y/mobile-hosts/master/AdguardMobileSpyware.txt"

+#"https://raw.githubusercontent.com/EnergizedProtection/block/master/ultimate/formats/hosts.txt"

+#"https://raw.githubusercontent.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites/master/hacked-domains.list"

+#"https://someonewhocares.org/hosts/zero/hosts"

+#"https://www.malwaredomainlist.com/hostslist/hosts.txt"

+#"https://mirror.cedia.org.ec/malwaredomains/justdomains"

+#"https://mirror.cedia.org.ec/malwaredomains/immortal_domains.txt"

+#"https://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt"

+#"https://zeustracker.abuse.ch/blocklist.php?download=baddomains"

+#)

+

 array=("https://adaway.org/hosts.txt"

 "https://kawi.fr/android-hosts.txt"

 "http://winhelp2002.mvps.org/hosts.txt"

@@ -27,6 +54,17 @@ array=("https://adaway.org/hosts.txt"

 "http://www.malwaredomainlist.com/hostslist/hosts.txt"

 "https://sites.google.com/site/logroid/files/hosts.txt?attredirects=0"

 "https://github.com/yous/YousList/raw/master/hosts.txt"

+"https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-blocklist.txt"  

+"https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt"

+"https://raw.githubusercontent.com/r-a-y/mobile-hosts/master/AdguardMobileAds.txt"

+"https://raw.githubusercontent.com/r-a-y/mobile-hosts/master/AdguardMobileSpyware.txt"

+"https://mirror.cedia.org.ec/malwaredomains/justdomains"

+"https://mirror.cedia.org.ec/malwaredomains/immortal_domains.txt"

+"https://kawi.fr/auto-hosts.txt"

+"https://kawi.fr/eulerian-domain.hosts"

+"https://kawi.fr/criteo-domain.hosts"

+"https://hostfiles.frogeye.fr/firstparty-trackers-hosts.txt"

+"https://raw.githubusercontent.com/x0uid/SpotifyAdBlock/master/hosts"

 )

 REDIRECTION="127.0.0.1"

@@ -35,6 +73,10 @@ WGET="wget -q"

 TEMPHTML="/tmp/host.txt"

 TEMPHTML2="/tmp/host2.txt"

 OUTPUT="/var/www/html/completehosts-uniq.txt"

+OUTPUTOLD="/var/www/html/completehosts-uniq.old.txt"

+CLEAN_OUTPUT="/tmp/cleanhosts.txt"

+CLEAN_OUTPUTOLD="/tmp/cleanhostsold.txt"

+LOG="/tmp/hostsmod.log"

 function clean_temp_files()

 {

@@ -44,11 +86,16 @@ function clean_temp_files()

 trap clean_temp_files EXIT

-rm -f $TEMPHTML $TEMPHTML2

+if [ -e $OUTPUT ]

+then

+  mv $OUTPUT $OUTPUTOLD

+fi

+

+rm -f $TEMPHTML $TEMPHTML2 $CLEAN_OUTPUT $CLEAN_OUTPUTOLD $LOG

 cpt=0

 for link in ${array[@]}

 do

-  echo "+- Source [$link]"

+  echo "+- Source [$link]" >> $LOG

   $WGET $link -O $TEMPHTML

   dos2unix $TEMPHTML &> /dev/null

   cat $TEMPHTML >> $TEMPHTML2

@@ -57,20 +104,18 @@ do

   cpt=$(( $cpt + 1 ))

 done

-echo "+- Cleaning"

 grep -v -e '^#' -e '^$' $TEMPHTML2|sort -u|iconv -t ASCII//TRANSLIT -f UTF-8 > $TEMPHTML

 rm -f $TEMPHTML2

 NUM=$(cat $TEMPHTML |wc -l)

-echo "+- # Domains Before $NUM"

-echo "+- Extracting domains"

+#echo "+- Extracting domains"

 awk '{print $2}' $TEMPHTML > $TEMPHTML2

-echo "+- Removing Dup"

+#echo "+- Removing Dup"

 cat $TEMPHTML2 |sort -u |uniq > $TEMPHTML

 awk '{print redir " " $1}' redir=$REDIRECTION $TEMPHTML | grep -v -e "^127.0.0.1 0.0.0.0$" -e "^127.0.0.1 $" > $TEMPHTML2

-cat $TEMPHTML2 | sort -u | uniq > $TEMPHTML

+cat $TEMPHTML2 | grep -v coinhive | sort -u | uniq > $TEMPHTML

 echo "#hostfile created $(date) by merging and deduplicating :" > $OUTPUT

@@ -78,6 +123,24 @@ for link in ${array[@]}

 do

   echo "# - $link" >> $OUTPUT

 done

+grep -v -e xx.fbcdn.net -e '127.0.0.1 akamaihd.net' -e 'lh3.googleusercontent.com' $TEMPHTML > $TEMPHTML2

+

+mv $TEMPHTML2 $TEMPHTML

 cat $TEMPHTML >> $OUTPUT

-NUM=$(cat $TEMPHTML |wc -l)

-echo "+- # Domains $NUM"

+NUMAFTER=$(cat $TEMPHTML |wc -l)

+

+grep -v "hostfile created" $OUTPUT | grep -v '127.0.0.1 akamaihd.net' > $CLEAN_OUTPUT

+grep -v "hostfile created" $OUTPUTOLD > $CLEAN_OUTPUTOLD

+MD5NEW=$(md5sum $CLEAN_OUTPUT |awk '{print $1}')

+MD5OLD=$(md5sum $CLEAN_OUTPUTOLD |awk '{print $1}')

+

+if [ $MD5NEW != $MD5OLD ]

+then

+  cat $LOG

+  echo "+- # Domains Before Deduplicating $NUM"

+  echo "+- # Domains After Deduplicating  $NUMAFTER"

+  NUMUNIQOLD=$(cat $CLEAN_OUTPUTOLD|grep "^127"|wc -l)

+  NUMUNIQ=$(cat $CLEAN_OUTPUT|grep "^127"|wc -l)

+  echo "+- # $(( $NUMUNIQ - $NUMUNIQOLD )) New Additions"

+  diff $CLEAN_OUTPUTOLD $CLEAN_OUTPUT |grep '^>'

+fi