@@ -1,3 +1,24 @@

 # newsfetch.py

 Python scrapper to make 1 big HTML from NewsPaper RSS

+

+## Dependencies

+

+```bash

+sudo apt-get install libxml2-dev libxslt-dev

+sudo pip install bs4 feedparser lxml slimmer

+```

+

+## Usage

+```bash

+newsfetch.py -u <rss url> -o <output filename>

+```

+

+## Default Parameters

+- url : **http://www.lemonde.fr/rss/une.xml**

+- output : **default.html**

+

+## How it works

+The feed is parsed and a list of available article is created.

+The article content (i.e. Feed link) is fetch automatically and the content is extracted :

+- **<article>...</article>**

@@ -6,16 +6,19 @@ import base64

 import pprint

 import urllib2

 import lxml.html

+import re

 import sys

 import getopt

+from slimmer import html_slimmer 

 verbose = False

 output_filename = 'default.html'

-rss_url = 'rss_url'

+rss_url = 'http://www.lemonde.fr/rss/une.xml'

 options, remainder = getopt.getopt(sys.argv[1:], 'o:v', ['output=', 

                                                          'verbose',

+                                                         'url=',

                                                          ])

 for opt, arg in options:

     if opt in ('-o', '--output'):

@@ -57,15 +60,19 @@ class ArticleDetails(Printable):

     print("ArticleDetails:link      :{:s}".format(self.link))

     print("ArticleDetails:summary   :{:s}".format(self.summary))

     print("ArticleDetails:enclosure :{:s}".format(self.enclosure))

-    #~ print("ArticleDetails:content   :{:s}".format(self.content))    

     print("ArticleDetails:content_only :{:s}".format(self.content))    

     print("ArticleDetails:content   : ================")

+

+def process_match(m):

+    # Process the match here.

+    return ''

+    

 debug=1

 feed_details=FeedDetails()

-

-d = feedparser.parse('http://www.lemonde.fr/rss/une.xml')

+print rss_url

+d = feedparser.parse(rss_url)

 feed_details.title=d['feed']['title'].encode('utf-8').strip()

 feed_details.link=d['feed']['link'].encode('utf-8').strip()

 feed_details.subtitle=d['feed']['subtitle'].encode('utf-8').strip()

@@ -84,20 +91,20 @@ f.write("	<meta http-equiv=\"content-type\" content=\"text/html; charset=UTF-8\"

 f.write("	<meta name=\"viewport\" content=\"width=450px, user-scalable=no\">\n")

 f.write("	<link rel=\"stylesheet\" type=\"text/css\" href=\"style.css\" />\n")

 f.write("	<link rel=\"icon\" type=\"image/ico\" href=\"favicon.ico\">\n")

-f.write("	<!-- Touch Screen Detection -->\n")

-f.write("	<script>\n")

-f.write("	function isTouchDevice(){\n")

-f.write("		return true == (\"ontouchstart\" in window || window.DocumentTouch && document instanceof DocumentTouch);\n")

-f.write("	}\n")

-f.write("	</script>\n")

-f.write("	<script type=\"text/javascript\">\n")

-f.write("	/* Hack for Mobile */\n")

-f.write("	if(isTouchDevice()===true) {\n")

-f.write("		document.getElementById(\"img\").style.width = 400px;\n")

-f.write("		document.getElementById(\"extract-content\").style.width = 400px;\n")

-f.write("		document.getElementById(\"article-current\").style.width = 440px;\n")

-f.write("	}\n")

-f.write("	</script>\n")

+#~ f.write("	<!-- Touch Screen Detection -->\n")

+#~ f.write("	<script>\n")

+#~ f.write("	function isTouchDevice(){\n")

+#~ f.write("		return true == (\"ontouchstart\" in window || window.DocumentTouch && document instanceof DocumentTouch);\n")

+#~ f.write("	}\n")

+#~ f.write("	</script>\n")

+#~ f.write("	<script type=\"text/javascript\">\n")

+#~ f.write("	/* Hack for Mobile */\n")

+#~ f.write("	if(isTouchDevice()===true) {\n")

+#~ f.write("		document.getElementById(\"img\").style.width = 400px;\n")

+#~ f.write("		document.getElementById(\"extract-content\").style.width = 400px;\n")

+#~ f.write("		document.getElementById(\"article-current\").style.width = 440px;\n")

+#~ f.write("	}\n")

+#~ f.write("	</script>\n")

 f.write("	<script>\n")

 f.write("	function onArticle(index) {\n")

 f.write("		var string_index = \"article-\"+index;\n")

@@ -128,7 +135,7 @@ for article in d.entries:

   #if article_details.enclosure is not None:

     #img_content=urllib2.urlopen(article.link).read()

     #article_details.enclosure = "data:image/jpg;base64,"+base64.b64encode(img_content)

-  f.write("\t<ul><div onclick=\"onArticle("+str(cpt)+")\" style=\"display:inline;\">\n")  

+  f.write("<ul><div onclick=\"onArticle("+str(cpt)+")\" style=\"display:inline;\">\n")  

   f.write("\t<img src=\""+article_details.enclosure+"\" style=\"display:inline;\"><br>\n")

   f.write("\t<div id=\"nav-up\" style=\"display:inline;\"><a href=\"#top\">&#8670;</a></div>\n")

   f.write("\t<div id=\"nav-up\" style=\"display:inline;\"><a href=\"#article-top\">&#8615;</a></div>\n")

@@ -146,24 +153,52 @@ for article in articles:

   article.content = response.read()

   soup_mysite = BeautifulSoup(article.content,"lxml")

   content = soup_mysite.find("article")

-  article.content_only = content

+  article.content_only = str(content)

+  article.content_only = article.content_only.replace(" href=\"/", " href=\"http://www.lemonde.fr/")

+  article.content_only = article.content_only.replace('<script>require(["twitter/widgets"]);</script>','')

+  article.content_only = article.content_only.replace('<script async src="//platform.twitter.com/widgets.js" charset="utf-8"></script></div>','')

+  article.content_only = article.content_only.replace('<div class="toolbar"></div>','')

+  article.content_only = article.content_only.replace('<figure class="illustration_haut   " style="width: 534px">','<figure>')

+  article.content_only = article.content_only.replace('<figure class="illustration_haut">','<figure>')

+  article.content_only = article.content_only.replace('<span id="publisher" itemprop="Publisher" data-source="LE MONDE">Le Monde</span>','Le Monde')

+  article.content_only = article.content_only.replace(' data-lazyload="false" ',' ')

+  article.content_only = article.content_only.replace('Par<span>','Par <span>')

+  article.content_only = article.content_only.replace('<span class="source_image" ','<span ')

+  article.content_only = article.content_only.replace('<div class="bg_gris_moyen signature">','<div>')

+  article.content_only = article.content_only.replace('<p itemprop="author" class="auteur txt2_120">','<p>')

+  article.content_only = article.content_only.replace('<li class="clearfix" itemprop="author">','<li>')

+  article.content_only = article.content_only.replace('<img width="534"','<img width="400"')

+  article.content_only = article.content_only.replace('onclick="return false;" ','')

+  article.content_only = article.content_only.replace('target="_blank"','')

+  article.content_only = article.content_only.replace('class="lien_interne rub"','')

+  regexConjug = re.compile(r'<a class=\"lien_interne conjug\".*?>(.+?)</a>')

+  article.content_only = regexConjug.sub('\\1',article.content_only)

+  #~ article.content_only = article.content_only.replace('','')

+  #~ article.content_only = article.content_only.replace('','')

+  #~ article.content_only = article.content_only.replace('','')

+  #~ article.content_only = article.content_only.replace('','')

+  #~ article.content_only = article.content_only.replace('','')

+  #~ article.content_only = html_slimmer(article.content_only)

+  article.content_only = html_slimmer(article.content_only.strip().replace('\n',' ').replace('\t',' ').replace('\r',' '))

+  

+

+

   cpt_prev=cpt-1

   if cpt_prev < 0:

     cpt_prev = 0

   cpt_next=cpt+1

   if cpt_next > cpt_num:

     cpt_next = cpt_num

-  

-  f.write("<!-- ==================== article "+str(cpt)+" ============== -->\n")

+

   f.write("<div class=\"article\" id=\"article-"+str(cpt)+"\" style=\"display: none;\">\n")

-  f.write("<hr>\n<a name=\"article-"+str(cpt)+"\">\n")

+  f.write("<hr><a name=\"article-"+str(cpt)+"\">")

   f.write("<div id=\"nav-up\" style=\"display:inline;\"><a href=\"#top\">&#8670;</a></div>\n")

   f.write("<div id=\"nav-up\" style=\"display:inline;\"><a href=\"#article-top\">&#8613;</a></div>&nbsp;\n")

   f.write("<div id=\"nav-source\" style=\"display:inline;\"><a href=\""+article.link+"\" target=\"new-"+str(cpt)+"\">source</a></div>&nbsp;")

   f.write("<div id=\"nav-prev\" onclick=\"onArticle("+str(cpt_prev)+")\" style=\"display:inline;\">&#8612;</div>\n")

   f.write("<div id=\"nav-next\" onclick=\"onArticle("+str(cpt_next)+")\" style=\"display:inline;\">&#8614;</div>\n")

   f.write("<div class=\"extract-content\" id=\""+str(cpt)+"\">\n")

-  f.write(str(content))

+  f.write(article.content_only)

   f.write("\n</div>\n")

   f.write("<div id=\"nav-up\" style=\"display:inline;\"><a href=\"#top\">&#8670;</a></div>\n")

   f.write("<div id=\"nav-up\" style=\"display:inline;\"><a href=\"#article-top\">&#8613;</a></div>&nbsp;\n")