@@ -22,3 +22,8 @@ newsfetch.py -u <rss url> -o <output filename>

 The feed is parsed and a list of available article is created.

 The article content (i.e. Feed link) is fetch automatically and the content is extracted :

 - **<article>...</article>**

+

+## Examples

+- ./newsfetch.py --url http://www.vice.com/fr/rss

+- ./newsfetch.py --url http://www.lemonde.fr/rss/une.xml

+- ./newsfetch.py --url https://www.slate.fr/rss.xml

@@ -103,6 +103,7 @@ f.write("<h1 id=\"top\">"+feed_details.title+"</h1>\n")

 articles=list()

 cpt=0

+

 for article in d.entries:

   article_details = ArticleDetails()

   article_details.title = article.title.encode('utf-8').strip()

@@ -112,8 +113,11 @@ for article in d.entries:

   for link in article.links:

     if "enclosure" == link.rel:

       article_details.enclosure = link.href

+  

+  #~ article_details.content = article.content.value.encode('utf-8').strip()

+  #~ print len(article_details.content)

-  # Not Wroking as is. Generated image is too big

+  # Not Working as is. Generated image is too big

   #if article_details.enclosure is not None:

     #img_content=urllib2.urlopen(article.link).read()

     #article_details.enclosure = "data:image/jpg;base64,"+base64.b64encode(img_content)

@@ -131,13 +135,47 @@ f.write("<div id=\"article-current\"></div>\n\n")

 cpt=0

 for article in articles:

   print("-- {:d} : {:s}".format(cpt,article.title))

-  response = urllib2.urlopen(article.link)

-  article.content = response.read()

+  #~ print("  -- {:s}".format(article.link))

+  opener = urllib2.build_opener()

+  opener.addheaders = [('User-Agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:42.0) Gecko/20100101 Firefox/42.0')]

+  try:

+    response = opener.open(article.link)

+    article.content = response.read()

+  except:

+    print("  -- {:d}".format(response.code))

+    article.content = None

+        

+  if None == article.content:

+    cpt_prev=cpt-1

+    if cpt_prev < 0:

+      cpt_prev = 0

+    cpt_next=cpt+1

+    if cpt_next > cpt_num:

+      cpt_next = cpt_num

+    f.write("<div class=\"article\" id=\"article-"+str(cpt)+"\" style=\"display: none;\">\n")

+    f.write("<hr><a name=\"article-"+str(cpt)+"\">")

+    f.write("<div id=\"nav-up\" style=\"display:inline;\"><a href=\"#top\">&#8670;</a></div>\n")

+    f.write("<div id=\"nav-up\" style=\"display:inline;\"><a href=\"#article-top\">&#8613;</a></div>&nbsp;\n")

+    f.write("<div id=\"nav-source\" style=\"display:inline;\"><a href=\""+article.link+"\" target=\"new-"+str(cpt)+"\">source</a></div>&nbsp;")

+    f.write("<div id=\"nav-prev\" onclick=\"onArticle("+str(cpt_prev)+")\" style=\"display:inline;\">&#8612;</div>\n")

+    f.write("<div id=\"nav-next\" onclick=\"onArticle("+str(cpt_next)+")\" style=\"display:inline;\">&#8614;</div>\n")

+    f.write("<div class=\"extract-content\" id=\""+str(cpt)+"\">\n")

+    f.write(article.content_only)

+    f.write("\n</div>\n")

+    f.write("<div id=\"nav-up\" style=\"display:inline;\"><a href=\"#top\">&#8670;</a></div>\n")

+    f.write("<div id=\"nav-up\" style=\"display:inline;\"><a href=\"#article-top\">&#8613;</a></div>&nbsp;\n")

+    f.write("<div id=\"nav-source\" style=\"display:inline;\"><a href=\""+article.link+"\" target=\"new-"+str(cpt)+"\">source</a></div>&nbsp;")

+    f.write("<div id=\"nav-prev\" onclick=\"onArticle("+str(cpt_prev)+")\" style=\"display:inline;\">&#8612;</div>\n")

+    f.write("<div id=\"nav-next\" onclick=\"onArticle("+str(cpt_next)+")\" style=\"display:inline;\">&#8614;</div>\n")

+    f.write("</div>\n\n")

+    cpt=cpt+1

+    continue

+    

   soup_mysite = BeautifulSoup(article.content,"lxml")

   content = soup_mysite.find("article")

   if content == None:

     content = soup_mysite.find('div', attrs={'class':'article-page'})

-

+    

   article.content_only = str(content)

   article.content_only = article.content_only.replace(" href=\"/", " href=\"http://www.lemonde.fr/")

   article.content_only = article.content_only.replace('<script>require(["twitter/widgets"]);</script>','')

@@ -193,8 +231,7 @@ for article in articles:

   article.content_only = regexConjug.sub('\\1',article.content_only)

   # Diet

-  #~ article.content_only = html_slimmer(article.content_only.strip().replace('\n',' ').replace('\t',' ').replace('\r',' '))

-  #~ article.content_only = article.content_only.encode("utf-8")

+  article.content_only = html_slimmer(article.content_only.strip().replace('\n',' ').replace('\t',' ').replace('\r',' '))

   cpt_prev=cpt-1

   if cpt_prev < 0: