@@ -2,45 +2,66 @@ from userio import *

 import requests

 import re

 import newsParser

+from requests_html import HTML

+from requests_html import HTMLSession

+

 def article(url):

   say("Article: "+url)

-  r = requests.get(url, allow_redirects=True, headers={'Accept-Encoding': 'deflate', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36'})

-  content = r.text

+  session = HTMLSession()

+  response = session.get(url,timeout=20)

+  pageContent=""

+  article_only=""

+  with response as r:

+    articleStrTitle = r.html.xpath('//meta[@property="og:title"]/@content')[0]

+    articleStrDescription = r.html.xpath('//meta[@property="og:description"]/@content')[0]

+    articleStrImageUrl = r.html.xpath('//meta[@property="og:image"]/@content')[0]

+    articleStrAuthor = r.html.xpath('//div[@class="author_wrapper"]/@content')

+    article=r.html.find("article")[0]

+    article_only+=article.html

+    lenBefore=len(article_only)

+    say("LengthBefore: "+str(lenBefore))

+    pageContent += "<meta property=\"og:type\" content=\"article\" />\n"

+    pageContent += "<meta property=\"og:title\" content=\""+articleStrTitle+"\" />\n"

+    pageContent += "<meta property=\"og:description\" content=\""+articleStrDescription+"\" />\n"

+    pageContent += "<meta property=\"og:url\" content=\""+url+"\" />\n"

+    pageContent += "<meta property=\"og:image\" content=\""+articleStrImageUrl+"\" />\n"

+    pageContent += "<meta property=\"og:image:type\" content=\"image/jpeg\" />\n"

+    #pageContent += "<meta name=\"author\" content=\""+articleStrAuthor+"\" />\n"

-  articleStrImageUrl = newsParser.articleImage(content)

-  articleStrTitle = newsParser.articleTitle(content)

-  articleStrDescription = newsParser.articleDescription(content)

-  

-  pageContent = ""

-  pageContent += "<meta property=\"og:type\" content=\"article\">\n"

-  pageContent += "<meta property=\"og:title\" content=\""+articleStrTitle+"\">\n"

-  pageContent += "<meta property=\"og:description\" content=\""+articleStrDescription+"\">\n"

-  pageContent += "<meta property=\"og:url\" content=\""+url+"\">\n"

-  pageContent += "<meta property=\"og:image\" content=\""+articleStrImageUrl+"\">\n"

-  pageContent += "<meta property=\"og:image:type\" content=\"image/jpeg\">"

-  

-  articleCstBegin = "<article"

-  articleCstEnd   = "<div class=\"related-entries"

-  articleCstEnd2   = "</article>"

-  indexBegin = content.index(articleCstBegin)

-  try:

-    indexEnd   = content.index(articleCstEnd)

-  except:

-    indexEnd   = content.index(articleCstEnd2)

-  article_only = content[indexBegin:indexEnd]

   article_only = re.sub(r"<amp-img", '<img', article_only)

   article_only = re.sub(r"</amp-img>", '', article_only)

   article_only = re.sub(r"<h2", '<h3', article_only)

   article_only = re.sub(r"</h2>", '</h3>', article_only)

   article_only = re.sub(r"<h1", '<h2', article_only)

   article_only = re.sub(r"</h1>", '</h2>', article_only)

-  article_only = re.sub(r"<a class=\"share(.*?)\" data-social-name=\"(.*?)\" href=\"(.*?)\" target=\"_blank\">", '<a href="">', article_only)

-  article_only = re.sub(r"<li class=\"(.*?) share-bar__item\">", '<li>', article_only)

-  article_only = re.sub(r"<div class=\"share-bar share-bar--sticky yr-share\">",'<div class="share-bar share-bar--sticky yr-share" style="display:none;">', article_only)

-  article_only = re.sub(r"<div class=\"(.*?) share-bar(.*?)>",'<div style="display:none;">', article_only)

-  article_only = re.sub(r"<div class=\"yr-share\">",'<div style="display:none;">', article_only)

-  article_only = article_only.replace("><", ">\n<")

- 

-  pageContent += "<article>"+article_only+"</article>"

+  article_only = re.sub(r'<script(.+?)</script>','',article_only,flags=re.M|re.S)

+  article_only = re.sub(r'<script(.+?)/>','',article_only)

+  article_only = re.sub(r'<form(.+?)</form>','',article_only,flags=re.M|re.S)

+  article_only = re.sub(r'<img src="(.+?)partage-(.+?).svg"/>','',article_only)

+  article_only = re.sub(r'<li data-socials=(.+?)</li>','',article_only,flags=re.M|re.S)

+  article_only = re.sub(r'<input type="email" (.+?)/>','',article_only)

+  article_only = re.sub(r'<link rel="stylesheet" type="text/css"(.+?)>','',article_only)

+  article_only = re.sub(r'<p class="asset asset-text">','<p>',article_only)

+  article_only = re.sub(r'<div class="homeNewsletter-notice">(.+?)</div>', '',article_only,flags=re.M|re.S)

+  article_only = re.sub(r'<button class="homeNewsletter-button">(.+?)</button>', '',article_only,flags=re.M|re.S)

+  article_only = re.sub(r'<div class="homeNewsletter-input">(.+?)</div>', '',article_only,flags=re.M|re.S)

+  article_only = re.sub(r'<div class="homeNewsletter-content">(.+?)</div>', '',article_only,flags=re.M|re.S)

+  article_only = re.sub(r'<div class="homeNewsletter-title">(.+?)</div>', '',article_only,flags=re.M|re.S)

+  article_only = re.sub(r'<div class="homeNewsletter-form">(.+?)</div>', '',article_only,flags=re.M|re.S)

+  #article_only = re.sub(r'','',article_only)

+  article_only = re.sub(r"href=\"/",'href="https://www.huffingtonpost.fr/',article_only)

+  article_only = re.sub(r"src=\"/",'src="https://www.huffingtonpost.fr/',article_only)

+  article_only = re.sub(r"^$",'',article_only)

+  article_only = re.sub(r'^\s*$', '',article_only,flags=re.M|re.S)

+  article_only = re.sub(r"><",'>\n<',article_only)

+  article_only = re.sub(r"> <",'>\n<',article_only)

+

+  #pageContent += "\n"+article_only+"\n"

+  pageContent += "<article>\n"+article_only+"\n</article>\n"

+  lenAfter=len(article_only)

+  lenGain=float(10000-int(float(100*lenAfter/lenBefore)*100))/100

+  say("LengthAfter : "+str(lenAfter))

+  say("Gain        : "+str(lenGain)+"%")

   return pageContent

+