GitPrep
Login
Showing 1 changed files with 49 additions and 41 deletions
+49 -41
newsParser/newsParser/newsChallengesFr.py
View
... ... 
@@ -2,54 +2,62 @@ from userio import *
2 2 
 import requests
3 3 
 import re
4 4 
 import newsParser
5 
+from requests_html import HTML
6 
+from requests_html import HTMLSession
7 
+
5 8
6 
-def articleAbonnes(content):
7 
-  articleAbonnes = "réservé aux abonnés"
8 
-  articleType = ""
9 
-  try:
10 
-    indexAbonnes = content.index(articleAbonnes)
11 
-    articleType = "Abonnés"
12 
-  except:
13 
-    articleType = ""
14 
-  return articleType
15 
-
16 9 
 def article(url):
17 10 
   say("Article: "+url)
18 
-  r = requests.get(url, allow_redirects=True)
19 
-  content = r.text
20 
-  articleStrImageUrl = articleImage(content)
21 
-  articleStrTitle = articleTitle(content)
22 
-  articleStrImageUrl = newsParser.articleImage(content)
23 
-  articleStrTitle = newsParser.articleTitle(content)
24 
-  articleStrType = articleAbonnes(content)
25 
-
26 
-  pageContent = ""
27 
-  pageContent += "<meta property=\"og:type\" content=\"article\">\n"
28 
-  pageContent += "<meta property=\"og:title\" content=\""+articleStrTitle+"\">\n"
29 
-  pageContent += "<meta property=\"og:description\" content=\""+articleStrDescription+"\">\n"
30 
-  pageContent += "<meta property=\"og:url\" content=\""+url+"\">\n"
31 
-  pageContent += "<meta property=\"og:image\" content=\""+articleStrImageUrl+"\">\n"
32 
-  pageContent += "<meta property=\"og:image:type\" content=\"image/jpeg\">"
33 
-  pageContent += "<h2>"+articleStrTitle+"</h2>\n"
34 
-  pageContent += "<img src=\""+articleStrImageUrl+"\">\n"
35 
-
36 
-  articleCstBegin = "<article "
37 
-  articleCstEnd   = "<div id=\"poool-widget\">"
38 
-  articleCstEnd2   = "</article>"
39 
-  indexBegin = content.index(articleCstBegin)
40 
-  try:
41 
-    indexEnd   = content.index(articleCstEnd)
42 
-  except:
43 
-    indexEnd   = content.index(articleCstEnd2)
44 
-  article_only = content[indexBegin:indexEnd]
11 
+  session = HTMLSession()
12 
+  response = session.get(url,timeout=20)
13 
+  pageContent=""
14 
+  article_only=""
15 
+  with response as r:
16 
+    articleStrTitle = r.html.xpath('//meta[@property="og:title"]/@content')[0]
17 
+    articleStrDescription = r.html.xpath('//meta[@property="og:description"]/@content')[0]
18 
+    articleStrImageUrl = r.html.xpath('//meta[@property="og:image"]/@content')[0]
19 
+    articleStrAuthor = r.html.xpath('//div[@class="author_wrapper"]/@content')
20 
+    article=r.html.find("main")[0]
21 
+    article_only+=article.html
22 
+    lenBefore=len(article_only)
23 
+    say("LengthBefore: "+str(lenBefore))
24 
+    pageContent += "<meta property=\"og:type\" content=\"article\" />\n"
25 
+    pageContent += "<meta property=\"og:title\" content=\""+articleStrTitle+"\" />\n"
26 
+    pageContent += "<meta property=\"og:description\" content=\""+articleStrDescription+"\" />\n"
27 
+    pageContent += "<meta property=\"og:url\" content=\""+url+"\" />\n"
28 
+    pageContent += "<meta property=\"og:image\" content=\""+articleStrImageUrl+"\" />\n"
29 
+    pageContent += "<meta property=\"og:image:type\" content=\"image/jpeg\" />\n"
30 
+    #pageContent += "<meta name=\"author\" content=\""+articleStrAuthor+"\" />\n"
31 
+
45 32 
   article_only = re.sub(r"<amp-img", '<img', article_only)
46 33 
   article_only = re.sub(r"</amp-img>", '', article_only)
47 34 
   article_only = re.sub(r"<h2", '<h3', article_only)
48 35 
   article_only = re.sub(r"</h2>", '</h3>', article_only)
49 36 
   article_only = re.sub(r"<h1", '<h2', article_only)
50 37 
   article_only = re.sub(r"</h1>", '</h2>', article_only)
51 
-
52 
-  article_only = re.sub(r"href=\"\/", 'href=\"//wwww.liberation.fr/', article_only)
53 
-  pageContent += article_only
54 
-  pageContent += "<p>"+articleStrType+"</p>"
38 
+  article_only = re.sub(r'<script(.+?)</script>','',article_only,flags=re.M|re.S)
39 
+  article_only = re.sub(r'<ul class="list-square">(.+?)</ul>','',article_only,flags=re.M|re.S)
40 
+  article_only = re.sub(r'<section class="ejs(.+?)</section>','',article_only,flags=re.M|re.S)
41 
+  article_only = re.sub(r'<form(.+?)</form>','',article_only,flags=re.M|re.S)
42 
+  article_only = re.sub(r'<a href="#comments"(.+?)</a>','',article_only,flags=re.M|re.S)
43 
+  article_only = re.sub(r'<a class="share-item (.+?)</a>','',article_only,flags=re.M|re.S)
44 
+  article_only = re.sub(r'<script(.+?)/>','',article_only)
45 
+  article_only = re.sub(r'<span class="label-comment">Réagir</span>','',article_only)
46 
+  article_only = re.sub(r'<div class="article-lire-reactions sm-visible button-reactions-first">(.+?)</div>','',article_only,flags=re.M|re.S)
47 
+  article_only = re.sub(r'<img data-src="https://www.challenges.fr/static_files/CHA-couv/couv_170.jpg" class="(.+?) lazyload" alt="Couverture du magazine"/>','',article_only,flags=re.M|re.S)
48 
+  article_only = re.sub(r'<svg version="1.1"(.+?)</svg>','',article_only,flags=re.M|re.S)
49 
+  #article_only = re.sub(r'','',article_only)
50 
+  article_only = re.sub(r"href=\"/",'href="https:///www.challenges.fr/',article_only)
51 
+  article_only = re.sub(r"src=\"/",'src="https:///www.challenges.fr/',article_only)
52 
+  article_only = re.sub(r"^$",'',article_only)
53 
+  article_only = re.sub(r'^\s*$', '',article_only,flags=re.M|re.S)
54 
+  article_only = re.sub(r"><",'>\n<',article_only)
55 
+
56 
+  #pageContent += "\n"+article_only+"\n"
57 
+  pageContent += "<article>\n"+article_only+"\n</article>\n"
58 
+  lenAfter=len(article_only)
59 
+  lenGain=float(10000-int(float(100*lenAfter/lenBefore)*100))/100
60 
+  say("LengthAfter : "+str(lenAfter))
61 
+  say("Gain        : "+str(lenGain)+"%")
55 62 
   return pageContent
63 
+
Gitprep Version v2.6.2 build on Mojolicious