GitPrep
Login
Showing 1 changed files with 52 additions and 44 deletions
+52 -44
newsParser/newsParser/newsCNA.py
View
... ... 
@@ -2,58 +2,66 @@ from userio import *
2 2 
 import requests
3 3 
 import re
4 4 
 import newsParser
5 
+from requests_html import HTML
6 
+from requests_html import HTMLSession
7 
+
5 8
6 
-def articleImage(content):
7 
-  articleImgBegin ="<meta property=\"og:image\" content=\""
8 
-  articleImgEnd   ="\">"
9 
-  indexImgBegin = content.index(articleImgBegin)
10 
-  indexImgEnd   = content.index(articleImgEnd,indexImgBegin)
11 
-  image = content[indexImgBegin+len(articleImgBegin):indexImgEnd]
12 
-  return image
13 
-
14 9 
 def article(url):
15 10 
   say("Article: "+url)
16 
-  r = requests.get(url, allow_redirects=True)
17 
-  content = r.text
18 
-  pageContent = ""
19 
-  articleCstBegin = "<article"
20 
-  articleCstEnd   = "<footer class=\"article__footer\">"
21 
-  indexBegin = content.index(articleCstBegin)
22 
-  indexEnd   = content.index(articleCstEnd)
23 
-  articleStrImageUrl = newsParser.articleImage(content)
24 
-
25 
-  articleStrTitle = newsParser.articleTitle(content)
26 
-  articleStrDescription = newsParser.articleDescription(content)
27 
-
28 
-  pageContent = ""
29 
-  pageContent += "<meta property=\"og:type\" content=\"article\">\n"
30 
-  pageContent += "<meta property=\"og:title\" content=\""+articleStrTitle+"\">\n"
31 
-  pageContent += "<meta property=\"og:description\" content=\""+articleStrDescription+"\">\n"
32 
-  pageContent += "<meta property=\"og:url\" content=\""+url+"\">\n"
33 
-  pageContent += "<meta property=\"og:image\" content=\""+articleStrImageUrl+"\">\n"
34 
-  pageContent += "<meta property=\"og:image:type\" content=\"image/jpeg\">"
11 
+  session = HTMLSession()
12 
+  response = session.get(url,timeout=20)
13 
+  pageContent=""
14 
+  article_only=""
15 
+  with response as r:
16 
+    articleStrTitle = r.html.xpath('//meta[@property="og:title"]/@content')[0]
17 
+    articleStrDescription = r.html.xpath('//meta[@property="og:description"]/@content')[0]
18 
+    articleStrImageUrl = r.html.xpath('//meta[@property="og:image"]/@content')[0]
19 
+    articleStrAuthor = r.html.xpath('//div[@class="author_wrapper"]/@content')
20 
+    article=r.html.find("main")[0]
21 
+    article_only+=article.html
22 
+    lenBefore=len(article_only)
23 
+    say("LengthBefore: "+str(lenBefore))
24 
+    pageContent += "<meta property=\"og:type\" content=\"article\" />\n"
25 
+    pageContent += "<meta property=\"og:title\" content=\""+articleStrTitle+"\" />\n"
26 
+    pageContent += "<meta property=\"og:description\" content=\""+articleStrDescription+"\" />\n"
27 
+    pageContent += "<meta property=\"og:url\" content=\""+url+"\" />\n"
28 
+    pageContent += "<meta property=\"og:image\" content=\""+articleStrImageUrl+"\" />\n"
29 
+    pageContent += "<meta property=\"og:image:type\" content=\"image/jpeg\" />\n"
30 
+    #pageContent += "<meta name=\"author\" content=\""+articleStrAuthor+"\" />\n"
35 31
36 
-  article_only = content[indexBegin:indexEnd]
37 32 
   article_only = re.sub(r"<amp-img", '<img', article_only)
38 33 
   article_only = re.sub(r"</amp-img>", '', article_only)
39 34 
   article_only = re.sub(r"<h2", '<h3', article_only)
40 35 
   article_only = re.sub(r"</h2>", '</h3>', article_only)
41 36 
   article_only = re.sub(r"<h1", '<h2', article_only)
42 37 
   article_only = re.sub(r"</h1>", '</h2>', article_only)
43 
-  newImage="<img src=\""+articleStrImageUrl+"\"><div class=\"article__author-link\">"
44 
-  article_only = re.sub(r"<div class=\"article__author-link\">", newImage, article_only)
45 
-  article_only = re.sub(r"<span class=\"advertisement__title\">Advertisement</span>", '', article_only)
46 
-
47 
-  article_only = re.sub(r"class=\"picture__image lazyload\"", '', article_only)
48 
-  article_only = re.sub(r"<a class=\"addthis_button(.*)</a>", '', article_only)
49 
-  article_only = re.sub(r"<div class=\"c-sharing--default is-article-top-position\"", '<div class="c-sharing--default is-article-top-position" style="display:none"', article_only)
50 
-  article_only = re.sub(r"<h3 class=\"save-for-later__title\">Bookmark</h3>", '', article_only)
51 
-  article_only = re.sub(r"<h2", '<h3', article_only)
52 
-  article_only = re.sub(r"</h2>", '</h3>', article_only)
53 
-  article_only = re.sub(r"<h1", '<h2', article_only)
54 
-  article_only = re.sub(r"</h1>", '</h2>', article_only)
55 
-  article_only = article_only.replace("><", ">\n<")
56 
-
57 
-  article_only = re.sub(r"href=\"\/", 'href=\"//www.channelnewsasia.com/', article_only)
58 
-  pageContent += "<article>"+article_only+"</article>"
38 
+  article_only = re.sub(r'<script(.+?)/>','',article_only)
39 
+  article_only = re.sub(r'<script(.+?)</script>','',article_only,flags=re.M|re.S)
40 
+  article_only = re.sub(r'<section class="block block-mc-content-share-bookmark block-content-share-bookmark clearfix">(.+?)</section>','',article_only,flags=re.M|re.S)
41 
+
42 
+  article_only = re.sub(r'<div id="ad-(.+?)</div>','',article_only,flags=re.M|re.S)
43 
+  article_only = re.sub(r'<section class="block block-ad-entity(.+?)</section>','',article_only,flags=re.M|re.S)
44 
+  article_only = re.sub(r'<div class="sponsors__item">(.+?)</div>','',article_only,flags=re.M|re.S)
45 
+  article_only = re.sub(r'<div class="sponsors">(.+?)</div>','',article_only,flags=re.M|re.S)
46 
+  article_only = re.sub(r'<a class="link link--trending"(.+?)</a>','',article_only,flags=re.M|re.S)
47 
+  article_only = re.sub(r'<div class="author-card__figure">(.+?)</div>','',article_only,flags=re.M|re.S)
48 
+  article_only = re.sub(r'<span class="social-link__text">(.+?)</span>','',article_only,flags=re.M|re.S)
49 
+  article_only = re.sub(r'<svg role="article"(.+?)</svg>','',article_only,flags=re.M|re.S)
50 
+  article_only = re.sub(r'<div class="OUTBRAIN"(.+?)</div>','',article_only,flags=re.M|re.S)
51 
+  article_only = re.sub(r'<div class="read-next__loader-icon"/>Content is loading...</div>','',article_only)
52 
+  article_only = re.sub(r'<a class="article__read-full-story-button"(.+?)</a>','',article_only,flags=re.M|re.S)
53 
+  #article_only = re.sub(r'','',article_only)
54 
+  article_only = re.sub(r"href=\"/",'href="https://www.channelnewsasia.com/',article_only)
55 
+  article_only = re.sub(r"src=\"/",'src="https://www.channelnewsasia.com/',article_only)
56 
+  article_only = re.sub(r"^$",'',article_only)
57 
+  article_only = re.sub(r'^\s*$', '',article_only,flags=re.M|re.S)
58 
+  article_only = re.sub(r"><",'>\n<',article_only)
59 
+
60 
+  pageContent += "<article>\n"+article_only+"\n</article>\n"
61 
+  #pageContent += article_only
62 
+  lenAfter=len(article_only)
63 
+  lenGain=float(10000-int(float(100*lenAfter/lenBefore)*100))/100
64 
+  say("LengthAfter : "+str(lenAfter))
65 
+  say("Gain        : "+str(lenGain)+"%")
59 66 
   return pageContent
67 
+
Gitprep Version v2.6.2 build on Mojolicious