# 下載 yahoo 新聞

#coding=UTF-8
#!/usr/bin/python

import requests
from bs4 import BeautifulSoup
import os
from datetime import datetime

# 字典，新聞分類。
newsDic = {
#   'politics' : '政治',
   # 'blogs' : '論壇',
#   'finance' : '財經',
#   'entertainment' : '娛樂',
   'sports' : '運動',
#   'society' : '社會地方',
#   'world' : '國際',
#   'lifestyle' : '生活',
#   'health' : '健康',
#   'technology' : '科技',
   }

# 全域變數
yahooNewsUrl = 'https://tw.news.yahoo.com/'
user_agent = {'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'}

def getNewsContent(fn, url):
   f = open(fn, 'w', encoding='utf-8')  # 開啟寫入檔案格式 utf8
   f.write(url + '\n')

   res = requests.get(url, headers=user_agent, timeout=30)
   if res.status_code == requests.codes.ok:
      soup = BeautifulSoup(res.text, 'html.parser')
      p = soup.find('h1')
#      print('[' + p.text + ']')
      if p != None:
         f.write(p.text + '\n')
      p = soup.find('time')
#      print('[' + p.text + ']')
      if p != None:
         f.write(p.text + '\n')
      p = soup.find('div', class_='author-name C(#000) Fw(b) Maw(300px) Whs(nw) Ov(h) Tov(e)')
#      print('[' + p.text + ']')
      if p != None:
         f.write(p.text + '\n')
      p = soup.find_all('p', class_='canvas-atom canvas-text Mb(1.0em) Mb(0)--sm Mt(0.8em)--sm')
      for s in p:
         if s.get('content') == s.text:
            f.write(s.text + '\n')
   f.close()

def getYahooNews2(subject, url):
   today = "{:%Y%m%d}".format(datetime.today())
   dir = os.path.dirname(__file__)
   path = os.path.join(dir, today)
   subDir = os.path.join(path, subject)
   if not os.path.isdir(subDir):
      os.mkdir(subDir)

   res = requests.get(url, headers=user_agent, timeout=30)  # 下載網頁內容
   if res.status_code == requests.codes.ok:  # 確認是否下載成功
      # 以 BeautifulSoup 解析 HTML 程式碼
      soup = BeautifulSoup(res.text, 'html.parser')
      # 以 CSS 的 class 抓出各類頭條新聞
#      a = soup.find_all('a')
      a = soup.find_all('a', class_='D(ib) Ov(h) Whs(nw) C($c-fuji-grey-l) C($c-fuji-blue-1-c):h Td(n) Fz(16px) Tov(e) Fw(700)')
      n = 0
      for s in a:
#         print(s.text)
#         print(s)
#         print('==========')
#         continue

         if s.get('href').find('http') >= 0:  # 部分超連結有完整 url
            url = s.get('href')
         else:
            url = yahooNewsUrl + subject + s.get('href')
         if url.find(yahooNewsUrl) == -1:  # 不是 Yahoo 新聞，會發生錯誤！
            continue
         if url.find('/video/') >= 0:  # 這個連結是影片，直接跳過。
            continue

         n = n + 1
         nStr = ('%02d.txt' % (n))
         fn = os.path.join(subDir, nStr)
         getNewsContent(fn, url)

def getYahooNews():
   today = "{:%Y%m%d}".format(datetime.today())
   dir = os.path.dirname(__file__)
   path = os.path.join(dir, today)
   if not os.path.isdir(path):
      os.mkdir(path)

   for d in newsDic:
      url = yahooNewsUrl + d
      getYahooNews2(d, url)

# 主程式 main
getYahooNews()
