# -*- coding:UTF-8 -*-
#!/usr/bin/python

import requests
from bs4 import BeautifulSoup
import os
from datetime import datetime
import codecs

# 字典，新聞分類。
newsDic = {
   'politics' : '政治',
   # 'blogs' : '論壇',
   'finance' : '財經',
   'entertainment' : '娛樂',
   'sports' : '運動',
   'society' : '社會地方',
   'world' : '國際',
   'lifestyle' : '生活',
   'health' : '健康',
   'technology' : '科技',
   }

# 全域變數
today = "{:%Y%m%d}".format(datetime.today())
dir = os.path.dirname(__file__)
path = os.path.join(dir, today)
yahooNewsUrl = 'https://tw.news.yahoo.com/'
user_agent = {'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'}

def printNames():
   if not os.path.isdir(path):
      os.mkdir(path)
   fn = os.path.join(dir, today + "\\names.txt")
   f = codecs.open(fn, 'w', 'utf-8')
   n = 1
   f.write('#\n# Title=Yahoo 奇摩新聞\n#\n')
   for d in newsDic:
      f.write('Name=%s\n' % newsDic[d])
      f.write('Path=~/%s\n' % d)
      f.write('Numb=%d\n' % n)
      f.write('#\n')
      n = n + 1
   f.close()

def myReplace(str):
   str = str.replace('<br>', '\n')
   return str

def getNewsContent(fn, url, str):
   f = codecs.open(fn, 'w', 'utf-8')
   f.write(url + '\n')

   res = requests.get(url, headers=user_agent, timeout=30)
   if res.status_code == requests.codes.ok:
      soup = BeautifulSoup(res.text, 'html.parser')
      p = soup.find('h1')
#      print('[' + p.text + ']')
      if p != None:
         f.write(p.text + '\n')
      p = soup.find('time')
#      print('[' + p.text + ']')
      if p != None:
         f.write(p.text + '\n')
      p = soup.find('div', class_='author-name C(#000) Fw(b) Maw(300px) Whs(nw) Ov(h) Tov(e)')
#      print('[' + p.text + ']')
      if p != None:
         f.write(p.text + '\n')
      p = soup.find_all('p', class_=str)
      bLine = False
      for s in p:
         if s.get('content') == s.text:
            bLine = True
            if s.text.find('▲') == 0:
               continue
#            print('[' + s.text + ']')
            f.write(myReplace(s.text) + '\n')
      if not bLine:  # 沒有新聞內容，少印一行。
         f.write(myReplace(s.text) + '\n')
   f.close()

def getYahooNews2(subject, url, str):
   subDir = path + '\\' + subject
   if not os.path.isdir(subDir):
      os.mkdir(subDir)
   fn = subDir + '\\names.txt'
   f = codecs.open(fn, 'w', 'utf-8')
   f.write('#\n# Title=%s新聞\n#\n' % newsDic[subject])

   res = requests.get(url, headers=user_agent, timeout=30)  # 下載網頁內容
   if res.status_code == requests.codes.ok:  # 確認是否下載成功
      # 以 BeautifulSoup 解析 HTML 程式碼
      soup = BeautifulSoup(res.text, 'html.parser')
      # 以 CSS 的 class 抓出各類頭條新聞
      a = soup.find_all('a', class_=str)
      n = 1
      for s in a:
         if s.get('href').find('http') >= 0:  # 部分超連結有完整 url
            url = s.get('href')
         else:
            url = yahooNewsUrl + subject + s.get('href')
         if url.find(yahooNewsUrl) == -1:  # 不是 Yahoo 新聞，會發生錯誤！
            continue
         if url.find('/video/') >= 0:  # 這個連結是影片，直接跳過。
            continue

         f.write('Name=%s\n' % s.text)
         f.write('Path=~/%02d.txt\n' % n)
         f.write('Numb=%d\n' % n)
         f.write('#\n')

         fn = ('%s\\%02d.txt' % (subDir, n))
         n = n + 1
         getNewsContent(fn, url, 'canvas-atom canvas-text Mb(1.0em) Mb(0)--sm Mt(0.8em)--sm')
   f.close()

def getYahooNews():
   for d in newsDic:
      url = yahooNewsUrl + d
      getYahooNews2(d, url, 'D(ib) Ov(h) Whs(nw) C($c-fuji-grey-l) C($c-fuji-blue-1-c):h Td(n) Fz(16px) Tov(e) Fw(700)')

# 主程式
printNames()
getYahooNews()

# 測試錯誤點
#fn = 'd:\\t.txt'
#getNewsContent(fn, url, 'canvas-atom canvas-text Mb(1.0em) Mb(0)--sm Mt(0.8em)--sm')

input("Press enter to continue......")