from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import urllib.request
import threading
import sqlite3
import os
import datetime
from selenium.webdriver.common.keys import Keys
import time


class JD:
    header = {
        "User-Agent": "Mozilla/5.0(Windows;U;Windows NT 6.0 x64;en-US;rv:1.9pre)Gecko/2008072531 Minefield/3.0.2pre"
    }
    # 图片存储的位置
    imagepath = "download"

    def startUp(self, url, key):
        chrome_options = Options()
        chrome_options.add_argument("——headless")
        chrome_options.add_argument("——disable-gpu")
        self.driver = webdriver.Chrome(chrome_options=chrome_options)
        self.threads = []
        self.No = 0
        self.imgNo = 0
        try:
            self.con = sqlite3.connect("phones.db")

            self.cursor = self.con.cursor()
            try:
                self.cursor.execute("drop table phones")

            except:
                pass
            try:
                sql = "create table phones(mNo varchar(32) primary key,mMark varchar(256),mPrice varchar(32),mNote varchar(1024),mFile varchar(256))"
                self.cursor.execute(sql)
            except:
                pass
        except Exception as err:
            print(err)
        try:
            if not os.path.exists(JD.imagepath):
                os.mkdir(JD.imagepath)
            images = os.listdir(JD.imagepath)
            for image in images:
                s = os.path.join(JD.imagepath, image)
                os.remove(s)
        except Exception as err:
            print(err)
        self.driver.get(url)
        keyinput = self.driver.find_element_by_id("key")
        keyinput.send_keys(key)
        keyinput.send_keys(Keys.ENTER)

    def closeUp(self):
        try:
            self.con.commit()
            self.con.close()
            self.driver.close()
        except Exception as err:
            print(err)

    def insertDB(self, mNo, mMark, mPrice, mNote, mFile):
        try:
            sql = "insert into phones (mNo,mMark,mPrice,mNote,mFile) values (?,?,?,?,?)"

            self.cursor.execute(sql, (mNo, mMark, mPrice, mNote, mFile))
        except Exception as err:
            print(err)

    def showDB(self):
        try:
            con = sqlite3.connect("phones.db")
            cursor = con.cursor()
            print("%-8s%-16s%-8s%-16s%s" % ("No", "Mark", "Price", "Image", "Note"))
            cursor.execute("select mNO,mMark,mPrice,mFile,mNote from phones order by mNo")
            rows = cursor.fetchall()
            for row in rows:
                print("%-8s%-16s%-8s%-16s%s" % (row[0], row[1], row[2], row[3], row[4]))

            con.close()
        except Exception as err:
            print(err)

    def downloadDB(self, src1, src2, mFile):
        data = None
        if src1:
            try:
                req = urllib.request.Request(src1, headers=JD.header)
                resp = urllib.request.urlopen(req, timeout=100)
                data = resp.read()
            except:
                pass
        if not data and src2:
            try:
                req = urllib.request.Request(src2, headers=JD.header)
                resp = urllib.request.urlopen(req, timeout=100)
                data = resp.read()
            except:
                pass
        if data:
            print("download begin!", mFile)
            fobj = open(JD.imagepath + "\\" + mFile, "wb")
            fobj.write(data)
            fobj.close()
            print("download finish!", mFile)

    def processJD(self):
        time.sleep(10)
        try:
            print(self.driver.current_url)
            lis = self.driver.find_elements_by_xpath("//div[@id='J_goodsList']//li[@class='gl-item']")
            time.sleep(1)
            for li in lis:
                time.sleep(1)
                try:
                    src1 = li.find_element_by_xpath(".//div[@class='p-img']//a//img").get_attribute("src")
                    time.sleep(1)
                except:
                    src1 = ""
                try:
                    src2 = li.find_element_by_xpath(".//div[@class='p-img']//a//img").get_attribute("data-lazy-img")
                    time.sleep(1)
                except:
                    src2 = ""
                try:
                    price = li.find_element_by_xpath(".//div[@class='p-price']//i").text
                    time.sleep(1)
                except:
                    price = "0"

                note = li.find_element_by_xpath(".//div[@class='p-name p-name-type-2']//em").text
                mark = note.split(" ")[0]
                mark = mark.replace("爱心东东\n", "")
                mark = mark.replace(",", "")
                note = note.replace("爱心东东\n", "")
                note = note.replace(",", "")
                time.sleep(1)
                self.No = self.No + 1
                no = str(self.No)
                while len(no) < 6:
                    no = "0" + no
                print(no, mark, price)
                if src1:
                    src1 = urllib.request.urljoin(self.driver.current_url, src1)
                    p = src1.rfind(".")
                    mFile = no + src1[p:]
                elif src2:
                    src2 = urllib.request.urljoin(self.driver.current_url, src2)
                    p = src2.rfind(".")
                    mFile = no + src2[p:]
                if src1 or src2:
                    T = threading.Thread(target=self.downloadDB, args=(src1, src2, mFile))
                    T.setDaemon(False)
                    T.start()
                    self.threads.append(T)
                else:
                    mFile = ""
                self.insertDB(no, mark, price, note, mFile)

        except Exception as err:
            print(err)

    def executeJD(self, url, key):
        starttime = datetime.datetime.now()
        print("starting!")
        self.startUp(url, key)
        print("processing!")
        self.processJD()
        print("closing!")
        self.closeUp()
        for t in self.threads:
            t.join()
        print("complete!")
        endtime = datetime.datetime.now()
        elapsed = (endtime - starttime).seconds
        print("Total", elapsed, "seconds elasped")


url = "https://www.jd.com"
spider = JD()
while True:
    print("1.爬取")
    print("2.显示")
    print("3.退出")
    s = input("请选择（1，2，3）；")
    if s == "1":
        JD().executeJD(url, '手机')
        continue
    elif s == "2":
        JD().showDB()
        continue
    elif s == "3":
        break