import os
import sys

from selenium import webdriver
from pandas.io import json

from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.wait import WebDriverWait

username = "your username"
password = "your password"

def parse(gameName, totalPage):
    file_object = open('%s.sql' % gameName, 'wb+')
    url = 'https://buff.163.com/api/market/goods/all?game=%s&page_num=%s&page_size=70'
    page = 0
    while page < totalPage:
        page = page + 1
        driver.get(url % (gameName, page))
        #time.sleep(1)
        # result = json.loads(driver.find_element_by_tag_name('pre').text)
        element = WebDriverWait(driver, 10).until(
            expected_conditions.presence_of_element_located((By.TAG_NAME, "pre"))
        )
        result = json.loads(element.text)
        goods = result['data']['items']
        for g in goods:
            line = "insert into goods (`goods_name`, `market_hash_name`, `game_name`, `image`, `steam_price`,`steam_price_cny`) values ( '%s', '%s', '%s', '%s', %s, %s);" % (
                g['name'].replace("'", "\\'"), g['market_hash_name'].replace("'", "\\'"), g['game'].replace("'", "\\'"),
                g['goods_info']['icon_url'].replace("'", "\\'"), g['goods_info']['steam_price'],
                g['goods_info']['steam_price_cny'])
            print(line);
            file_object.write(line.encode())
            file_object.write('\n'.encode())
            tags = g['goods_info']['info']['tags']
            for key in tags.keys():
                line = "insert into goods_tag (`goods_id`, `game_name`, `category`, `internal_name`, `localized_name`) values ((%s), '%s', '%s', '%s', '%s');" % (
                    "select id from goods where game_name = '%s' and market_hash_name = '%s'" % (gameName, g['market_hash_name'].replace("'",
                                                                                                             "\\'")),
                    gameName, tags[key]['category'].replace("'", "\\'"),
                    tags[key]['internal_name'].replace("'", "\\'"), tags[key]['localized_name'].replace("'", "\\'"))
                print(line)
                file_object.write(line.encode())
                file_object.write('\n'.encode())
    file_object.close()


if __name__ == '__main__':
    driver = webdriver.Chrome()
    driver.get('https://buff.163.com')
    element = WebDriverWait(driver, sys.maxsize).until(
        expected_conditions.presence_of_element_located((By.XPATH, "//div[@class='nav nav_entries']/ul/li[1]/a"))
    )
    if os.path.exists("cookies"):
        file_object = open("cookies", 'r')
        for line in file_object.readlines():
            line = line.replace('\n', '')
            driver.add_cookie(json.loads(line))
        file_object.close()
        driver.get('https://buff.163.com')
        element = WebDriverWait(driver, sys.maxsize).until(
            expected_conditions.presence_of_element_located((By.ID, "navbar-user-name"))
        )
    else:
        element.click()
        element = WebDriverWait(driver, sys.maxsize).until(
            expected_conditions.presence_of_element_located((By.XPATH, "//div[@id='j_login']/iframe"))
        )
        driver.switch_to.frame(element)
        element = WebDriverWait(driver, sys.maxsize).until(
            expected_conditions.presence_of_element_located((By.CLASS_NAME, "tab0"))
        )
        element.click()
        driver.find_element_by_id("phoneipt").send_keys(username)
        driver.find_element_by_xpath("//input[@name='email'][@type='password']").send_keys(password)
        driver.switch_to.default_content()
        element = WebDriverWait(driver, sys.maxsize).until(
            expected_conditions.presence_of_element_located((By.ID, "navbar-user-name"))
        )

        cookies = driver.get_cookies()
        file_object = open("cookies", 'w')
        for line in cookies:
            if 'expiry' in line:
                del line['expiry']
            file_object.write(json.dumps(line) + '\n')
        file_object.close()

    gameName = 'csgo'
    totalPage = 240
    parse(gameName, totalPage)

    gameName = 'dota2'
    totalPage = 509
    parse(gameName, totalPage)

    gameName = 'h1z1'
    totalPage = 14
    parse(gameName, totalPage)

    gameName = 'pubg'
    totalPage = 4
    parse(gameName, totalPage)