2 Star 0 Fork 0

刘洋 / 2019数据采集与融合

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
该仓库未声明开源许可证文件(LICENSE),使用请关注具体项目描述及其代码上游依赖。
克隆/下载
task2_1.py 3.08 KB
一键复制 编辑 原始数据 按行查看 历史
刘洋 提交于 2021-11-24 09:42 . 作业1和2
import random
import time
import urllib.request
import pymysql
from selenium import webdriver
from selenium.webdriver.common.by import By
class CSDB:
con = pymysql.connect(host="127.0.0.1", port=3306, user="root", passwd="20010109l?y!", db="class",
charset="utf8")
cursor = con.cursor(pymysql.cursors.DictCursor)
def createDB(self):
try:
self.cursor.execute('create table Course(Id varchar (10),cCourse varchar (40),cCollege varchar (20),cSchedule varchar (30),'
'cCourseStatus varchar (30), clmgUrl varchar (255))')
except Exception as err:
print(err)
print(1)
def insert(self,id,course,college,schedule,coursestatus,clmgurl):
try:
self.cursor.execute('insert into Course(Id,cCourse,cCollege,cSchedule,cCourseStatus,clmgUrl) '
'values (%s,%s,%s,%s,%s,%s)', (id, course, college, schedule, coursestatus, clmgurl))
except Exception as err:
print(err)
def closeDB(self):
self.con.commit()
self.con.close()
def download(url,name):
req = urllib.request.Request(url,)
data = urllib.request.urlopen(req, timeout=100)
data = data.read()
fobj = open(r"images/" + str(name) + ".jpg", "wb")
fobj.write(data)
fobj.close()
print("downloaded" + (name) + ".jpg")
db = CSDB()
#db.createDB()
#db.closeDB()
option = webdriver.ChromeOptions()
option.add_experimental_option("detach", True)
driver = webdriver.Chrome(chrome_options=option)
driver.get('https://www.icourse163.org/')
sign = driver.find_element(By.XPATH, '/html/body/div[4]/div[2]/div[1]/div/div/div[1]/div[3]/div[3]/div').click()
time.sleep(10)
find = driver.find_element(By.XPATH, '//*[@id="j-indexNav-bar"]/div/div/div/div/div[7]/div[3]/div/div/a/span')
# 图标被遮挡,导致无法直接用find.click()点击
driver.execute_script("arguments[0].click();", find)
#课程名
course = driver.find_elements(By.XPATH, '//*[@id="j-coursewrap"]/div/div[1]/div/div[1]/a/div[2]/div[1]/div[1]/div/span[2]')
#学校
college = driver.find_elements(By.XPATH, '//*[@id="j-coursewrap"]/div/div[1]/div/div[1]/a/div[2]/div[1]/div[2]/a')
#课时
sche = driver.find_elements(By.XPATH, '//*[@id="j-coursewrap"]/div/div[1]/div/div[1]/a/div[2]/div[2]/div[1]/div[1]/div[1]/a/span')
#状态
status = driver.find_elements(By.XPATH, '/html/body/div[4]/div[2]/div[3]/div/div[1]/div[3]/div/div[2]/div/div/div[2]/div[1]/div[2]/div/div[1]/div/div[1]/a/div[2]/div[2]/div[2]')
#url
url = driver.find_elements(By.XPATH, '/html/body/div[4]/div[2]/div[3]/div/div[1]/div[3]/div/div[2]/div/div/div[2]/div[1]/div[2]/div/div[1]/div/div[1]/a/div[1]/img')
for i in range(len(course)):
db.insert(i+1, course[i].text, college[i].text,sche[i].text,status[i].text, url[i].get_attribute("src"))
db.con.commit()
print(i+1,course[i].text,college[i].text,sche[i].text,status[i].text,url[i].get_attribute("src"))
course1 = course[i].text
url1 = url[i].get_attribute("src")
download(url1,course1)
1
https://gitee.com/liu-yangz/crawl_project.git
git@gitee.com:liu-yangz/crawl_project.git
liu-yangz
crawl_project
2019数据采集与融合
master

搜索帮助