1 Star 0 Fork 0

Gitee 极速下载/Scrapling

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
文件
此仓库是为了提升国内下载速度的镜像仓库,每日同步一次。 原始仓库: https://github.com/D4Vinci/Scrapling
克隆/下载
benchmarks.py 4.48 KB
一键复制 编辑 原始数据 按行查看 历史
import functools
import time
import timeit
from statistics import mean
import requests
from autoscraper import AutoScraper
from bs4 import BeautifulSoup
from lxml import etree, html
from mechanicalsoup import StatefulBrowser
from parsel import Selector
from pyquery import PyQuery as pq
from selectolax.parser import HTMLParser
from scrapling import Adaptor
large_html = (
"<html><body>" + '<div class="item">' * 5000 + "</div>" * 5000 + "</body></html>"
)
def benchmark(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
benchmark_name = func.__name__.replace("test_", "").replace("_", " ")
print(f"-> {benchmark_name}", end=" ", flush=True)
# Warm-up phase
timeit.repeat(
lambda: func(*args, **kwargs), number=2, repeat=2, globals=globals()
)
# Measure time (1 run, repeat 100 times, take average)
times = timeit.repeat(
lambda: func(*args, **kwargs),
number=1,
repeat=100,
globals=globals(),
timer=time.process_time,
)
min_time = round(mean(times) * 1000, 2) # Convert to milliseconds
print(f"average execution time: {min_time} ms")
return min_time
return wrapper
@benchmark
def test_lxml():
return [
e.text
for e in etree.fromstring(
large_html,
# Scrapling and Parsel use the same parser inside so this is just to make it fair
parser=html.HTMLParser(recover=True, huge_tree=True),
).cssselect(".item")
]
@benchmark
def test_bs4_lxml():
return [e.text for e in BeautifulSoup(large_html, "lxml").select(".item")]
@benchmark
def test_bs4_html5lib():
return [e.text for e in BeautifulSoup(large_html, "html5lib").select(".item")]
@benchmark
def test_pyquery():
return [e.text() for e in pq(large_html)(".item").items()]
@benchmark
def test_scrapling():
# No need to do `.extract()` like parsel to extract text
# Also, this is faster than `[t.text for t in Adaptor(large_html, auto_match=False).css('.item')]`
# for obvious reasons, of course.
return Adaptor(large_html, auto_match=False).css(".item::text")
@benchmark
def test_parsel():
return Selector(text=large_html).css(".item::text").extract()
@benchmark
def test_mechanicalsoup():
browser = StatefulBrowser()
browser.open_fake_page(large_html)
return [e.text for e in browser.page.select(".item")]
@benchmark
def test_selectolax():
return [node.text() for node in HTMLParser(large_html).css(".item")]
def display(results):
# Sort and display results
sorted_results = sorted(results.items(), key=lambda x: x[1]) # Sort by time
scrapling_time = results["Scrapling"]
print("\nRanked Results (fastest to slowest):")
print(f" i. {'Library tested':<18} | {'avg. time (ms)':<15} | vs Scrapling")
print("-" * 50)
for i, (test_name, test_time) in enumerate(sorted_results, 1):
compare = round(test_time / scrapling_time, 3)
print(f" {i}. {test_name:<18} | {str(test_time):<15} | {compare}")
@benchmark
def test_scrapling_text(request_html):
# Will loop over resulted elements to get text too to make comparison even more fair otherwise Scrapling will be even faster
return [
element.text
for element in Adaptor(request_html, auto_match=False)
.find_by_text("Tipping the Velvet", first_match=True)
.find_similar(ignore_attributes=["title"])
]
@benchmark
def test_autoscraper(request_html):
# autoscraper by default returns elements text
return AutoScraper().build(html=request_html, wanted_list=["Tipping the Velvet"])
if __name__ == "__main__":
print(
" Benchmark: Speed of parsing and retrieving the text content of 5000 nested elements \n"
)
results1 = {
"Raw Lxml": test_lxml(),
"Parsel/Scrapy": test_parsel(),
"Scrapling": test_scrapling(),
"Selectolax": test_selectolax(),
"PyQuery": test_pyquery(),
"BS4 with Lxml": test_bs4_lxml(),
"MechanicalSoup": test_mechanicalsoup(),
"BS4 with html5lib": test_bs4_html5lib(),
}
display(results1)
print("\n" + "=" * 25)
req = requests.get("https://books.toscrape.com/index.html")
print(
" Benchmark: Speed of searching for an element by text content, and retrieving the text of similar elements\n"
)
results2 = {
"Scrapling": test_scrapling_text(req.text),
"AutoScraper": test_autoscraper(req.text),
}
display(results2)
Loading...
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
Python
1
https://gitee.com/mirrors/Scrapling.git
git@gitee.com:mirrors/Scrapling.git
mirrors
Scrapling
Scrapling
dev

搜索帮助