Chapter13-ImageProcessing.ipynb · bob.wang/python-scraping

加入 Gitee

与超过 1200万开发者一起发现、参与优秀开源项目，私有仓库也完全免费：）

该仓库未声明开源许可证文件（LICENSE），使用请关注具体项目描述及其代码上游依赖。

克隆/下载

Chapter13-ImageProcessing.ipynb 23.80 KB

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'level': [1, 2, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 5, 5, 5, 5, 5, 5], 'page_num': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'block_num': [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'par_num': [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'line_num': [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2], 'word_num': [0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 1, 2, 3, 4, 5, 6], 'left': [0, 23, 23, 23, 23, 76, 99, 164, 215, 295, 317, 378, 425, 467, 500, 555, 23, 23, 140, 198, 239, 304, 407], 'top': [0, 26, 26, 26, 26, 26, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 54, 55, 55, 59, 59, 55, 54], 'width': [600, 555, 555, 555, 44, 15, 57, 42, 70, 15, 52, 40, 32, 24, 45, 23, 513, 107, 49, 32, 57, 92, 129], 'height': [103, 51, 51, 22, 17, 17, 13, 21, 17, 17, 21, 17, 17, 17, 17, 22, 23, 17, 17, 13, 13, 22, 23], 'conf': [-1, -1, -1, -1, 86, 86, 86, 73, 82, 95, 73, 80, 95, 85, 87, 85, -1, 82, 91, 86, 86, 86, 65], 'text': ['', '', '', '', 'This', 'is', 'some', 'text,', 'written', 'in', 'Arial,', 'that', 'will', 'be', 'read', 'by', '', 'Tesseract.', 'Here', 'are', 'some', 'symbols:', '!@#$%\"&‘()']}\n",
      "b'This is some text, written in Arial, that will be read by\\nTesseract. Here are some symbols: !@#$%\"&\\xe2\\x80\\x98()\\n\\n'\n"
     ]
    }
   ],
   "source": [
    "from PIL import Image\n",
    "import pytesseract\n",
    "from pytesseract import Output\n",
    "\n",
    "print(pytesseract.image_to_data(Image.open('files/'\n",
    "    'textOriginal.png'),\n",
    "    output_type=Output.DICT))\n",
    "print(pytesseract.image_to_string(Image.open('files/textOriginal.png'), output_type=Output.BYTES))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2803508502\n",
      "100.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "numbers = [100, 102, 98, 97, 103]\n",
    "print(np.std(numbers))\n",
    "print(np.mean(numbers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ThIS IS some (ext written In Anal, that will be read by\n",
      "Tesseract Here are some symbols: l@#$%\"&'()\n"
     ]
    }
   ],
   "source": [
    "from PIL import Image\n",
    "import pytesseract\n",
    "\n",
    "def cleanFile(filePath, newFilePath):\n",
    "    image = Image.open(filePath)\n",
    "\n",
    "    #Set a threshold value for the image, and save\n",
    "    image = image.point(lambda x: 0 if x<143 else 255)\n",
    "    image.save(newFilePath)\n",
    "    return image\n",
    "\n",
    "image = cleanFile('files/textBad.png', 'files/textCleaned.png')\n",
    "\n",
    "#call tesseract to do OCR on the newly created image\n",
    "print(pytesseract.image_to_string(image))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is some text, written in Arial, that will be read by\n",
      "Tesseract. Here are some symbols: !@#$%\"&‘()\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from PIL import Image\n",
    "import subprocess\n",
    "\n",
    "def cleanFile(filePath, newFilePath):\n",
    "    image = Image.open(filePath)\n",
    "\n",
    "    #Set a threshold value for the image, and save\n",
    "    image = image.point(lambda x: 0 if x<135 else 255)\n",
    "    image.save(newFilePath)\n",
    "\n",
    "    #call tesseract to do OCR on the newly created image\n",
    "    subprocess.call(['tesseract', newFilePath, 'output'])\n",
    "    \n",
    "    #Open and read the resulting data file\n",
    "    outputFile = open('output.txt', 'r')\n",
    "    print(outputFile.read())\n",
    "    outputFile.close()\n",
    "\n",
    "cleanFile('files/textOriginal.png', 'files/text_2.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "threshold: 80, confidence: 61.8333333333 numChars 18\n",
      "threshold: 85, confidence: 64.9130434783 numChars 23\n",
      "threshold: 90, confidence: 62.2564102564 numChars 39\n",
      "threshold: 95, confidence: 64.5135135135 numChars 37\n",
      "threshold: 100, confidence: 60.7878787879 numChars 66\n",
      "threshold: 105, confidence: 61.9078947368 numChars 76\n",
      "threshold: 110, confidence: 64.6329113924 numChars 79\n",
      "threshold: 115, confidence: 69.7397260274 numChars 73\n",
      "threshold: 120, confidence: 72.9078947368 numChars 76\n",
      "threshold: 125, confidence: 73.582278481 numChars 79\n",
      "threshold: 130, confidence: 75.6708860759 numChars 79\n",
      "threshold: 135, confidence: 76.8292682927 numChars 82\n",
      "threshold: 140, confidence: 72.1686746988 numChars 83\n",
      "threshold: 145, confidence: 75.5662650602 numChars 83\n",
      "threshold: 150, confidence: 77.5443037975 numChars 79\n",
      "threshold: 155, confidence: 79.1066666667 numChars 75\n",
      "threshold: 160, confidence: 78.4666666667 numChars 75\n",
      "threshold: 165, confidence: 80.1428571429 numChars 70\n",
      "threshold: 170, confidence: 78.4285714286 numChars 70\n",
      "threshold: 175, confidence: 76.3731343284 numChars 67\n",
      "threshold: 180, confidence: 76.7575757576 numChars 66\n",
      "threshold: 185, confidence: 79.4920634921 numChars 63\n",
      "threshold: 190, confidence: 76.0793650794 numChars 63\n",
      "threshold: 195, confidence: 70.6153846154 numChars 65\n"
     ]
    }
   ],
   "source": [
    "import pytesseract\n",
    "from pytesseract import Output\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "\n",
    "def cleanFile(filePath, threshold):\n",
    "    image = Image.open(filePath)\n",
    "    #Set a threshold value for the image, and save\n",
    "    image = image.point(lambda x: 0 if x<threshold else 255)\n",
    "    return image\n",
    "\n",
    "def getConfidence(image):\n",
    "    data = pytesseract.image_to_data(image, output_type=Output.DICT)\n",
    "    text = data['text']\n",
    "    confidences = []\n",
    "    numChars = []\n",
    "    \n",
    "    for i in range(len(text)):\n",
    "        if data['conf'][i] > -1:\n",
    "            confidences.append(data['conf'][i])\n",
    "            numChars.append(len(text[i]))\n",
    "            \n",
    "    return np.average(confidences, weights=numChars), sum(numChars)\n",
    "    \n",
    "filePath = 'files/textBad.png'\n",
    "\n",
    "start = 80\n",
    "step = 5\n",
    "end = 200\n",
    "\n",
    "for threshold in range(start, end, step):\n",
    "    image = cleanFile(filePath, threshold)\n",
    "    scores = getConfidence(image)\n",
    "    print('threshold: {}, confidence: {}, numChars '.format(str(threshold), str(scores[0], str(scores[1]))))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found image: https://d1b14unh5d6w7g.cloudfront.net/1427027277.01.S003.LXXXXXXX.jpg?Expires=1517813908&Signature=Esov1bmGrixpRerzigTNris1X5NxXhYh0M0h2Vkzho4KQmOMUetNtvC4Q2NcPHEJarhgoTEGgQk33rK/NlL6g7bGd7kEuqDtTVeDnfVFv6N45Fh5zjinOfWRfbaWlbKi0y51WDRPROG/ITUSztyovyxhtTtiAiviUocae/fO4Ic=&Key-Pair-Id=APKAIUO27P366FGALUMQ\n",
      "rammed Mam 3‘\n",
      "\n",
      "Copyright © 2007\n",
      "\n",
      "The text in this book has been formatted\n",
      "and typeset to make reading easier and\n",
      "more enjoyable. Spelling and punctuation\n",
      "have been modernized and standardized,\n",
      "and hyphens have been avoided when\n",
      "possible. Obvious typographical errors\n",
      "have been corrected. In addition, the text\n",
      "has been formatted to the specifications\n",
      "indicated on the titIe page. The original\n",
      "text is out of copyright. The edited text in\n",
      "this edition and the formatting are the\n",
      "copyright of Objective Systems Pty Ltd\n",
      "ACN 085119 953.\n",
      "» 1m\n",
      "\n",
      "LaDY'M‘Hed mew\n",
      "Found image: https://d1b14unh5d6w7g.cloudfront.net/1427027277.01.S006.LXXXXXXX.jpg?Expires=1517813908&Signature=hHkVlIbhqEXCF0eEwjT+XFQHLowD/B0emBi4Xe7ThHptbtNnPqZqeM1KztJlHBD7Xf1V16GWe4APyf9yYWHSS6QQ3isvEakHB5Me/NBS8fcXj718gDbFEl4v9VgoLpxjsRy8g3c/Xay87LlMhYlgqK634w4cG5hrd9Gcbi1lDuE=&Key-Pair-Id=APKAIUO27P366FGALUMQ\n",
      "CuDYnlNed Manna:\n",
      "\n",
      "TABLE OF CONTENTS\n",
      "\n",
      "Chapter} 1\n",
      "Chapter 11 17\n",
      "Chapter 111 31\n",
      "Chapter IV 43\n",
      "Chapter v 55\n",
      "Chapter VI 53\n",
      "Chapter V11 59\n",
      "Chapter vm 77\n",
      "chapter IX 91\n",
      "Chapter X 97\n",
      "Chapter X] 101\n",
      "Chapter xu 107\n",
      "\n",
      "Canyrmhted mum\n",
      "Found image: https://d1b14unh5d6w7g.cloudfront.net/1427027277.01.S003.LXXXXXXX.jpg?Expires=1517813908&Signature=Esov1bmGrixpRerzigTNris1X5NxXhYh0M0h2Vkzho4KQmOMUetNtvC4Q2NcPHEJarhgoTEGgQk33rK/NlL6g7bGd7kEuqDtTVeDnfVFv6N45Fh5zjinOfWRfbaWlbKi0y51WDRPROG/ITUSztyovyxhtTtiAiviUocae/fO4Ic=&Key-Pair-Id=APKAIUO27P366FGALUMQ\n",
      "Found image: https://d1b14unh5d6w7g.cloudfront.net/1427027277.01.S006.LXXXXXXX.jpg?Expires=1517813908&Signature=hHkVlIbhqEXCF0eEwjT+XFQHLowD/B0emBi4Xe7ThHptbtNnPqZqeM1KztJlHBD7Xf1V16GWe4APyf9yYWHSS6QQ3isvEakHB5Me/NBS8fcXj718gDbFEl4v9VgoLpxjsRy8g3c/Xay87LlMhYlgqK634w4cG5hrd9Gcbi1lDuE=&Key-Pair-Id=APKAIUO27P366FGALUMQ\n",
      "Found image: https://d1b14unh5d6w7g.cloudfront.net/1427027277.01.S007.LXXXXXXX.jpg?Expires=1517813916&Signature=cC2WHvp6Zb3YMWwf0XmPG2xhhna9IDpIWTZog3643bMtqb5krx+PPwGBIS5Z2+VD/0BoxQYW7TuUrQ2QA1ouuzFHBrMMl13iFKOPdeOcNuTFvqsNx2zZGrUXLvmrafaiEHiF097ruUQpJT2dvhQk3kAPuYWkPxhNKLRyuFjJ8H8=&Key-Pair-Id=APKAIUO27P366FGALUMQ\n",
      "Conyﬂihud Mmml\n",
      "\n",
      "Copyﬂghud anhl\n",
      "Found image: https://d1b14unh5d6w7g.cloudfront.net/1427027277.01.S008.LXXXXXXX.jpg?Expires=1517813908&Signature=HuGS9+TRIYXlx5jvFLtR/LGK1ThQSTiurvNJjd2sfHMPRdkhCb22LJM+baLu5nYZtDQguGgY9kdgAROBKYs+tqas89iVjs1bVNG6m4kMctJHokK5yfTVe6szwhf11+yY7JnhFn8qcOMmL7IbyBvV3s/gQIRxWb+j4lILGIoBj/M=&Key-Pair-Id=APKAIUO27P366FGALUMQ\n",
      "canyngmed Malena»\n",
      "\n",
      "Chapter I\n",
      "\n",
      "During an Interval In the Melvmskl trial In the large\n",
      "building of the Law Courts the members and public\n",
      "prosecutor met in [van Egorowch Shebek‘s private\n",
      "room, where the conversation turned on the celebrated\n",
      "Krasovski case. Fedor Vasillevich warmly maintained\n",
      "that it was not subject to their jurisdiction, Ivan\n",
      "Egorovich maintained the contrary, while Peter\n",
      "ivanowch, not havmg entered into the discussmn at\n",
      "the start, took no part in it but looked through the\n",
      "Gazette which had Just been handed in.\n",
      "\n",
      "“Gentlemen,” he said, “Ivan Ilych has died!\"\n",
      "“You don't say 50!”\n",
      "\n",
      "“Here, read it yourself,\" repiied Peter Ivanovich,\n",
      "handing Fedor VaSIllevich the paper still damp from\n",
      "the press, Surrounded by a black border were the\n",
      "words: “Praskovya Fedorovna Golovina, with profound\n",
      "sorrow, informs relatives and friends of the demise\n",
      "of her beloved husband Ivan Ilych Goiovin, Member\n",
      "of the Court of Justice, which occurred on February\n",
      "the 4th of this year 1882. The funeral will take place\n",
      "on Friday at one o'clock in the afternoon.“\n",
      "\n",
      "Ivan Iiych had been a colleague of the gentlemen\n",
      "present and was liked by them ali. He had been ill for\n",
      "\n",
      "annnhted Malenai\n",
      "Found image: https://d1b14unh5d6w7g.cloudfront.net/1427027277.01.S006.LXXXXXXX.jpg?Expires=1517813908&Signature=hHkVlIbhqEXCF0eEwjT+XFQHLowD/B0emBi4Xe7ThHptbtNnPqZqeM1KztJlHBD7Xf1V16GWe4APyf9yYWHSS6QQ3isvEakHB5Me/NBS8fcXj718gDbFEl4v9VgoLpxjsRy8g3c/Xay87LlMhYlgqK634w4cG5hrd9Gcbi1lDuE=&Key-Pair-Id=APKAIUO27P366FGALUMQ\n",
      "Found image: https://d1b14unh5d6w7g.cloudfront.net/1427027277.01.S007.LXXXXXXX.jpg?Expires=1517813916&Signature=cC2WHvp6Zb3YMWwf0XmPG2xhhna9IDpIWTZog3643bMtqb5krx+PPwGBIS5Z2+VD/0BoxQYW7TuUrQ2QA1ouuzFHBrMMl13iFKOPdeOcNuTFvqsNx2zZGrUXLvmrafaiEHiF097ruUQpJT2dvhQk3kAPuYWkPxhNKLRyuFjJ8H8=&Key-Pair-Id=APKAIUO27P366FGALUMQ\n",
      "Found image: https://d1b14unh5d6w7g.cloudfront.net/1427027277.01.S008.LXXXXXXX.jpg?Expires=1517813908&Signature=HuGS9+TRIYXlx5jvFLtR/LGK1ThQSTiurvNJjd2sfHMPRdkhCb22LJM+baLu5nYZtDQguGgY9kdgAROBKYs+tqas89iVjs1bVNG6m4kMctJHokK5yfTVe6szwhf11+yY7JnhFn8qcOMmL7IbyBvV3s/gQIRxWb+j4lILGIoBj/M=&Key-Pair-Id=APKAIUO27P366FGALUMQ\n",
      "Found image: https://d1b14unh5d6w7g.cloudfront.net/1427027277.01.S009.LXXXXXXX.jpg?Expires=1517813916&Signature=Z9uc1Ch9ZhdwrfxfYHfEuPMRBNuTpNZeCC5M6PX5zf9IN75gBvITca59UVgXzCx0vgszKfu5Td7jp4lWobt6b66UBpmbOP/VPHfTtjlkxAHCF8RXA3ilXK+t8Ss8J8GD7fTD/KjdIin+APbwocIH19VlczSLvpL2ynq1Wd7mW/U=&Key-Pair-Id=APKAIUO27P366FGALUMQ\n",
      "canyngmed Material\n",
      "\n",
      "some weeks with an illness said to be incurable.\n",
      "His past had been kept open for him, but there\n",
      "had been conjectures that in case of his death\n",
      "Alexeev might receive his appointment, and that\n",
      "either Vinnikov or Shtabel would succeed Alexeev.\n",
      "So on receiving the news of Ivan llych's death the\n",
      "first thought of each of the gentlemen in that prie\n",
      "vate room was of the changes and promotions it\n",
      "might occasion among themselves or theiracquaine\n",
      "tances.\n",
      "\n",
      "“I shall be sure to get Shtabel's place or Vin‘\n",
      "nikov's,\" thought FedorVasilievich. “I was promised\n",
      "that long ago, and the promotion means an extra\n",
      "eight hundred rubles a year for me besides the\n",
      "allowance.”\n",
      "\n",
      "“Now I must apply for my brother-in-Iaw’s transfer\n",
      "from Kaluga,\" thought Peter Ivanovich. “My Wife\n",
      "will be very glad, and then she won't be able to\n",
      "say that I never do anything for her relatiuns.”\n",
      "“I thought he would never leave his bed again,\"\n",
      "said Peter Ivanovich aloud. “It‘s very sad.”\n",
      "\n",
      "“But what really was the matter with him?\"\n",
      "\n",
      "“The doctors couldn't say - at least they could,\n",
      "but each of them said something different. When\n",
      "last I saw him I though he was getting better.\"\n",
      "\n",
      "CnDYnnhled Material\n",
      "Found image: https://d1b14unh5d6w7g.cloudfront.net/1427027277.01.S007.LXXXXXXX.jpg?Expires=1517813916&Signature=cC2WHvp6Zb3YMWwf0XmPG2xhhna9IDpIWTZog3643bMtqb5krx+PPwGBIS5Z2+VD/0BoxQYW7TuUrQ2QA1ouuzFHBrMMl13iFKOPdeOcNuTFvqsNx2zZGrUXLvmrafaiEHiF097ruUQpJT2dvhQk3kAPuYWkPxhNKLRyuFjJ8H8=&Key-Pair-Id=APKAIUO27P366FGALUMQ\n",
      "Found image: https://d1b14unh5d6w7g.cloudfront.net/1427027277.01.S008.LXXXXXXX.jpg?Expires=1517813908&Signature=HuGS9+TRIYXlx5jvFLtR/LGK1ThQSTiurvNJjd2sfHMPRdkhCb22LJM+baLu5nYZtDQguGgY9kdgAROBKYs+tqas89iVjs1bVNG6m4kMctJHokK5yfTVe6szwhf11+yY7JnhFn8qcOMmL7IbyBvV3s/gQIRxWb+j4lILGIoBj/M=&Key-Pair-Id=APKAIUO27P366FGALUMQ\n",
      "Found image: https://d1b14unh5d6w7g.cloudfront.net/1427027277.01.S009.LXXXXXXX.jpg?Expires=1517813916&Signature=Z9uc1Ch9ZhdwrfxfYHfEuPMRBNuTpNZeCC5M6PX5zf9IN75gBvITca59UVgXzCx0vgszKfu5Td7jp4lWobt6b66UBpmbOP/VPHfTtjlkxAHCF8RXA3ilXK+t8Ss8J8GD7fTD/KjdIin+APbwocIH19VlczSLvpL2ynq1Wd7mW/U=&Key-Pair-Id=APKAIUO27P366FGALUMQ\n",
      "Found image: https://d1b14unh5d6w7g.cloudfront.net/1427027277.01.S00A.LXXXXXXX.jpg?Expires=1517813916&Signature=X0hJbtaKDIHCfhjTcLIt80MY0p6DxRyq+vPbVRcvxuvu8pRrOZdysPayP0gZQlKmh1R+vPCs46euKSsN8dRmfBUSWZbox303sPZQH48shcFHPAgxhoG9Z+C47sCPx++fjA+dkBkBwKoYKAo2QnBEIzkyRZusli0wT7Iz3sedt0E=&Key-Pair-Id=APKAIUO27P366FGALUMQ\n",
      "canymmeu Malena»\n",
      "\n",
      "3\n",
      "\n",
      "“And I haven't been to see him since the holidays.\n",
      "I always meant to go.\"\n",
      "\n",
      "“Had he any property?”\n",
      "\n",
      "“I think his wife had a little — but something quiet\n",
      "trifling.\"\n",
      "\n",
      "“We shall have to go to see her, but they live so\n",
      "terribly far away.\"\n",
      "\n",
      "“Far away from you, you mean. Everything's far\n",
      "away from your place.\"\n",
      "\n",
      "“You see, he never can forgive my living on the\n",
      "other side of the river,\" said Peter Ivanovich,\n",
      "smiling at Shebek. Then, still talking of the dis\n",
      "tances between different parts of the city, they\n",
      "returned to the Court.\n",
      "\n",
      "Besides considerations as to the possible transfers\n",
      "and promotions likely to result from Ivan Ilych’s\n",
      "death, the mere fact of the death of a near acquain-\n",
      "tance aroused, as usual, in all who heard of it the\n",
      "complacent feeﬁng that, “it is he who is dead and\n",
      "not 1.\"\n",
      "\n",
      "Each one thought or felt, “Well, he's dead but I’m\n",
      "alive!” But the more intimate of Ivan Ilych's ac-\n",
      "quaintances, his so-called friends, could not help\n",
      "\n",
      "annnmea Mum\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-5e973ceda051>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     23\u001b[0m     \u001b[0;31m# While the right arrow is available for clicking, turn through pages\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[0mdriver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_element_by_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'sitbReaderRightPageTurner'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclick\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m     \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m     \u001b[0;31m# Get any new pages that have loaded (multiple pages can load at once,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m     \u001b[0;31m# but duplicates will not be added to a set)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import time\n",
    "from urllib.request import urlretrieve\n",
    "from PIL import Image\n",
    "import pytesseract\n",
    "from selenium import webdriver\n",
    "from PIL import Image\n",
    "\n",
    "# Create new Selenium driver\n",
    "driver = webdriver.Chrome(executable_path='drivers/chromedriver/chromedriver')\n",
    "\n",
    "driver.get(\n",
    "    'https://www.amazon.com/Death-Ivan-Ilyich-Nikolayevich-Tolstoy/dp/1427027277')\n",
    "time.sleep(2)\n",
    "\n",
    "# Click on the book preview button\n",
    "driver.find_element_by_id('imgBlkFront').click()\n",
    "imageList = []\n",
    "\n",
    "# Wait for the page to load\n",
    "time.sleep(5)\n",
    "\n",
    "while 'pointer' in driver.find_element_by_id('sitbReaderRightPageTurner').get_attribute('style'):\n",
    "    # While the right arrow is available for clicking, turn through pages\n",
    "    driver.find_element_by_id('sitbReaderRightPageTurner').click()\n",
    "    time.sleep(2)\n",
    "    # Get any new pages that have loaded (multiple pages can load at once,\n",
    "    # but duplicates will not be added to a set)\n",
    "    pages = driver.find_elements_by_xpath(\n",
    "        '//div[@class=\\'pageImage\\']/div/img')\n",
    "    if not len(pages):\n",
    "        print('No pages found')\n",
    "    for page in pages:\n",
    "        image = page.get_attribute('src')\n",
    "        print('Found image: {}'.format(image))\n",
    "        if image not in imageList:\n",
    "            urlretrieve(image, 'page.jpg')\n",
    "            imageList.append(image)\n",
    "            print(pytesseract.image_to_string(Image.open('page.jpg')))\n",
    "\n",
    "driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Captcha solution attempt: \n",
      "There was a problem reading the CAPTCHA correctly!\n"
     ]
    }
   ],
   "source": [
    "from urllib.request import urlretrieve\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import subprocess\n",
    "import requests\n",
    "from PIL import Image\n",
    "from PIL import ImageOps\n",
    "\n",
    "def cleanImage(imagePath):\n",
    "    image = Image.open(imagePath)\n",
    "    image = image.point(lambda x: 0 if x<143 else 255)\n",
    "    borderImage = ImageOps.expand(image,border=20,fill='white')\n",
    "    borderImage.save(imagePath)\n",
    "\n",
    "html = urlopen('http://www.pythonscraping.com/humans-only')\n",
    "bs = BeautifulSoup(html, 'html.parser')\n",
    "#Gather prepopulated form values\n",
    "imageLocation = bs.find('img', {'title': 'Image CAPTCHA'})['src']\n",
    "formBuildId = bs.find('input', {'name':'form_build_id'})['value']\n",
    "captchaSid = bs.find('input', {'name':'captcha_sid'})['value']\n",
    "captchaToken = bs.find('input', {'name':'captcha_token'})['value']\n",
    "\n",
    "captchaUrl = 'http://pythonscraping.com'+imageLocation\n",
    "urlretrieve(captchaUrl, 'captcha.jpg')\n",
    "cleanImage('captcha.jpg')\n",
    "p = subprocess.Popen(['tesseract', 'captcha.jpg', 'captcha'], stdout=\n",
    "    subprocess.PIPE,stderr=subprocess.PIPE)\n",
    "p.wait()\n",
    "f = open('captcha.txt', 'r')\n",
    "\n",
    "#Clean any whitespace characters\n",
    "captchaResponse = f.read().replace(' ', '').replace('\\n', '')\n",
    "print('Captcha solution attempt: '+captchaResponse)\n",
    "\n",
    "if len(captchaResponse) == 5:\n",
    "    params = {'captcha_token':captchaToken, 'captcha_sid':captchaSid,   \n",
    "              'form_id':'comment_node_page_form', 'form_build_id': formBuildId, \n",
    "              'captcha_response':captchaResponse, 'name':'Ryan Mitchell', \n",
    "              'subject': 'I come to seek the Grail', \n",
    "              'comment_body[und][0][value]': \n",
    "               '...and I am definitely not a bot'}\n",
    "    r = requests.post('http://www.pythonscraping.com/comment/reply/10', \n",
    "                          data=params)\n",
    "    responseObj = BeautifulSoup(r.text, 'html.parser')\n",
    "    if responseObj.find('div', {'class':'messages'}) is not None:\n",
    "        print(responseObj.find('div', {'class':'messages'}).get_text())\n",
    "else:\n",
    "    print('There was a problem reading the CAPTCHA correctly!')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

一键复制编辑原始数据按行查看历史

提交于 2018-11-05 08:26 . More cleanup

马建仓 AI 助手

尝试更多

代码解读

代码找茬

代码优化

https://gitee.com/bobwang95/python-scraping.git

git@gitee.com:bobwang95/python-scraping.git

bobwang95

python-scraping

master

bob.wang / python-scraping

简介

发行版

贡献者

近期动态

bob.wang / python-scraping .gitee-modal { width: 500px !important; }

简介

发行版

贡献者

近期动态

搜索帮助

bob.wang / python-scraping