大家好我是图恩，网站前期运行期间有写过一个爬虫去简书爬取前端相关文章，然后每天定时爬取一定数量的文章发布到网站，目的就是为了充实网站内容。
但是前期由于技术不过关导致爬取的内容没有分段，整篇文章就是一段，阅读十分不方便，但是等图恩发现这个问题的时候网站已经运行了差不多一年了，所以之前爬取的内容也就没做处理，但是对爬虫做了该进，该进后的爬虫可以爬取到具有样式的内容了，以下为参考代码。
# -*- coding: UTF-8 -*-\nimport datetime\nimport requests\nimport json\nfrom bs4 import BeautifulSoup\nimport mysql.connector\nimport urllib\nimport time\nimport importlib\nimport sys\nreload(sys)\nsys.setdefaultencoding('utf-8')\nurl = 'https://www.jianshu.com/c/f489ec955505'\n#构造请求头\nheaders = {\n 'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36',\n 'sec-fetch-mode': 'cors',\n 'sec-fetch-site': 'same-origin',\n 'accept': 'application/json',\n 'accept-encoding': 'gzip, deflate, br',\n 'accept-language': 'zh-CN,zh;q=0.9',\n }\nparams = {\n 'order_by': 'added_at',\n 'page': 1\n}\n\nallList = []\ndef getList():\n for it in range(0, 1):\n params['page'] += 1\n time.sleep(1)\n strhtml = requests.get(url, params=params, headers=headers)\n content = json.loads(strhtml.text)\n allList.extend(content['notes'])\n\ndef insertPage():\n list = []\n for item in allList:\n result = {\n 'slug': item['slug'],\n 'title': item['title']\n }\n list.append(result)\n itemList = []\n for item2 in list:\n time.sleep(1)\n html = requests.get('https://www.jianshu.com/p/' + item2['slug'],headers=headers)\n jsonhtml = html.text\n soup = BeautifulSoup(jsonhtml, 'html.parser')\n h1 = soup.find(name ='h1').text\n artitleTemp = soup.find(name ='article')\n print("content---------", artitleTemp)\n if artitleTemp is not None:\n # 获取内容后一定要用str方法转成字符串，否则数据库会报错提示内容不匹配\n artitle = str(artitleTemp)\n result2 = {\n 'title': h1,\n 'content': artitle\n }\n itemList.append(result2)\n print("开始连接数据库")\n conn = mysql.connector.connect(host='localhost', user='root', password='troot',database='test', buffered=True)\n cursor = conn.cursor()\n print("数据库连接成功")\n for sql in itemList:\n title = sql['title']\n sqlcontent = sql['content']\n print (title)\n print ('----------------------即将写入数据库文章内容----------')\n time.sleep(1)\n inserted = "select * from pages where title like '%s'" % title\n cursor.execute(inserted)\n result3 = cursor.fetchone()\n print("result3------")\n if result3 is None:\n date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")\n time_stamp = int(round(time.time() * 1000))\n print (time_stamp)\n print (title)\n if title is not None:\n if sqlcontent is not None:\n s = "insert into pages(title,content) values(%s,%s)"\n cursor.execute(s, (title, sqlcontent))\n s2 = "select id from pages where title = '%s'" % title\n cursor.execute(s2)\n conn.commit()\n\n cursor.close()\na = 1\nif (a > 0):\n time.sleep(1)\n print ('---------start task-----------------')\n getList()\n insertPage()特别注意的是获取内容后如果要写入数据库一定要先用str方法转字符串才能正常写入，否则会报错。