以小米手环7为例,分别爬取小米手环7用户评价中的好评、中评、差评
使用工具:PyCharm Community
需要python库:requests
安装方法:File-->Settings-->Project -->Python Interpreter
代码如下:
好评:
import requests if __name__ == "__main__": #爬取好评100页 for page in range(0,100): url = 'https://club.jd.com/comment/productPageComments.action' param = { 'productId': '100039939514', 'score': '3', #好评score 'sortType': '5', 'page': page, 'pageSize': '10', 'isShadowSku': '0', 'fold': '1', } #UA伪装 headers = { 'cookie': 'shshshfpa=353c3350-9f6e-c6e4-75c2-e45fb0638a20-1677122793; shshshfpb=cYvrYbQje1MA2t7vxC5UUEw; __jdv=76161171|direct|-|none|-|1679360388347; __jdu=1677051379551729066919; areaId=14; PCSYCityID=CN_340000_340100_0; shshshfpx=353c3350-9f6e-c6e4-75c2-e45fb0638a20-1677122793; __jda=122270672.1677051379551729066919.1677051379.1677240645.1679360388.14; __jdc=122270672; jsavif=1; shshshfp=560297ae18037fe111337616ab2a555f; token=06336cfeaa30940f5c417f6798e29f98,2,932978; __tk=115a0c213a52a38c2ce94507d97fc721,2,932978; ipLoc-djd=14-1116-3431-57939; 3AB9D23F7A4B3C9B=T6XOSS2CQO2OX3CXET3VGDVF7I5HMHLXB4ZJR7Y73ZLZJCFPBJOSJNGAPFVEW5DQB6OJQEHGFPLPICSY2LRQX6UUGM; jwotest_product=99; CA1AN5BV0CA8DS2EPC=166bb245180140fcb233e32ead6800cb; PCA9D23F7A4B3CSS=7fe8a2d8af887bd902df1a00848ab151; 3AB9D23F7A4B3CSS=jdd03T6XOSS2CQO2OX3CXET3VGDVF7I5HMHLXB4ZJR7Y73ZLZJCFPBJOSJNGAPFVEW5DQB6OJQEHGFPLPICSY2LRQX6UUGMAAAAMHAHEIWMQAAAAACMTZKWKC62MG3AX; _gia_d=1; shshshsID=55c4b556288dea72398a8eb93ef6dc03_8_1679362134063; __jdb=122270672.9.1677051379551729066919|14.1679360388; JSESSIONID=D6FE691B40A1D1D5386BAA5EDD77C29D.s1', 'referer': 'https://item.jd.com/', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36' } response = requests.get(url=url, params=param, headers=headers) for index in response.json()['comments']: content = index['content'] print(content) with open('good_comments2.txt', mode='a', encoding='utf-8') as fp: fp.write(content) fp.write('\n')
中评:
import requests if __name__ == "__main__": #爬取中评55页 for page in range(0,55): url = 'https://club.jd.com/comment/productPageComments.action' param = { 'productId': '100023000435', 'score': '2', #中评score 'sortType': '5', 'page': page, 'pageSize': '10', 'isShadowSku': '0', 'fold': '1', } headers = { 'cookie': 'shshshfpa=353c3350-9f6e-c6e4-75c2-e45fb0638a20-1677122793; shshshfpb=cYvrYbQje1MA2t7vxC5UUEw; __jdv=76161171|direct|-|none|-|1679360388347; __jdu=1677051379551729066919; areaId=14; PCSYCityID=CN_340000_340100_0; shshshfpx=353c3350-9f6e-c6e4-75c2-e45fb0638a20-1677122793; __jda=122270672.1677051379551729066919.1677051379.1677240645.1679360388.14; __jdc=122270672; jsavif=1; shshshfp=560297ae18037fe111337616ab2a555f; token=06336cfeaa30940f5c417f6798e29f98,2,932978; __tk=115a0c213a52a38c2ce94507d97fc721,2,932978; ipLoc-djd=14-1116-3431-57939; 3AB9D23F7A4B3C9B=T6XOSS2CQO2OX3CXET3VGDVF7I5HMHLXB4ZJR7Y73ZLZJCFPBJOSJNGAPFVEW5DQB6OJQEHGFPLPICSY2LRQX6UUGM; jwotest_product=99; CA1AN5BV0CA8DS2EPC=166bb245180140fcb233e32ead6800cb; PCA9D23F7A4B3CSS=7fe8a2d8af887bd902df1a00848ab151; 3AB9D23F7A4B3CSS=jdd03T6XOSS2CQO2OX3CXET3VGDVF7I5HMHLXB4ZJR7Y73ZLZJCFPBJOSJNGAPFVEW5DQB6OJQEHGFPLPICSY2LRQX6UUGMAAAAMHAHEIWMQAAAAACMTZKWKC62MG3AX; _gia_d=1; shshshsID=55c4b556288dea72398a8eb93ef6dc03_8_1679362134063; __jdb=122270672.9.1677051379551729066919|14.1679360388; JSESSIONID=D6FE691B40A1D1D5386BAA5EDD77C29D.s1', 'referer': 'https://item.jd.com/', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36' } response = requests.get(url=url, params=param, headers=headers) for index in response.json()['comments']: content = index['content'] with open('middle_comments.txt', mode='a', encoding='utf-8') as fp: fp.write(content) fp.write('\n')
差评:
import requests if __name__ == "__main__": #爬取差评69页 for page in range(0,69): url = 'https://club.jd.com/comment/productPageComments.action' param = { 'productId': '100023203263', 'score': '1', #差评score 'sortType': '5', 'page': page, 'pageSize': '10', 'isShadowSku': '0', 'fold': '1', } headers = { 'cookie': 'shshshfpa=353c3350-9f6e-c6e4-75c2-e45fb0638a20-1677122793; shshshfpb=cYvrYbQje1MA2t7vxC5UUEw; __jdv=76161171|direct|-|none|-|1679360388347; __jdu=1677051379551729066919; areaId=14; PCSYCityID=CN_340000_340100_0; shshshfpx=353c3350-9f6e-c6e4-75c2-e45fb0638a20-1677122793; __jda=122270672.1677051379551729066919.1677051379.1677240645.1679360388.14; __jdc=122270672; jsavif=1; shshshfp=560297ae18037fe111337616ab2a555f; token=06336cfeaa30940f5c417f6798e29f98,2,932978; __tk=115a0c213a52a38c2ce94507d97fc721,2,932978; ipLoc-djd=14-1116-3431-57939; 3AB9D23F7A4B3C9B=T6XOSS2CQO2OX3CXET3VGDVF7I5HMHLXB4ZJR7Y73ZLZJCFPBJOSJNGAPFVEW5DQB6OJQEHGFPLPICSY2LRQX6UUGM; jwotest_product=99; CA1AN5BV0CA8DS2EPC=166bb245180140fcb233e32ead6800cb; PCA9D23F7A4B3CSS=7fe8a2d8af887bd902df1a00848ab151; 3AB9D23F7A4B3CSS=jdd03T6XOSS2CQO2OX3CXET3VGDVF7I5HMHLXB4ZJR7Y73ZLZJCFPBJOSJNGAPFVEW5DQB6OJQEHGFPLPICSY2LRQX6UUGMAAAAMHAHEIWMQAAAAACMTZKWKC62MG3AX; _gia_d=1; shshshsID=55c4b556288dea72398a8eb93ef6dc03_8_1679362134063; __jdb=122270672.9.1677051379551729066919|14.1679360388; JSESSIONID=D6FE691B40A1D1D5386BAA5EDD77C29D.s1', 'referer': 'https://item.jd.com/', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36' } response = requests.get(url=url, params=param, headers=headers) for index in response.json()['comments']: content = index['content'] with open('bad_comments.txt', mode='a', encoding='utf-8') as fp: fp.write(content) fp.write('\n')
其中重要参数来源:
打开开发者工具,快捷键F12键,或鼠标右键-->检查-->网络
url = 'https://club.jd.com/comment/productPageComments.action'
不包括?号后参数
param参数:
headers:标头-->请求标头:cookie、referer、user-agent