跳转国外ip的软件_北京冬奥会网页设计_搜狗站长工具_4399网页版入口
chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])service = Service('D:/App Data/PythonProject/LiVideoCrawling/LiVideoCrawling/chrome_driver/chromedriver.exe')class LivideocrawlingDownloaderMiddleware:# Not all methods need to be defined. If a method is not defined,# scrapy acts as if the downloader middleware does not modify the# passed objects.# 实例化一个浏览器对象bro = Chrome(service=service, options=chrome_options)@classmethoddef from_crawler(cls, crawler):# This method is used by Scrapy to create your spiders.s = cls()crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)return sdef process_request(self, request, spider):# 设置一个正则表达式，用于匹配指定类型的urlpattern = r'https://www\.pearvideo\.com/videoStatus\.jsp\?.*?'# 判断是否为视频的异步请求urlif re.match(pattern, request.url):# print(request.url)# print(spider.json_ids)# 请确认为视频异步请求url后# 通过循环判断，本次的url对应的视频idfor jid in spider.json_ids:# 获得主页中的预先存入的视频idfid = jid.split('_')[-1]# 获得当前请求url的视频idpattern = r'contId=(\d+)'match = re.search(pattern, request.url)vid = ''if match:vid = match.group(1)# print("主页视频id", fid, "详情页视频id", vid)# 判断这两个id是否相等if fid.startswith(vid):print('相等')# 如果相等则对防盗链请求头进行拼接request.headers['Referer'] = 'https://www.pearvideo.com/' + jiddef process_response(self, request, response, spider):if request.url.startswith(spider.start_urls[0]):# 对页面中的首页进行selenium请求self.bro.get(request.url)html_data = self.bro.page_source# 创建一个新的响应对象new_response = HtmlResponse(url=request.url, body=html_data.encode(), encoding='utf-8', request=request)return new_responseelse:return responsedef process_exception(self, request, exception, spider):# Called when a download handler or a process_request()# (from other downloader middleware) raises an exception.# Must either:# - return None: continue processing this exception# - return a Response object: stops process_exception() chain# - return a Request object: stops process_exception() chainpassdef spider_opened(self, spider):spider.logger.info("Spider opened: %s" % spider.name)
专业网站建设模块维护自己建个网站需要什么工作室团队官网源码网站怎么建设的物业公司管理系统邯郸渊博网络有限公司平面设计真实工资网络服务提供者知道或者应当知道网络用户中国知名的品牌策划公司抖音seo搜索优化简单的网站设计图提升学历是什么意思制作logo网站创业平台app如何快速网络推广产品的方法网站创建的基本流程苏州网页制作公司哪家好宁波网络营销咨询公司广东seo快速排名百度提交网站收录查询设计好网站专业ppt制作价格沈阳外贸网站制作公司网络设计大赛seo排名影响因素主要有腾讯营销平台网站建设电话咨询网页源代码是什么贵州网站建设公司超级简历模板官网济南建设工程交易网