文章摘要:其实这里casperjs与python没有直接关系,主要依赖casperjs调用phantomjs webkit获取html文件内容。长期以来,爬虫抓取 客户端javascript渲染生成的html页面 都极为 困难,Java里面有 HtmlUnit,而Python里,我们可以使用独立的跨平台的CasperJS。
创建site.js(接口文件,输入:url,输出:html file)
//USAGE: E:\toolkit\n1k0-casperjs-e3a77d0\bin>python casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile='temp.html' var fs = require('fs'); var casper = require('casper').create({ pageSettings: { loadImages: false,loadplugins: false,userAgent: 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/34.0.1847.137 Safari/537.36 LBbroWSER' },logLevel: "debug",//日志等级 verbose: true // 记录日志到控制台 }); var url = casper.cli.raw.get('url'); var outputfile = casper.cli.raw.get('outputfile'); //请求页面 casper.start(url,function () { fs.write(outputfile,this.getHTML(),'w'); }); casper.run();
python 代码,checkout_proxy.py
import json import sys #import requests #import requests.utils,pickle from bs4 import BeautifulSoup import os.path,os import threading #from multiprocessing import Process,Manager from datetime import datetime import traceback import logging import re,random import subprocess import shutil import platform output_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),'proxy.txt') global_log = 'http_proxy' + datetime.Now().strftime('%Y-%m-%d') + '.log' if not os.path.exists(os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs')): os.mkdir(os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs')) global_log = os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs',global_log) logging.basicConfig(level=logging.DEBUG,format='[%(asctime)s] [%(levelname)s] [%(module)s] [%(funcName)s] [%(lineno)d] %(message)s',filename=global_log,filemode='a') log = logging.getLogger(__name__) #manager = Manager() #PROXY_LIST = manager.list() mutex = threading.Lock() PROXY_LIST = [] def isWindows(): if "Windows" in str(platform.uname()): return True else: return False def getTagsByAttrs(tagName,pageContent,attrName,attrRegValue): soup = BeautifulSoup(pageContent) return soup.find_all(tagName,{ attrName : re.compile(attrRegValue) }) def getTagsByAttrsExt(tagName,filename,attrRegValue): if os.path.isfile(filename): f = open(filename,'r') soup = BeautifulSoup(f) f.close() return soup.find_all(tagName,{ attrName : re.compile(attrRegValue) }) else: return None class Site1Thread(threading.Thread): def __init__(self,outputFilePath): threading.Thread.__init__(self) self.outputFilePath = outputFilePath self.fileName = str(random.randint(100,1000)) + ".html" self.setName('Site1Thread') def run(self): site1_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),'site.js') site2_file = os.path.join(self.outputFilePath,'site.js') if not os.path.isfile(site2_file) and os.path.isfile(site1_file): shutil.copy(site1_file,site2_file) #proc = subprocess.Popen(["bash","-c","cd %s && ./casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE) if isWindows(): proc = subprocess.Popen(["cmd","/c","%s/casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s" % (self.outputFilePath,stdout=subprocess.PIPE) else: proc = subprocess.Popen(["bash",stdout=subprocess.PIPE) out=proc.communicate()[0] htmlFileName = '' #因为输出路径在windows不确定,所以这里加了所有可能的路径判断 if os.path.isfile(self.fileName): htmlFileName = self.fileName elif os.path.isfile(os.path.join(self.outputFilePath,self.fileName)): htmlFileName = os.path.join(self.outputFilePath,self.fileName) elif os.path.isfile(os.path.join(os.path.dirname(os.path.realpath(__file__)),self.fileName)): htmlFileName = os.path.join(os.path.dirname(os.path.realpath(__file__)),self.fileName) if (not os.path.isfile(htmlFileName)): print 'Failed to get html content from http://spys.ru/free-proxy-list/IE/' print out sys.exit(3) mutex.acquire() PROXYList= getTagsByAttrsExt('font',htmlFileName,'class','spy14$') for proxy in PROXYList: tdContent = proxy.renderContents() lineElems = re.split('[<>]',tdContent) if re.compile(r'\d+').search(lineElems[-1]) and re.compile('(\d+\.\d+\.\d+)').search(lineElems[0]): print lineElems[0],lineElems[-1] PROXY_LIST.append("%s:%s" % (lineElems[0],lineElems[-1])) mutex.release() try: if os.path.isfile(htmlFileName): os.remove(htmlFileName) except: pass if __name__ == '__main__': try: if(len(sys.argv)) < 2: print "Usage:%s [casperjs path]" % (sys.argv[0]) sys.exit(1) if not os.path.exists(sys.argv[1]): print "casperjs path: %s does not exist!" % (sys.argv[1]) sys.exit(2) if os.path.isfile(output_file): f = open(output_file) lines = f.readlines() f.close for line in lines: PROXY_LIST.append(line.strip()) thread1 = Site1Thread(sys.argv[1]) thread1.start() thread1.join() f = open(output_file,'w') for proxy in set(PROXY_LIST): f.write(proxy+"\n") f.close() print "Done!" except SystemExit: pass except: errMsg = traceback.format_exc() print errMsg log.error(errMsg)
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。