使用XRAY多进程批量扫描

前提是需要通过某种方法收集到域名信息，
原本的想法是oneforall(subfinder)+nmap+httpx，
后来看到这篇文章如何利用xray、burp、lsc构成自动化挖src平台
发现LangSrcCurise似乎也不错，就直接拿这个用好了。需要配置好数据库信息，然后多进程就可以跑起来了。

import subprocess
import time
from multiprocessing import Pool,Process,Queue,JoinableQueue
import pymysql
import os

#rad路径(建议使用相对路径)
radPath="..\\rad_windows_amd64.exe"
#xray路径(建议使用相对路径)
xrayPath="..\\xray_windows_amd64.exe"

def Scan(q):
    while q.get():
        target = q.get()
        pid = os.getpid()
        # 开始xray扫描，这里的xray用的是高级版，直接就爬虫上了，后续有更好的扫描方法也可以优化
        outputPath=time.strftime("%Y%m%d%H%M%S",time.localtime())
        cmd = [xrayPath,"webscan","--browser-crawler",target,"--html-output","..\\result\\"+outputPath+target.replace("http://","").replace("https://","")+".html"]
        xrayshell = subprocess.Popen(cmd,stdout=subprocess.PIPE)
        # 持续获取输出，并打印
        j = 0
        while not xrayshell.poll():
            if 1:
                print('from subprocess', pid,':', end='')
                line = xrayshell.stdout.readline().strip()
                print(line.decode())
                if len(line) == 0:
                    j=j+1
                    print(j)
                    if j > 10:
                        break
                else :
                    j = 0
        #发现不输出之后关闭xray，向队列发出信号，消费完成
        xrayshell.kill()
        q.task_done()


def geturl(startnumber,q):
    connect = pymysql.connect(host='IP地址', port=3306, user='用户', passwd='密码', db='langsrccurise',
                              charset='utf8mb4')
    #读取数据行数
    with connect.cursor() as cursor:
        cursor.execute("select COUNT(*) FROM URL",())
        max = cursor.fetchone()
    print(max[0])
    for counter in range(int(startnumber),int(max[0])):
        with connect.cursor() as cursor:
            sql = "select url FROM URL WHERE uid=%s"
            cursor.execute(sql, (str(counter),))
            url = cursor.fetchone()
            q.put(url[0])
            #获取到计数的那一行，并发送给队列

        f = open('count.txt', 'w+')
        counter = str(counter+1)
        f.write(counter)
        f.close()
    connect.close()
    q.join()

if __name__ == '__main__':
    #在外部引用一个文件用来计数
    f = open('count.txt', 'r')
    counter = f.readline()
    f.close()

    q = JoinableQueue(10)
    #设置最大长度为10的缓冲队列，5个xray扫描进程
    p1 = Process(target=geturl, args=(counter, q))
    c1 = Process(target=Scan, args=(q,))
    c2 = Process(target=Scan, args=(q,))
    c3 = Process(target=Scan, args=(q,))
    c4 = Process(target=Scan, args=(q,))
    c5 = Process(target=Scan, args=(q,))
    p1.start()
    c1.start()
    c2.start()
    c3.start()
    c4.start()
    c5.start()
    p1.join()
    q.put(None)
    q.put(None)