import re, requests, urllib.parse, os, sys
from bs4 import BeautifulSoup
import  random, pandas as pd
import traceback, colorama, time
from rich.prompt import Prompt
import warnings
from urllib3.exceptions import InsecureRequestWarning

info = input('声明：利用本程序禁止进行违法活动，如有进行违法活动与作者无关！（按Enter键表示您/您们已同意）')
if info != '':
    quit()

# 设置 HTTP 头部
def get_random_user_agent():
    browser_type = random.choice(['360', 'Firefox', '2345', 'IE', 'Edge'])
    if browser_type == '360':
        wt = ['6.1', '6.2', '6.3', '10.0']
    elif browser_type == 'Firefox':
        wt = ['10.0']
    elif browser_type == '2345':
        wt = ['5.1', '6.1', '6.2', '10.0']
    elif browser_type == 'IE':
        wt = ['5.1', '6.0', '6.1', '6.2', '6.3']
    elif browser_type == 'Edge':
        wt = ['10.0']
    Windows_type = random.choice(wt)
    os_type = random.choice(['64', '32'])
    browser_num = random.randint(1000, 2000)
    browser_num2 = random.randint(10, 20)
    browser_num3 = random.randint(100, 999)
    browser_num4 = random.randint(0, 9)
    browser_num21 = random.randint(10, 20)
    browser_num31 = random.randint(100, 999)
    browser_num41 = random.randint(0, 9)
    cp = random.randint(15, 100)
    if browser_type == '360':
        ua = f'Mozilla/5.0 (Windows NT {Windows_type}; Win{os_type}; x{os_type}) AppleWebKit/{browser_num3}.{browser_num21} (KHTML, like Gecko) Chrome/{browser_num31}.{browser_num41}.{browser_num41}.{browser_num4} Safari/{browser_num3}.{browser_num2} QIHU 360SE/{browser_num21}.{browser_num41}.{browser_num}.0'
    elif browser_type == 'Firefox':
        ua = f'Mozilla/5.0 (Windows NT {Windows_type}; Win{os_type}; x{os_type}; rv:{browser_num31}.{browser_num4}) Gecko/20100101 Firefox/{browser_num3}.{browser_num41}'
    elif browser_type == '2345':
        ua = f'Mozilla/5.0 (Windows NT {Windows_type}; Win{os_type}; x{os_type}) AppleWebKit/{browser_num31}.{browser_num2} (KHTML, like Gecko) Chrome/{browser_num3}.{browser_num41}.0.0 Safari/537.{browser_num21} 2345Explorer/11.0.0.{browser_num}'
    elif browser_type == 'IE':
        ua = f'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT {Windows_type}; Trident/4.0)'
    elif browser_type == 'Edge':
        ua = f'Mozilla/5.0 (Windows NT {Windows_type}; Win{os_type}; x{os_type}) AppleWebKit/{browser_num31}.{browser_num2} (KHTML, like Gecko) Chrome/{browser_num3}.0.0.0 Safari/537.36 Edg/120.0.0.0'
    return ua

# 定义一个函数来获取网页的标题和链接
def get_links_and_title(url, ip_input, use_proxy=False):
    ua = get_random_user_agent()
    headers = {'X-Forwarded-For': ip_input, 'X-Real-IP': ip_input, 'User-Agent': ua}
    proxies = None
    try:
        start_time = time.time()  # 开始时间
        response = requests.get(url, headers=headers, timeout=10, allow_redirects=True, proxies=proxies, verify=False)
        response.raise_for_status()  # 检查请求是否成功
        delay = time.time() - start_time  # 计算延迟
        soup = BeautifulSoup(response.text, 'html.parser')  # 解析网页内容
        title = soup.title.string if soup.title else "No Title"  # 获取标题
        links = [a['href'] for a in soup.find_all('a', href=True)]  # 获取所有链接
        print(f'网址：{url}')
        print(f'状态：{response}')
        print(f'标题：{title}')
        print(f'延迟：{delay}')
        print('----------------')
        # 处理相对链接
        base_url = url.split('?')[0].split('#')[0]
        links = [urllib.parse.urljoin(base_url, link) for link in links]
        return links, title, delay
    except Exception as e:
        print(f"Error fetching {url}: {e}")
        return [], "Error", -1

# 主爬虫函数
def spider(seed_url, max_depth=25, ip_input=None, use_proxy=False):
    visited = set()  # 已访问的网页集合
    to_visit = [(seed_url, 0)]  # 待访问的网页队列，包含网址和深度
    results = []  # 保存结果
    domain = urllib.parse.urlparse(seed_url).netloc  # 获取目标域名

    while to_visit:
        current_url, depth = to_visit.pop(0)
        if depth > max_depth or current_url in visited:
            continue

        visited.add(current_url)
        links, title, delay = get_links_and_title(current_url, ip_input, use_proxy)
        results.append((current_url, title, delay))

        # 将新链接加入待访问队列
        for link in links:
            if link.startswith('http'):
                link_domain = urllib.parse.urlparse(link).netloc
                if link_domain == domain and link not in visited:
                    to_visit.append((link, depth + 1))

        # 增加请求间隔
        time.sleep(0.1)

    return results

# 将结果写入文件
def write_results_to_file(results, file_path):
    with open(file_path, 'w', encoding='utf-8') as f:
        for url, title, delay in results:
            f.write(f"URL: {url}\nTitle: {title}\nDelay: {delay:.2f}s\n\n")

# 主程序
if __name__ == "__main__":
    seed_url = input("请输入种子网站网址: ")
    ip_input = Prompt.ask('请输入访问时的ipv4地址（输入有效且目标直接信任 HTTP 头）：', default=str(requests.get('https://checkip.amazonaws.com/', timeout=5).text.strip()))
    use_proxy = Prompt.ask('是否使用代理？（是/否）', choices=["是", "否"], default="否") == "是"
    ssl = Prompt.ask('开启未验证证书警告？（是/否）', choices=["是", "否"], default="否")
    if ssl == '否':
        # 忽略 InsecureRequestWarning
        warnings.filterwarnings('ignore', category=InsecureRequestWarning)
    print('----------------')
    results = spider(seed_url, max_depth=25, ip_input=ip_input, use_proxy=use_proxy)