From 36ff49c4fe52123cfe5e80637d12763be8b787a2 Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Mon, 3 Mar 2025 04:53:23 +0800 Subject: [PATCH 01/10] feat: add dns query, tested --- actions_requirements.txt | 3 +- fetch_ips.py | 82 ++++++++++++++++++++++++++++++++++------ requirements.txt | 1 + 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/actions_requirements.txt b/actions_requirements.txt index f8a7048fd..e56db2a40 100644 --- a/actions_requirements.txt +++ b/actions_requirements.txt @@ -1,3 +1,4 @@ requests-html==0.10.0 retry==0.9.2 -lxml_html_clean \ No newline at end of file +lxml_html_clean +aiodns==3.0.0 \ No newline at end of file diff --git a/fetch_ips.py b/fetch_ips.py index 89dfac011..e16497508 100644 --- a/fetch_ips.py +++ b/fetch_ips.py @@ -6,8 +6,11 @@ # Date : 2020-05-19 15:27 # Desc : 获取最新的 GitHub 相关域名对应 IP import re -from typing import Any, Optional +from typing import Any, List, Optional from datetime import datetime +import sys +import asyncio +import aiodns from pythonping import ping from requests_html import HTMLSession @@ -16,7 +19,7 @@ from retry import retry from common import GITHUB_URLS, write_hosts_content -def get_best_ip(ip_list: list) -> str: +def select_ip_from_list(ip_list: list) -> str: ping_timeout = 1 best_ip = '' min_ms = ping_timeout * 1000 @@ -35,7 +38,7 @@ def get_best_ip(ip_list: list) -> str: @retry(tries=3) -def get_ip(session: Any, github_url: str) -> Optional[str]: +def get_ip_list_from_ipaddress_com(session: Any, github_url: str) -> Optional[List[str]]: url = f'https://sites.ipaddress.com/{github_url}' headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)' @@ -45,17 +48,69 @@ def get_ip(session: Any, github_url: str) -> Optional[str]: rs = session.get(url, headers=headers, timeout=5) pattern = r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b" ip_list = re.findall(pattern, rs.html.text) - best_ip = get_best_ip(ip_list) - if best_ip: - return best_ip - else: - raise Exception("url: {github_url}, ipaddress empty") + return ip_list except Exception as ex: print("get: {url}, error: {ex}") raise Exception -def main() -> None: +DNS_SERVER_LIST = [ + "1.1.1.1", # Cloudflare + "8.8.8.8", # Google + "101.101.101.101", # Quad101 + "101.102.103.104", # Quad101 +] + + +def windows_compatibility_check(): + if sys.platform == "win32": + # 检查 pycares 是否正常加载 + try: + import pycares + except ImportError: + raise RuntimeError("请先执行 'pip install pycares'") + + +async def get_ip_list_from_dns( + domain, + record_type="A", + dns_server_list=["1.2.4.8", "114.114.114.114"], +): + # Windows 兼容性检查 + windows_compatibility_check() + + # 配置 DNS 服务器 + resolver = aiodns.DNSResolver() + resolver.nameservers = dns_server_list + + try: + # 执行异步查询 + result = await resolver.query(domain, record_type) + return [answer.host for answer in result] + except aiodns.error.DNSError as e: + print(f"DNS 查询失败: {e}") + return [] + + +async def get_ip(session: Any, github_url: str) -> Optional[str]: + ip_list_web = [] + try: + ip_list_web = get_ip_list_from_ipaddress_com(session, github_url) + except Exception as ex: + raise Exception("url: {github_url}, ipaddress empty") + ip_list_dns = [] + try: + ip_list_dns = await get_ip_list_from_dns(github_url, dns_server_list=DNS_SERVER_LIST) + except Exception as ex: + pass + ip_list = list(set(ip_list_web + ip_list_dns)) + if len(ip_list) == 0: + return None + best_ip = select_ip_from_list(ip_list) + return best_ip + + +async def main() -> None: current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') print(f'{current_time} - Start script.') session = HTMLSession() @@ -63,7 +118,7 @@ def main() -> None: content_list = [] for index, github_url in enumerate(GITHUB_URLS): try: - ip = get_ip(session, github_url) + ip = await get_ip(session, github_url) content += ip.ljust(30) + github_url + "\n" content_list.append((ip, github_url,)) @@ -77,5 +132,8 @@ def main() -> None: print(f'{current_time} - End script.') -if __name__ == '__main__': - main() +if __name__ == "__main__": + if sys.platform == "win32": + # Windows 事件循环策略配置 + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + asyncio.run(main()) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d8386203f..12b8ea3d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ requests-html==0.10.0 pythonping==1.1.4 retry==0.9.2 lxml_html_clean +aiodns==3.0.0 \ No newline at end of file From 2edabee4aba0a9e13dcf094c2316b8ec8b3f938d Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Mon, 3 Mar 2025 05:27:32 +0800 Subject: [PATCH 02/10] fix: 'continue' when ip not found --- fetch_ips.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fetch_ips.py b/fetch_ips.py index e16497508..9e72a2280 100644 --- a/fetch_ips.py +++ b/fetch_ips.py @@ -119,7 +119,9 @@ async def main() -> None: for index, github_url in enumerate(GITHUB_URLS): try: ip = await get_ip(session, github_url) - + if ip is None: + print(f"{github_url}: IP Not Found") + continue content += ip.ljust(30) + github_url + "\n" content_list.append((ip, github_url,)) except Exception: From 795d87aecf5ce93345a7bf0add4e258ea05315f0 Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Mon, 3 Mar 2025 05:40:56 +0800 Subject: [PATCH 03/10] feat: do not multi-ping --- fetch_ips.py | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/fetch_ips.py b/fetch_ips.py index 9e72a2280..dc6cac791 100644 --- a/fetch_ips.py +++ b/fetch_ips.py @@ -6,7 +6,7 @@ # Date : 2020-05-19 15:27 # Desc : 获取最新的 GitHub 相关域名对应 IP import re -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional from datetime import datetime import sys import asyncio @@ -19,21 +19,28 @@ from retry import retry from common import GITHUB_URLS, write_hosts_content -def select_ip_from_list(ip_list: list) -> str: - ping_timeout = 1 - best_ip = '' - min_ms = ping_timeout * 1000 - ip_set = set(ip_list) - for ip in ip_set: - ping_result = ping(ip, timeout=ping_timeout) - print(f'ping {ip} {ping_result.rtt_avg_ms} ms') - if ping_result.rtt_avg_ms == ping_timeout * 1000: - # 超时认为 IP 失效 - continue - else: - if ping_result.rtt_avg_ms < min_ms: - min_ms = ping_result.rtt_avg_ms - best_ip = ip +PING_LIST: Dict[str, int] = dict() + + +def ping(ip: str) -> int: + global PING_LIST + if ip in PING_LIST: + return PING_LIST[ip] + ping_timeout_sec = 1 + ping_times = [ping(ip, timeout=ping_timeout_sec).rtt_avg_ms for _ in range(3)] + ping_times.sort() + print(f'Ping {ip}: {ping_times} ms') + PING_LIST[ip] = ping_times[1] # 取中位数 + return PING_LIST[ip] + + +def select_ip_from_list(ip_list: List[str]) -> Optional[str]: + if len(ip_list) == 0: + return None + ping_results = [(ip, ping(ip)) for ip in ip_list] + ping_results.sort(lambda x: x[1]) + best_ip = ping_results[0][0] + print(f"{ping_results}, selected {best_ip}") return best_ip @@ -50,7 +57,7 @@ def get_ip_list_from_ipaddress_com(session: Any, github_url: str) -> Optional[Li ip_list = re.findall(pattern, rs.html.text) return ip_list except Exception as ex: - print("get: {url}, error: {ex}") + print(f"get: {url}, error: {ex}") raise Exception @@ -88,7 +95,7 @@ async def get_ip_list_from_dns( result = await resolver.query(domain, record_type) return [answer.host for answer in result] except aiodns.error.DNSError as e: - print(f"DNS 查询失败: {e}") + print(f"{domain}: DNS 查询失败: {e}") return [] From 7e2dbb3d45e838f9dab09680d568acfc3f2a1d60 Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Mon, 3 Mar 2025 05:53:33 +0800 Subject: [PATCH 04/10] fix: ping func dup --- fetch_ips.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/fetch_ips.py b/fetch_ips.py index dc6cac791..dc9cf5887 100644 --- a/fetch_ips.py +++ b/fetch_ips.py @@ -22,7 +22,7 @@ from common import GITHUB_URLS, write_hosts_content PING_LIST: Dict[str, int] = dict() -def ping(ip: str) -> int: +def ping_cached(ip: str) -> int: global PING_LIST if ip in PING_LIST: return PING_LIST[ip] @@ -37,7 +37,7 @@ def ping(ip: str) -> int: def select_ip_from_list(ip_list: List[str]) -> Optional[str]: if len(ip_list) == 0: return None - ping_results = [(ip, ping(ip)) for ip in ip_list] + ping_results = [(ip, ping_cached(ip)) for ip in ip_list] ping_results.sort(lambda x: x[1]) best_ip = ping_results[0][0] print(f"{ping_results}, selected {best_ip}") @@ -104,15 +104,17 @@ async def get_ip(session: Any, github_url: str) -> Optional[str]: try: ip_list_web = get_ip_list_from_ipaddress_com(session, github_url) except Exception as ex: - raise Exception("url: {github_url}, ipaddress empty") + pass ip_list_dns = [] try: ip_list_dns = await get_ip_list_from_dns(github_url, dns_server_list=DNS_SERVER_LIST) except Exception as ex: pass ip_list = list(set(ip_list_web + ip_list_dns)) + ip_list.sort() if len(ip_list) == 0: return None + print(f"{github_url}: {ip_list}") best_ip = select_ip_from_list(ip_list) return best_ip @@ -124,6 +126,7 @@ async def main() -> None: content = "" content_list = [] for index, github_url in enumerate(GITHUB_URLS): + print(f'Start Processing url: {index + 1}/{len(GITHUB_URLS)}, {github_url}') try: ip = await get_ip(session, github_url) if ip is None: @@ -133,7 +136,6 @@ async def main() -> None: content_list.append((ip, github_url,)) except Exception: continue - print(f'Process url: {index + 1}/{len(GITHUB_URLS)}, {github_url}') write_hosts_content(content, content_list) # print(hosts_content) From a3026462e59853bb776c5d85b3858fdc97778a84 Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Mon, 3 Mar 2025 06:03:17 +0800 Subject: [PATCH 05/10] fix: not selecting --- fetch_ips.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetch_ips.py b/fetch_ips.py index dc9cf5887..83e0fe9d6 100644 --- a/fetch_ips.py +++ b/fetch_ips.py @@ -38,7 +38,7 @@ def select_ip_from_list(ip_list: List[str]) -> Optional[str]: if len(ip_list) == 0: return None ping_results = [(ip, ping_cached(ip)) for ip in ip_list] - ping_results.sort(lambda x: x[1]) + ping_results.sort(key=lambda x: x[1]) best_ip = ping_results[0][0] print(f"{ping_results}, selected {best_ip}") return best_ip From c8b7df7c4f7cd4154903ce01120ded8128669e98 Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Mon, 3 Mar 2025 06:06:31 +0800 Subject: [PATCH 06/10] feat: discard some ip results --- fetch_ips.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fetch_ips.py b/fetch_ips.py index 83e0fe9d6..a6578f5e8 100644 --- a/fetch_ips.py +++ b/fetch_ips.py @@ -20,6 +20,7 @@ from common import GITHUB_URLS, write_hosts_content PING_LIST: Dict[str, int] = dict() +DISCARD_LIST: List[str] = ["1.0.1.1", "1.2.1.1", "127.0.0.1"] def ping_cached(ip: str) -> int: @@ -110,7 +111,10 @@ async def get_ip(session: Any, github_url: str) -> Optional[str]: ip_list_dns = await get_ip_list_from_dns(github_url, dns_server_list=DNS_SERVER_LIST) except Exception as ex: pass - ip_list = list(set(ip_list_web + ip_list_dns)) + ip_list_set = set(ip_list_web + ip_list_dns) + for discard_ip in DISCARD_LIST: + ip_list_set.discard(discard_ip) + ip_list = list(ip_list_set) ip_list.sort() if len(ip_list) == 0: return None From 7a0019f45fa386d1788633aaebbd42d7cdde3084 Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Mon, 3 Mar 2025 06:11:15 +0800 Subject: [PATCH 07/10] feat: do not select when all timeout --- fetch_ips.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/fetch_ips.py b/fetch_ips.py index a6578f5e8..817e79c53 100644 --- a/fetch_ips.py +++ b/fetch_ips.py @@ -19,16 +19,18 @@ from retry import retry from common import GITHUB_URLS, write_hosts_content -PING_LIST: Dict[str, int] = dict() +PING_TIMEOUT_SEC: int = 1 DISCARD_LIST: List[str] = ["1.0.1.1", "1.2.1.1", "127.0.0.1"] +PING_LIST: Dict[str, int] = dict() + + def ping_cached(ip: str) -> int: global PING_LIST if ip in PING_LIST: return PING_LIST[ip] - ping_timeout_sec = 1 - ping_times = [ping(ip, timeout=ping_timeout_sec).rtt_avg_ms for _ in range(3)] + ping_times = [ping(ip, timeout=PING_TIMEOUT_SEC).rtt_avg_ms for _ in range(3)] ping_times.sort() print(f'Ping {ip}: {ping_times} ms') PING_LIST[ip] = ping_times[1] # 取中位数 @@ -41,6 +43,10 @@ def select_ip_from_list(ip_list: List[str]) -> Optional[str]: ping_results = [(ip, ping_cached(ip)) for ip in ip_list] ping_results.sort(key=lambda x: x[1]) best_ip = ping_results[0][0] + # 全都超时？那就不选了 + if ping_results[0][1] == PING_TIMEOUT_SEC * 1000: + print(f"{ping_results}, no selection") + return None print(f"{ping_results}, selected {best_ip}") return best_ip From 46d7a49012ae50be6ef20b1b5f11aebddc424193 Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Mon, 3 Mar 2025 06:21:56 +0800 Subject: [PATCH 08/10] feat: add not available sign when no result --- fetch_ips.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetch_ips.py b/fetch_ips.py index 817e79c53..714423575 100644 --- a/fetch_ips.py +++ b/fetch_ips.py @@ -141,7 +141,7 @@ async def main() -> None: ip = await get_ip(session, github_url) if ip is None: print(f"{github_url}: IP Not Found") - continue + ip = "# Not available" content += ip.ljust(30) + github_url + "\n" content_list.append((ip, github_url,)) except Exception: From 8917164aaca8fd42ca1d1dbc87d7c36de2486d1e Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Mon, 3 Mar 2025 06:37:58 +0800 Subject: [PATCH 09/10] fix: add not available sign only --- fetch_ips.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fetch_ips.py b/fetch_ips.py index 714423575..e33b7d304 100644 --- a/fetch_ips.py +++ b/fetch_ips.py @@ -43,10 +43,6 @@ def select_ip_from_list(ip_list: List[str]) -> Optional[str]: ping_results = [(ip, ping_cached(ip)) for ip in ip_list] ping_results.sort(key=lambda x: x[1]) best_ip = ping_results[0][0] - # 全都超时？那就不选了 - if ping_results[0][1] == PING_TIMEOUT_SEC * 1000: - print(f"{ping_results}, no selection") - return None print(f"{ping_results}, selected {best_ip}") return best_ip @@ -142,7 +138,11 @@ async def main() -> None: if ip is None: print(f"{github_url}: IP Not Found") ip = "# Not available" - content += ip.ljust(30) + github_url + "\n" + content += ip.ljust(30) + github_url + global PING_LIST + if PING_LIST.get(ip) is not None and PING_LIST.get(ip) == PING_TIMEOUT_SEC * 1000: + content += " # Not Available" + content += "\n" content_list.append((ip, github_url,)) except Exception: continue From ee666b4ab8b8e0077fd65ae938af85afeba1a6dd Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Mon, 3 Mar 2025 06:40:18 +0800 Subject: [PATCH 10/10] fix: optimize tips --- fetch_ips.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fetch_ips.py b/fetch_ips.py index e33b7d304..87b313c4b 100644 --- a/fetch_ips.py +++ b/fetch_ips.py @@ -137,11 +137,11 @@ async def main() -> None: ip = await get_ip(session, github_url) if ip is None: print(f"{github_url}: IP Not Found") - ip = "# Not available" + ip = "# IP Address Not Found" content += ip.ljust(30) + github_url global PING_LIST if PING_LIST.get(ip) is not None and PING_LIST.get(ip) == PING_TIMEOUT_SEC * 1000: - content += " # Not Available" + content += " # Timeout" content += "\n" content_list.append((ip, github_url,)) except Exception: