From 96375faefe9a5c3acee0a3c0b85f26218d655415 Mon Sep 17 00:00:00 2001 From: sinspired Date: Thu, 14 Nov 2024 22:03:58 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=BE=93=E5=85=A5=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E8=BF=87=E5=A4=9A=E6=97=B6=E8=B5=84=E6=BA=90=E6=B6=88?= =?UTF-8?q?=E8=80=97=E8=BF=87=E5=A4=9A=E9=97=AE=E9=A2=98=EF=BC=9B=E6=9B=B4?= =?UTF-8?q?=E6=96=B0custom=5Fip=5Franges,=E8=A6=86=E7=9B=96=E6=9B=B4?= =?UTF-8?q?=E5=85=A8=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/http.go | 5 - common/utils.go | 458 +++++++++++++-------- configs/config.toml | 4 +- data/all_google_translate_ip_ranges.txt | 63 ++- data/custom_google_translate_ip_ranges.txt | 10 +- 5 files changed, 333 insertions(+), 207 deletions(-) diff --git a/common/http.go b/common/http.go index 03080d7..090f209 100644 --- a/common/http.go +++ b/common/http.go @@ -7,13 +7,8 @@ import ( "log/slog" "net" "net/http" - "strings" ) -func isIPv4(ip string) bool { - return strings.Contains(ip, ".") -} - func dialContext(destination string, destinationPort uint16) func(ctx context.Context, network, address string) (net.Conn, error) { var addr string if isIPv4(destination) { diff --git a/common/utils.go b/common/utils.go index 815d0e4..b6b7ff5 100644 --- a/common/utils.go +++ b/common/utils.go @@ -4,103 +4,238 @@ import ( "bufio" "fmt" "io" - "log" "log/slog" + "math/rand" "net/netip" "os" "reflect" "runtime" "strings" "sync" + "time" ) -type IPArray struct { - IPs []string - ipsMutex sync.Mutex +const ( + maxWorkers = 10 + batchSize = 1000 + // IPv6采样数量 + ipv6SampleSize = 1000 +) + +// IPProcessor 提供IP处理功能 +type IPProcessor struct { + resultChan chan string + errorChan chan error + done chan struct{} } -func (arr *IPArray) AddIP(ip string) { - arr.ipsMutex.Lock() - if arr.IPs == nil { - arr.IPs = make([]string, 0) +// NewIPProcessor 创建新的处理器 +func NewIPProcessor() *IPProcessor { + return &IPProcessor{ + resultChan: make(chan string, batchSize), + errorChan: make(chan error, maxWorkers), + done: make(chan struct{}), } - arr.IPs = append(arr.IPs, ip) - arr.ipsMutex.Unlock() } -func CIDRToIPs(cidrAddress string, iparr *IPArray, wg *sync.WaitGroup) { - defer wg.Done() - p, err := netip.ParsePrefix(cidrAddress) +// generateRandomIPv6 在给定前缀内生成随机IPv6地址 +func generateRandomIPv6(prefix netip.Prefix) (netip.Addr, error) { + // 获取网络前缀部分 + network := prefix.Addr().AsSlice() + ones := prefix.Bits() + + // 计算需要随机化的字节数 + randomBytes := 16 - (ones+7)/8 + + // 复制原始地址 + result := make([]byte, 16) + copy(result, network) + + // 随机化剩余字节 + for i := 16 - randomBytes; i < 16; i++ { + result[i] = byte(rand.Intn(256)) + } + + // 转换回netip.Addr + addr, ok := netip.AddrFromSlice(result) + if !ok { + return netip.Addr{}, fmt.Errorf("failed to create IPv6 address") + } + + return addr, nil +} + +// processCIDR 处理单个CIDR区间 +func processCIDR(cidr string, resultChan chan<- string) error { + p, err := netip.ParsePrefix(cidr) if err != nil { - slog.Error("invalid cidr:", slog.String("CIDR", cidrAddress), slog.Any("Error", err)) + return fmt.Errorf("invalid CIDR %s: %w", cidr, err) } + p = p.Masked() - addr := p.Addr() - for { - if !p.Contains(addr) { - break + + if p.Addr().Is6() { + // 对IPv6进行随机采样 + slog.Info("Processing IPv6 CIDR with random sampling", "range", cidr) + for i := 0; i < ipv6SampleSize; i++ { + addr, err := generateRandomIPv6(p) + if err != nil { + slog.Error("Failed to generate IPv6 address", "error", err) + continue + } + + select { + case resultChan <- addr.String(): + case <-time.After(100 * time.Millisecond): + slog.Warn("Failed to send IPv6", "ip", addr.String()) + } + } + } else { + // IPv4处理逻辑保持不变 + addr := p.Addr() + for { + if !p.Contains(addr) { + break + } + + select { + case resultChan <- addr.String(): + case <-time.After(100 * time.Millisecond): + slog.Warn("Failed to send IPv4", "ip", addr.String()) + } + + addr = addr.Next() } - iparr.AddIP(addr.String()) - addr = addr.Next() } + + return nil +} + +// GetIPs 主函数 +func GetIPs(config *Config) []string { + // 初始化随机数种子 + rand.Seed(time.Now().UnixNano()) + + cidrs, err := loadCIDRs(config) + if err != nil { + slog.Error("Failed to load CIDRs", "error", err) + return nil + } + + processor := NewIPProcessor() + cidrChan := make(chan string, maxWorkers) + var wg sync.WaitGroup + + // 启动工作协程 + workerCount := min(maxWorkers, len(cidrs)) + wg.Add(workerCount) + for i := 0; i < workerCount; i++ { + go func() { + defer wg.Done() + for cidr := range cidrChan { + if err := processCIDR(cidr, processor.resultChan); err != nil { + select { + case processor.errorChan <- err: + default: + slog.Error("Process CIDR failed", "cidr", cidr, "error", err) + } + } + } + }() + } + + // 收集结果 + var results []string + resultBatch := make([]string, 0, batchSize) + + go func() { + defer close(processor.done) + for ip := range processor.resultChan { + resultBatch = append(resultBatch, ip) + + if len(resultBatch) >= batchSize { + newBatch := make([]string, len(resultBatch)) + copy(newBatch, resultBatch) + results = append(results, newBatch...) + resultBatch = resultBatch[:0] + } + } + if len(resultBatch) > 0 { + results = append(results, resultBatch...) + } + }() + + // 发送CIDR到workers + for _, cidr := range cidrs { + select { + case cidrChan <- cidr: + case err := <-processor.errorChan: + slog.Error("CIDR processing error", "error", err) + } + } + close(cidrChan) + + wg.Wait() + close(processor.resultChan) + <-processor.done + + slog.Info("IP加载完成", "数量", len(results)) + return results +} + +// isIPv4 检查是否为IPv4地址 +func isIPv4(ip string) bool { + addr, err := netip.ParseAddr(ip) + if err != nil { + return false + } + return addr.Is4() } func loadCIDRs(config *Config) ([]string, error) { - var cidrs []string siteCfg := RetrieveSiteCfg(config) - customIPRangesFile := siteCfg.CustomIPRangesFile - ipRangesFile := siteCfg.IPRangesFile + targetFile := siteCfg.CustomIPRangesFile withIPv6 := siteCfg.WithIPv6 - targetFile := customIPRangesFile - _, err := os.Stat(targetFile) - if err == nil { - slog.Info("found custom ip ranges file.") - } else if os.IsNotExist(err) { - slog.Warn("custom ip ranges file does not exist, use default ip ranges file instead!") - targetFile = ipRangesFile - } else { - slog.Warn("custom ip ranges file stat error: use default ip ranges file instead!", "error", err) - targetFile = ipRangesFile + + // 检查自定义文件是否存在 + if _, err := os.Stat(targetFile); err != nil { + if os.IsNotExist(err) { + slog.Warn("Custom IP ranges file not found, using default") + targetFile = siteCfg.IPRangesFile + } else { + return nil, fmt.Errorf("failed to check IP ranges file: %w", err) + } } + f, err := os.Open(targetFile) if err != nil { - slog.Error("Could not open ip address ranges:", "file", targetFile) - return cidrs, err + return nil, fmt.Errorf("failed to open IP ranges file %s: %w", targetFile, err) } defer f.Close() + + var cidrs []string scanner := bufio.NewScanner(f) for scanner.Scan() { line := scanner.Text() + // 跳过空行和注释行 + if line == "" || (line[0] == '#') { + continue + } + // 如果不支持IPv6且当前行是IPv6地址，则跳过 if !withIPv6 && !isIPv4(line) { continue } - cidrs = append(cidrs, scanner.Text()) + cidrs = append(cidrs, line) } - if err := scanner.Err(); err != nil { - slog.Error("Could not load ip address ranges file:", targetFile) - return cidrs, err - } - return cidrs, err -} -func GetIPs(config *Config) []string { - var iparr IPArray - cidrs, err := loadCIDRs(config) - if err != nil { - slog.Error("get ips failed!") - return iparr.IPs - } - var wg sync.WaitGroup - for _, cidrAddress := range cidrs { - wg.Add(1) - go CIDRToIPs(cidrAddress, &iparr, &wg) + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("failed to read IP ranges file: %w", err) } - wg.Wait() - slog.Info("Load IPs:", "Count", len(iparr.IPs)) - return iparr.IPs + slog.Info("Load CIDRs:", "Count", len(cidrs)) + return cidrs, nil } -// RetrieveSiteCfg assumed that the site name must exist in the configuration file and no error handling required +// RetrieveSiteCfg 获取站点配置 func RetrieveSiteCfg(config *Config) Site { siteName := config.General.Site sv := reflect.ValueOf(config.Sites) @@ -113,24 +248,114 @@ func RetrieveSiteCfg(config *Config) Site { return Site{} } +// min 辅助函数 +func min(a, b int) int { + if a < b { + return a + } + return b +} + +// writeToFile 写入文件 func writeToFile(scanRecords ScanRecordArray, config *Config) { siteCfg := RetrieveSiteCfg(config) outputFile := siteCfg.IPOutputFile f, err := os.Create(outputFile) if err != nil { - slog.Error("Failed to create file", err) + slog.Error("Failed to create file", "error", err) + return } + defer f.Close() + w := bufio.NewWriter(f) for _, record := range scanRecords { - _, err := w.WriteString(record.IP + "\n") - if err != nil { - slog.Error("write to output file failed", "error", err) + if _, err := w.WriteString(record.IP + "\n"); err != nil { + slog.Error("Write to output file failed", "error", err) } } - err = w.Flush() + + if err := w.Flush(); err != nil { + slog.Error("Flush failed", "error", err) + } +} + +// writeToHosts 写入hosts文件 +func writeToHosts(ip string, domains []string) { + var hostsFile string + switch runtime.GOOS { + case "windows": + hostsFile = "C:\\Windows\\System32\\drivers\\etc\\hosts" + case "darwin": + hostsFile = "/private/etc/hosts" + case "linux": + hostsFile = "/etc/hosts" + default: + slog.Info("Unknown operating system, please configure hosts manually") + return + } + + if err := backupHosts(hostsFile); err != nil { + slog.Error("Backup hosts failed", "error", err) + return + } + + if err := modifyHosts(hostsFile, ip, domains); err != nil { + slog.Error("Modify hosts failed", "error", err) + return + } + + slog.Info("Successfully written to hosts file") +} + +// backupHosts 备份hosts文件 +func backupHosts(hostsFile string) error { + return Copy(hostsFile, "hosts.backup") +} + +// Copy 文件复制 +func Copy(src, dst string) error { + sourceFile, err := os.Open(src) + if err != nil { + return err + } + defer sourceFile.Close() + + destFile, err := os.Create(dst) if err != nil { - slog.Error("flush failed", "error", err) + return err } + defer destFile.Close() + + _, err = io.Copy(destFile, sourceFile) + return err +} + +// modifyHosts 修改hosts文件 +func modifyHosts(hostsFile string, ip string, domains []string) error { + content, err := os.ReadFile(hostsFile) + if err != nil { + return err + } + + lines := strings.Split(string(content), "\n") + var newLines []string + gLabel := "# Google Translate" + + // 删除旧的条目 + for _, line := range lines { + if !strContainSlice(line, domains) && !strings.Contains(line, gLabel) { + newLines = append(newLines, line) + } + } + + // 添加新的条目 + newLines = append(newLines, "", gLabel) + for _, domain := range domains { + newLines = append(newLines, fmt.Sprintf("%s %s", ip, domain)) + } + + // 写入文件 + return os.WriteFile(hostsFile, []byte(strings.Join(newLines, "\n")), 0644) } func printResult(scanRecords ScanRecordArray, config *Config) { @@ -139,7 +364,7 @@ func printResult(scanRecords ScanRecordArray, config *Config) { customIPRangesFile := site.CustomIPRangesFile _, err := os.Stat(customIPRangesFile) if err == nil { - log.Printf("No available ip found! Please delete the %s file and re-run will scan all IP ranges", customIPRangesFile) + slog.Error("No available ip found! Please delete the %s file and re-run will scan all IP ranges", customIPRangesFile) } else { slog.Info("No available ip found!") } @@ -179,99 +404,15 @@ func askForConfirmation() bool { } } -func writeToHosts(ip string, domains []string) { - var hostsFile string - switch runtime.GOOS { - case "windows": - hostsFile = "C:\\Windows\\System32\\drivers\\etc\\hosts" - case "darwin": - hostsFile = "/private/etc/hosts" - case "linux": - hostsFile = "/etc/hosts" - default: - slog.Info("Your operating system is unknown, please configure hosts yourself.") - return - } - backupPath := "hosts" - err := Copy(hostsFile, backupPath) - if err != nil { - slog.Error("Backup hosts failed, please modify the hosts file yourself.", err) - return - } - err = modifyHosts(hostsFile, ip, domains) - if err != nil { - slog.Error("Modify hosts failed, please modify the hosts file yourself.", err) - return - } - slog.Info("Successfully written to hosts file") -} - -func Copy(srcPath, dstPath string) (err error) { - r, err := os.Open(srcPath) - if err != nil { - return err - } - // ignore error: file was opened read-only. - defer r.Close() - w, err := os.Create(dstPath) - if err != nil { - return err - } - defer func() { - err := w.Close() - if err != nil { - } - }() - _, err = io.Copy(w, r) - return err -} - -// modifyHosts: only use for Google Translate -func modifyHosts(hostsFile string, ip string, domains []string) error { - f, err := os.OpenFile(hostsFile, os.O_RDWR, 0o644) - if err != nil { - return err - } - defer f.Close() - - lineSeparator := "\n" - if runtime.GOOS == "windows" { - lineSeparator = "\r\n" - } - var builder strings.Builder - scanner := bufio.NewScanner(f) - // var gLabel []string - // gLabel = append(gLabel, "# Google翻译") - gLabel := []string{"# Google翻译"} - for scanner.Scan() { - line := scanner.Text() - if strContainSlice(line, domains) || strContainSlice(line, gLabel) { - continue +// strContainSlice 检查字符串是否包含切片中的任何元素 +func strContainSlice(s string, ls []string) bool { + for _, substr := range ls { + if strings.Contains(s, substr) { + return true } - builder.WriteString(line + lineSeparator) } - builder.WriteString(lineSeparator) - - builder.WriteString("# Google翻译\n") - for _, domain := range domains { - line := fmt.Sprintf("%s %s", ip, domain) + lineSeparator - builder.WriteString(line) - } - err = f.Truncate(0) - if err != nil { - return err - } - _, err = f.Seek(0, 0) - if err != nil { - return err - } - _, err = f.WriteString(builder.String()) - if err != nil { - return err - } - return nil + return false } - func AssertSiteName(config *Config) bool { siteName := config.General.Site for _, site := range config.Sites { @@ -281,12 +422,3 @@ func AssertSiteName(config *Config) bool { } return false } - -func strContainSlice(s string, ls []string) bool { - for _, substr := range ls { - if strings.Contains(s, substr) { - return true - } - } - return false -} diff --git a/configs/config.toml b/configs/config.toml index 57d3677..e9ba154 100644 --- a/configs/config.toml +++ b/configs/config.toml @@ -4,7 +4,7 @@ Site = "GoogleTranslate" # A boolean that turns on/off debug mode. true or false Debug = false # workers -Workers = 300 +Workers = 1000 # Limit the maximum number of IPs scanned. No limit if it is less than or equal to 0. ScannedLimit = 0 # Limit the maximum number of IPs found. No limit if it is less than or equal to 0. @@ -20,7 +20,7 @@ Count = 3 # Millisecond Timeout = 500 # true: it's legal if it succeeds every time. false: it's legal if it has one succeeds -all = true +all = false [HTTP] # Standard HTTPS ports are 443 and 8443. diff --git a/data/all_google_translate_ip_ranges.txt b/data/all_google_translate_ip_ranges.txt index 5824fe3..b5548d1 100644 --- a/data/all_google_translate_ip_ranges.txt +++ b/data/all_google_translate_ip_ranges.txt @@ -1,43 +1,12 @@ -8.8.4.0/24 -8.8.8.0/24 -8.34.208.0/20 -8.35.192.0/20 -23.236.48.0/20 -23.251.128.0/19 -34.0.0.0/15 -34.2.0.0/16 -34.3.0.0/23 -34.3.3.0/24 -34.3.4.0/24 -34.3.8.0/21 -34.3.16.0/20 -34.3.32.0/19 -34.3.64.0/18 -34.3.128.0/17 -34.4.0.0/14 -34.8.0.0/13 -34.16.0.0/12 -34.32.0.0/11 -34.64.0.0/10 -34.128.0.0/10 -35.184.0.0/13 -35.192.0.0/14 -35.196.0.0/15 -35.198.0.0/16 -35.199.0.0/17 -35.199.128.0/18 -35.200.0.0/13 -35.208.0.0/12 -35.224.0.0/12 -35.240.0.0/13 +35.184.0.0/20 +35.192.0.0/20 64.15.112.0/20 -64.233.160.0/19 + 66.22.228.0/23 -66.102.0.0/20 + 66.249.64.0/19 70.32.128.0/19 72.14.192.0/18 -74.125.0.0/16 104.154.0.0/15 104.196.0.0/14 104.237.160.0/19 @@ -45,10 +14,28 @@ 107.178.192.0/18 108.59.80.0/20 108.170.192.0/18 -108.177.0.0/17 130.211.0.0/16 +136.22.160.0/20 +136.22.176.0/21 +136.22.184.0/23 +136.22.186.0/24 +35.196.0.0/15 +35.198.0.0/16 +35.199.0.0/17 +35.199.128.0/18 +35.200.0.0/13 +35.208.0.0/12 +35.224.0.0/12 +35.240.0.0/13 +136.124.0.0/15 142.250.0.0/15 146.148.0.0/17 +152.65.208.0/22 +152.65.214.0/23 +152.65.218.0/23 +152.65.222.0/23 +152.65.224.0/19 +162.120.128.0/17 162.216.148.0/22 162.222.176.0/21 172.110.32.0/21 @@ -76,7 +63,11 @@ 2404:6800::/32 2404:f340::/32 2600:1900::/28 +2605:ef80::/32 +2606:40::/32 2606:73c0::/32 +2607:1c0:241:40::/60 +2607:1c0:300::/40 2607:f8b0::/32 2620:11a:a000::/40 2620:120:e000::/40 diff --git a/data/custom_google_translate_ip_ranges.txt b/data/custom_google_translate_ip_ranges.txt index a084556..bc82431 100644 --- a/data/custom_google_translate_ip_ranges.txt +++ b/data/custom_google_translate_ip_ranges.txt @@ -2,4 +2,12 @@ 64.233.160.0/19 108.177.0.0/17 142.250.0.0/15 -74.125.0.0/16 \ No newline at end of file +74.125.0.0/16 +34.128.8.0/24 +35.228.0.0/16 +66.102.1.0/16 +172.217.0.0/16 +172.253.0.0/16 +173.194.0.0/16 +209.85.192.0/18 +216.58.192.0/18