using System; using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Xml.Linq; using Newtonsoft.Json.Linq; namespace LocalizationTool.Services { /// /// AI 助手联网查资料(web_search / fetch_url 两个工具的实现)。 /// 搜索后端由配置选择: /// bing = 必应搜索 RSS(默认,免 Key,国内可直连) /// bocha = 博查 AI 搜索(api.bochaai.com,需 ApiKey,国内服务) /// tavily = Tavily(api.tavily.com,需 ApiKey,海外服务) /// public class WebSearchProvider { private const string UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"; private readonly Config.WebSearchConfig _cfg; public WebSearchProvider(Config.WebSearchConfig cfg) { _cfg = cfg ?? new Config.WebSearchConfig(); } /// bing 后端免 Key;其余后端要求填 ApiKey。 public bool IsConfigured => string.Equals(_cfg.Provider, "bing", StringComparison.OrdinalIgnoreCase) || !string.IsNullOrWhiteSpace(_cfg.ApiKey); /// 联网搜索:返回 [{title, url, snippet}]。query 为必应时支持中英文。 public async Task SearchAsync(string query, int count) { if (string.IsNullOrWhiteSpace(query)) throw new ArgumentException("搜索关键词不能为空"); if (!IsConfigured) throw new InvalidOperationException($"搜索后端 {_cfg.Provider} 需要 ApiKey,请在配置文件 WebSearch 段填写"); count = Math.Min(Math.Max(count, 1), 10); switch ((_cfg.Provider ?? "").Trim().ToLowerInvariant()) { case "bocha": return await SearchBochaAsync(query, count).ConfigureAwait(false); case "tavily": return await SearchTavilyAsync(query, count).ConfigureAwait(false); default: return await SearchBingRssAsync(query, count).ConfigureAwait(false); } } /// 必应搜索 RSS 输出:免 Key、结构稳定,直接解析 XML。 private static async Task SearchBingRssAsync(string query, int count) { var url = $"https://www.bing.com/search?q={Uri.EscapeDataString(query)}&format=rss&count={count}"; using (var client = new HttpClient()) { client.Timeout = TimeSpan.FromSeconds(20); client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent); using (var resp = await client.GetAsync(url).ConfigureAwait(false)) { resp.EnsureSuccessStatusCode(); var xml = await resp.Content.ReadAsStringAsync().ConfigureAwait(false); var doc = XDocument.Parse(xml); var ns = doc.Root?.Name.Namespace ?? XNamespace.None; var arr = new JArray(); foreach (var item in doc.Descendants(ns + "item").Take(count)) { arr.Add(new JObject { ["title"] = (string)item.Element(ns + "title"), ["url"] = (string)item.Element(ns + "link"), ["snippet"] = (string)item.Element(ns + "description"), }); } if (arr.Count == 0) throw new Exception("搜索无结果或被必应拦截,请稍后重试或更换关键词"); return arr; } } } /// 博查 AI 搜索(国内,需 Key):https://open.bochaai.com private async Task SearchBochaAsync(string query, int count) { var payload = new JObject { ["query"] = query, ["count"] = count, ["summary"] = true, // 返回网页正文摘要,token 友好 }; var json = await PostJsonAsync("https://api.bochaai.com/v1/web-search", payload).ConfigureAwait(false); var arr = new JArray(); var pages = json["data"]?["webPages"]?["value"] as JArray; foreach (var p in (pages ?? new JArray()).Take(count)) { arr.Add(new JObject { ["title"] = (string)p["name"], ["url"] = (string)p["url"], ["snippet"] = (string)p["summary"] ?? (string)p["snippet"], }); } return arr; } /// Tavily 搜索(海外,需 Key):https://tavily.com private async Task SearchTavilyAsync(string query, int count) { var payload = new JObject { ["query"] = query, ["max_results"] = count, }; var json = await PostJsonAsync("https://api.tavily.com/search", payload).ConfigureAwait(false); var arr = new JArray(); foreach (var p in (json["results"] as JArray ?? new JArray()).Take(count)) { arr.Add(new JObject { ["title"] = (string)p["title"], ["url"] = (string)p["url"], ["snippet"] = (string)p["content"], }); } return arr; } private async Task PostJsonAsync(string url, JObject payload) { using (var client = new HttpClient()) { client.Timeout = TimeSpan.FromSeconds(30); client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _cfg.ApiKey); var body = new StringContent(payload.ToString(), Encoding.UTF8, "application/json"); using (var resp = await client.PostAsync(url, body).ConfigureAwait(false)) { var text = await resp.Content.ReadAsStringAsync().ConfigureAwait(false); if (!resp.IsSuccessStatusCode) throw new Exception($"搜索请求失败: {(int)resp.StatusCode} {Truncate(text, 300)}"); return JObject.Parse(text); } } } /// /// 抓取网页并提取正文文本(去掉 script/style/标签),供模型直接阅读。 /// 自动识别 UTF-8 / GBK 等编码;返回文本截断到 maxChars。 /// public async Task FetchUrlAsync(string url, int maxChars = 6000) { if (string.IsNullOrWhiteSpace(url) || !url.StartsWith("http", StringComparison.OrdinalIgnoreCase)) throw new ArgumentException("url 必须以 http/https 开头"); using (var client = new HttpClient()) { client.Timeout = TimeSpan.FromSeconds(25); client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent); client.DefaultRequestHeaders.Accept.ParseAdd("text/html,application/xhtml+xml,*/*"); using (var resp = await client.GetAsync(url, HttpCompletionOption.ResponseContentRead).ConfigureAwait(false)) { if (!resp.IsSuccessStatusCode) throw new Exception($"网页请求失败: {(int)resp.StatusCode} {resp.RequestMessage?.RequestUri}"); var bytes = await resp.Content.ReadAsByteArrayAsync().ConfigureAwait(false); var html = DecodeHtml(bytes, resp.Content.Headers.ContentType?.CharSet); var text = HtmlToText(html); if (string.IsNullOrWhiteSpace(text)) throw new Exception("网页没有可提取的正文(可能是纯脚本渲染的页面)"); return Truncate(text, maxChars); } } } /// 按 Content-Type 头 → 页面 meta 声明 → UTF-8 的顺序确定编码;gb2312 升级为 gb18030。 private static string DecodeHtml(byte[] bytes, string headerCharset) { var head = Encoding.ASCII.GetString(bytes, 0, Math.Min(bytes.Length, 4096)); var charset = headerCharset; if (string.IsNullOrWhiteSpace(charset)) { var m = Regex.Match(head, @"]+charset\s*=\s*[""']?\s*([\w-]+)", RegexOptions.IgnoreCase); if (m.Success) charset = m.Groups[1].Value; } try { if (!string.IsNullOrWhiteSpace(charset)) { charset = charset.Trim().ToLowerInvariant(); if (charset == "gb2312" || charset == "gbk") charset = "gb18030"; // gb2312 是 gb18030 子集,避免生僻字乱码 if (charset != "utf-8" && charset != "utf8") return Encoding.GetEncoding(charset).GetString(bytes); } } catch { } // 未知编码名 → 回落 UTF-8 return Encoding.UTF8.GetString(bytes); } /// HTML → 纯文本:去脚本/样式/注释,块级标签转换行,解码实体,压缩空白。 private static string HtmlToText(string html) { html = Regex.Replace(html, "", " ", RegexOptions.IgnoreCase); html = Regex.Replace(html, "", " ", RegexOptions.IgnoreCase); html = Regex.Replace(html, "", " "); html = Regex.Replace(html, "<(br|/p|/div|/li|/tr|/h[1-6])[^>]*>", "\n", RegexOptions.IgnoreCase); html = Regex.Replace(html, "<[^>]+>", " "); html = WebUtility.HtmlDecode(html); html = Regex.Replace(html, "[ \\t\\r\\f\\v]+", " "); html = Regex.Replace(html, " ?\\n ?", "\n"); html = Regex.Replace(html, "\\n{3,}", "\n\n"); return html.Trim(); } private static string Truncate(string s, int max) => string.IsNullOrEmpty(s) || s.Length <= max ? s : s.Substring(0, max) + "…(已截断)"; } }