using System;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml.Linq;
using Newtonsoft.Json.Linq;
namespace LocalizationTool.Services
{
///
/// AI 助手联网查资料(web_search / fetch_url 两个工具的实现)。
/// 搜索后端由配置选择:
/// bing = 必应搜索 RSS(默认,免 Key,国内可直连)
/// bocha = 博查 AI 搜索(api.bochaai.com,需 ApiKey,国内服务)
/// tavily = Tavily(api.tavily.com,需 ApiKey,海外服务)
///
public class WebSearchProvider
{
private const string UserAgent =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36";
private readonly Config.WebSearchConfig _cfg;
public WebSearchProvider(Config.WebSearchConfig cfg)
{
_cfg = cfg ?? new Config.WebSearchConfig();
}
/// bing 后端免 Key;其余后端要求填 ApiKey。
public bool IsConfigured =>
string.Equals(_cfg.Provider, "bing", StringComparison.OrdinalIgnoreCase)
|| !string.IsNullOrWhiteSpace(_cfg.ApiKey);
/// 联网搜索:返回 [{title, url, snippet}]。query 为必应时支持中英文。
public async Task SearchAsync(string query, int count)
{
if (string.IsNullOrWhiteSpace(query))
throw new ArgumentException("搜索关键词不能为空");
if (!IsConfigured)
throw new InvalidOperationException($"搜索后端 {_cfg.Provider} 需要 ApiKey,请在配置文件 WebSearch 段填写");
count = Math.Min(Math.Max(count, 1), 10);
switch ((_cfg.Provider ?? "").Trim().ToLowerInvariant())
{
case "bocha": return await SearchBochaAsync(query, count).ConfigureAwait(false);
case "tavily": return await SearchTavilyAsync(query, count).ConfigureAwait(false);
default: return await SearchBingRssAsync(query, count).ConfigureAwait(false);
}
}
/// 必应搜索 RSS 输出:免 Key、结构稳定,直接解析 XML。
private static async Task SearchBingRssAsync(string query, int count)
{
var url = $"https://www.bing.com/search?q={Uri.EscapeDataString(query)}&format=rss&count={count}";
using (var client = new HttpClient())
{
client.Timeout = TimeSpan.FromSeconds(20);
client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent);
using (var resp = await client.GetAsync(url).ConfigureAwait(false))
{
resp.EnsureSuccessStatusCode();
var xml = await resp.Content.ReadAsStringAsync().ConfigureAwait(false);
var doc = XDocument.Parse(xml);
var ns = doc.Root?.Name.Namespace ?? XNamespace.None;
var arr = new JArray();
foreach (var item in doc.Descendants(ns + "item").Take(count))
{
arr.Add(new JObject
{
["title"] = (string)item.Element(ns + "title"),
["url"] = (string)item.Element(ns + "link"),
["snippet"] = (string)item.Element(ns + "description"),
});
}
if (arr.Count == 0)
throw new Exception("搜索无结果或被必应拦截,请稍后重试或更换关键词");
return arr;
}
}
}
/// 博查 AI 搜索(国内,需 Key):https://open.bochaai.com
private async Task SearchBochaAsync(string query, int count)
{
var payload = new JObject
{
["query"] = query,
["count"] = count,
["summary"] = true, // 返回网页正文摘要,token 友好
};
var json = await PostJsonAsync("https://api.bochaai.com/v1/web-search", payload).ConfigureAwait(false);
var arr = new JArray();
var pages = json["data"]?["webPages"]?["value"] as JArray;
foreach (var p in (pages ?? new JArray()).Take(count))
{
arr.Add(new JObject
{
["title"] = (string)p["name"],
["url"] = (string)p["url"],
["snippet"] = (string)p["summary"] ?? (string)p["snippet"],
});
}
return arr;
}
/// Tavily 搜索(海外,需 Key):https://tavily.com
private async Task SearchTavilyAsync(string query, int count)
{
var payload = new JObject
{
["query"] = query,
["max_results"] = count,
};
var json = await PostJsonAsync("https://api.tavily.com/search", payload).ConfigureAwait(false);
var arr = new JArray();
foreach (var p in (json["results"] as JArray ?? new JArray()).Take(count))
{
arr.Add(new JObject
{
["title"] = (string)p["title"],
["url"] = (string)p["url"],
["snippet"] = (string)p["content"],
});
}
return arr;
}
private async Task PostJsonAsync(string url, JObject payload)
{
using (var client = new HttpClient())
{
client.Timeout = TimeSpan.FromSeconds(30);
client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent);
client.DefaultRequestHeaders.Authorization =
new AuthenticationHeaderValue("Bearer", _cfg.ApiKey);
var body = new StringContent(payload.ToString(), Encoding.UTF8, "application/json");
using (var resp = await client.PostAsync(url, body).ConfigureAwait(false))
{
var text = await resp.Content.ReadAsStringAsync().ConfigureAwait(false);
if (!resp.IsSuccessStatusCode)
throw new Exception($"搜索请求失败: {(int)resp.StatusCode} {Truncate(text, 300)}");
return JObject.Parse(text);
}
}
}
///
/// 抓取网页并提取正文文本(去掉 script/style/标签),供模型直接阅读。
/// 自动识别 UTF-8 / GBK 等编码;返回文本截断到 maxChars。
///
public async Task FetchUrlAsync(string url, int maxChars = 6000)
{
if (string.IsNullOrWhiteSpace(url) || !url.StartsWith("http", StringComparison.OrdinalIgnoreCase))
throw new ArgumentException("url 必须以 http/https 开头");
using (var client = new HttpClient())
{
client.Timeout = TimeSpan.FromSeconds(25);
client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent);
client.DefaultRequestHeaders.Accept.ParseAdd("text/html,application/xhtml+xml,*/*");
using (var resp = await client.GetAsync(url, HttpCompletionOption.ResponseContentRead).ConfigureAwait(false))
{
if (!resp.IsSuccessStatusCode)
throw new Exception($"网页请求失败: {(int)resp.StatusCode} {resp.RequestMessage?.RequestUri}");
var bytes = await resp.Content.ReadAsByteArrayAsync().ConfigureAwait(false);
var html = DecodeHtml(bytes, resp.Content.Headers.ContentType?.CharSet);
var text = HtmlToText(html);
if (string.IsNullOrWhiteSpace(text))
throw new Exception("网页没有可提取的正文(可能是纯脚本渲染的页面)");
return Truncate(text, maxChars);
}
}
}
/// 按 Content-Type 头 → 页面 meta 声明 → UTF-8 的顺序确定编码;gb2312 升级为 gb18030。
private static string DecodeHtml(byte[] bytes, string headerCharset)
{
var head = Encoding.ASCII.GetString(bytes, 0, Math.Min(bytes.Length, 4096));
var charset = headerCharset;
if (string.IsNullOrWhiteSpace(charset))
{
var m = Regex.Match(head, @"]+charset\s*=\s*[""']?\s*([\w-]+)", RegexOptions.IgnoreCase);
if (m.Success) charset = m.Groups[1].Value;
}
try
{
if (!string.IsNullOrWhiteSpace(charset))
{
charset = charset.Trim().ToLowerInvariant();
if (charset == "gb2312" || charset == "gbk") charset = "gb18030"; // gb2312 是 gb18030 子集,避免生僻字乱码
if (charset != "utf-8" && charset != "utf8")
return Encoding.GetEncoding(charset).GetString(bytes);
}
}
catch { } // 未知编码名 → 回落 UTF-8
return Encoding.UTF8.GetString(bytes);
}
/// HTML → 纯文本:去脚本/样式/注释,块级标签转换行,解码实体,压缩空白。
private static string HtmlToText(string html)
{
html = Regex.Replace(html, "