TeamAAS-2.5/Tools/LocalizationTool/Services/WebSearchProvider.cs

using System;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml.Linq;
using Newtonsoft.Json.Linq;

namespace LocalizationTool.Services
{
    /// <summary>
    /// AI 助手联网查资料（web_search / fetch_url 两个工具的实现）。
    /// 搜索后端由配置选择：
    ///   bing   = 必应搜索 RSS（默认，免 Key，国内可直连）
    ///   bocha  = 博查 AI 搜索（api.bochaai.com，需 ApiKey，国内服务）
    ///   tavily = Tavily（api.tavily.com，需 ApiKey，海外服务）
    /// </summary>
    public class WebSearchProvider
    {
        private const string UserAgent =
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36";

        private readonly Config.WebSearchConfig _cfg;

        public WebSearchProvider(Config.WebSearchConfig cfg)
        {
            _cfg = cfg ?? new Config.WebSearchConfig();
        }

        /// <summary>bing 后端免 Key；其余后端要求填 ApiKey。</summary>
        public bool IsConfigured =>
            string.Equals(_cfg.Provider, "bing", StringComparison.OrdinalIgnoreCase)
            || !string.IsNullOrWhiteSpace(_cfg.ApiKey);

        /// <summary>联网搜索：返回 [{title, url, snippet}]。query 为必应时支持中英文。</summary>
        public async Task<JArray> SearchAsync(string query, int count)
        {
            if (string.IsNullOrWhiteSpace(query))
                throw new ArgumentException("搜索关键词不能为空");
            if (!IsConfigured)
                throw new InvalidOperationException($"搜索后端 {_cfg.Provider} 需要 ApiKey，请在配置文件 WebSearch 段填写");

            count = Math.Min(Math.Max(count, 1), 10);

            switch ((_cfg.Provider ?? "").Trim().ToLowerInvariant())
            {
                case "bocha": return await SearchBochaAsync(query, count).ConfigureAwait(false);
                case "tavily": return await SearchTavilyAsync(query, count).ConfigureAwait(false);
                default: return await SearchBingRssAsync(query, count).ConfigureAwait(false);
            }
        }

        /// <summary>必应搜索 RSS 输出：免 Key、结构稳定，直接解析 XML。</summary>
        private static async Task<JArray> SearchBingRssAsync(string query, int count)
        {
            var url = $"https://www.bing.com/search?q={Uri.EscapeDataString(query)}&format=rss&count={count}";
            using (var client = new HttpClient())
            {
                client.Timeout = TimeSpan.FromSeconds(20);
                client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent);

                using (var resp = await client.GetAsync(url).ConfigureAwait(false))
                {
                    resp.EnsureSuccessStatusCode();
                    var xml = await resp.Content.ReadAsStringAsync().ConfigureAwait(false);
                    var doc = XDocument.Parse(xml);
                    var ns = doc.Root?.Name.Namespace ?? XNamespace.None;

                    var arr = new JArray();
                    foreach (var item in doc.Descendants(ns + "item").Take(count))
                    {
                        arr.Add(new JObject
                        {
                            ["title"] = (string)item.Element(ns + "title"),
                            ["url"] = (string)item.Element(ns + "link"),
                            ["snippet"] = (string)item.Element(ns + "description"),
                        });
                    }
                    if (arr.Count == 0)
                        throw new Exception("搜索无结果或被必应拦截，请稍后重试或更换关键词");
                    return arr;
                }
            }
        }

        /// <summary>博查 AI 搜索（国内，需 Key）：https://open.bochaai.com</summary>
        private async Task<JArray> SearchBochaAsync(string query, int count)
        {
            var payload = new JObject
            {
                ["query"] = query,
                ["count"] = count,
                ["summary"] = true,   // 返回网页正文摘要，token 友好
            };
            var json = await PostJsonAsync("https://api.bochaai.com/v1/web-search", payload).ConfigureAwait(false);

            var arr = new JArray();
            var pages = json["data"]?["webPages"]?["value"] as JArray;
            foreach (var p in (pages ?? new JArray()).Take(count))
            {
                arr.Add(new JObject
                {
                    ["title"] = (string)p["name"],
                    ["url"] = (string)p["url"],
                    ["snippet"] = (string)p["summary"] ?? (string)p["snippet"],
                });
            }
            return arr;
        }

        /// <summary>Tavily 搜索（海外，需 Key）：https://tavily.com</summary>
        private async Task<JArray> SearchTavilyAsync(string query, int count)
        {
            var payload = new JObject
            {
                ["query"] = query,
                ["max_results"] = count,
            };
            var json = await PostJsonAsync("https://api.tavily.com/search", payload).ConfigureAwait(false);

            var arr = new JArray();
            foreach (var p in (json["results"] as JArray ?? new JArray()).Take(count))
            {
                arr.Add(new JObject
                {
                    ["title"] = (string)p["title"],
                    ["url"] = (string)p["url"],
                    ["snippet"] = (string)p["content"],
                });
            }
            return arr;
        }

        private async Task<JObject> PostJsonAsync(string url, JObject payload)
        {
            using (var client = new HttpClient())
            {
                client.Timeout = TimeSpan.FromSeconds(30);
                client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent);
                client.DefaultRequestHeaders.Authorization =
                    new AuthenticationHeaderValue("Bearer", _cfg.ApiKey);

                var body = new StringContent(payload.ToString(), Encoding.UTF8, "application/json");
                using (var resp = await client.PostAsync(url, body).ConfigureAwait(false))
                {
                    var text = await resp.Content.ReadAsStringAsync().ConfigureAwait(false);
                    if (!resp.IsSuccessStatusCode)
                        throw new Exception($"搜索请求失败: {(int)resp.StatusCode} {Truncate(text, 300)}");
                    return JObject.Parse(text);
                }
            }
        }

        /// <summary>
        /// 抓取网页并提取正文文本（去掉 script/style/标签），供模型直接阅读。
        /// 自动识别 UTF-8 / GBK 等编码；返回文本截断到 maxChars。
        /// </summary>
        public async Task<string> FetchUrlAsync(string url, int maxChars = 6000)
        {
            if (string.IsNullOrWhiteSpace(url) || !url.StartsWith("http", StringComparison.OrdinalIgnoreCase))
                throw new ArgumentException("url 必须以 http/https 开头");

            using (var client = new HttpClient())
            {
                client.Timeout = TimeSpan.FromSeconds(25);
                client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent);
                client.DefaultRequestHeaders.Accept.ParseAdd("text/html,application/xhtml+xml,*/*");

                using (var resp = await client.GetAsync(url, HttpCompletionOption.ResponseContentRead).ConfigureAwait(false))
                {
                    if (!resp.IsSuccessStatusCode)
                        throw new Exception($"网页请求失败: {(int)resp.StatusCode} {resp.RequestMessage?.RequestUri}");

                    var bytes = await resp.Content.ReadAsByteArrayAsync().ConfigureAwait(false);
                    var html = DecodeHtml(bytes, resp.Content.Headers.ContentType?.CharSet);
                    var text = HtmlToText(html);

                    if (string.IsNullOrWhiteSpace(text))
                        throw new Exception("网页没有可提取的正文（可能是纯脚本渲染的页面）");
                    return Truncate(text, maxChars);
                }
            }
        }

        /// <summary>按 Content-Type 头 → 页面 meta 声明 → UTF-8 的顺序确定编码；gb2312 升级为 gb18030。</summary>
        private static string DecodeHtml(byte[] bytes, string headerCharset)
        {
            var head = Encoding.ASCII.GetString(bytes, 0, Math.Min(bytes.Length, 4096));
            var charset = headerCharset;
            if (string.IsNullOrWhiteSpace(charset))
            {
                var m = Regex.Match(head, @"<meta[^>]+charset\s*=\s*[""']?\s*([\w-]+)", RegexOptions.IgnoreCase);
                if (m.Success) charset = m.Groups[1].Value;
            }

            try
            {
                if (!string.IsNullOrWhiteSpace(charset))
                {
                    charset = charset.Trim().ToLowerInvariant();
                    if (charset == "gb2312" || charset == "gbk") charset = "gb18030";   // gb2312 是 gb18030 子集，避免生僻字乱码
                    if (charset != "utf-8" && charset != "utf8")
                        return Encoding.GetEncoding(charset).GetString(bytes);
                }
            }
            catch { }   // 未知编码名 → 回落 UTF-8

            return Encoding.UTF8.GetString(bytes);
        }

        /// <summary>HTML → 纯文本：去脚本/样式/注释，块级标签转换行，解码实体，压缩空白。</summary>
        private static string HtmlToText(string html)
        {
            html = Regex.Replace(html, "<script[\\s\\S]*?</script>", " ", RegexOptions.IgnoreCase);
            html = Regex.Replace(html, "<style[\\s\\S]*?</style>", " ", RegexOptions.IgnoreCase);
            html = Regex.Replace(html, "<!--[\\s\\S]*?-->", " ");
            html = Regex.Replace(html, "<(br|/p|/div|/li|/tr|/h[1-6])[^>]*>", "\n", RegexOptions.IgnoreCase);
            html = Regex.Replace(html, "<[^>]+>", " ");
            html = WebUtility.HtmlDecode(html);
            html = Regex.Replace(html, "[ \\t\\r\\f\\v]+", " ");
            html = Regex.Replace(html, " ?\\n ?", "\n");
            html = Regex.Replace(html, "\\n{3,}", "\n\n");
            return html.Trim();
        }

        private static string Truncate(string s, int max) =>
            string.IsNullOrEmpty(s) || s.Length <= max ? s : s.Substring(0, max) + "…(已截断)";
    }
}