using System.Text;
using System.Web;
using NewLife;
using NewLife.Caching;
using NewLife.Log;
using NewLife.Threading;
using Stardust.Data;
using Stardust.Data.Monitors;
using Stardust.Data.Nodes;

namespace Stardust.Server.Services;

//public interface IAlarmService
//{
//    /// <summary>添加需要统计的应用，去重</summary>
//    /// <param name="appId"></param>
//    void Add(Int32 appId);
//}

public class AlarmService : IHostedService
{
    /// <summary>计算周期。默认30秒</summary>
    public Int32 Period { get; set; } = 30;

    private TimerX _timer;
    private readonly ICache _cache;
    private readonly StarServerSetting _setting;
    private readonly ITracer _tracer;

    public AlarmService(StarServerSetting setting, ICacheProvider cacheProvider, ITracer tracer)
    {
        _setting = setting;
        _cache = cacheProvider.Cache;
        _tracer = tracer;

        Period = setting.AlarmPeriod;
    }

    public Task StartAsync(CancellationToken cancellationToken)
    {
        // 初始化定时器
        _timer = new TimerX(DoAlarm, null, 5_000, Period * 1000) { Async = true };

        return Task.CompletedTask;
    }

    public Task StopAsync(CancellationToken cancellationToken)
    {
        _timer.TryDispose();

        return Task.CompletedTask;
    }

    ///// <summary>添加需要统计的应用，去重</summary>
    ///// <param name="appId"></param>
    //public void Add(Int32 appId)
    //{
    //    if (!_bag.Contains(appId)) _bag.Add(appId);
    //}

    private void DoAlarm(Object state)
    {
        //while (_bag.TryTake(out var appId))
        //{
        //    //Process(appId);
        //}

        // 应用告警
        var list = AppTracer.FindAllWithCache();
        foreach (var item in list)
        {
            ProcessAppTracer(item);
            ProcessTraceItem(item);
            ProcessRingRate(item);
        }

        // 节点告警
        //var nodes = Node.FindAllWithCache();
        //foreach (var item in nodes)
        //{
        //    ProcessNode(item);
        //}
        var onlines = NodeOnline.FindAll();
        foreach (var item in onlines)
        {
            if (item.Node != null) ProcessNode(item.Node);
        }

        // Redis告警
        var rnodes = RedisNode.FindAllWithCache();
        foreach (var item in rnodes)
        {
            ProcessRedisNode(item);
        }

        if (Period > 0) _timer.Period = Period * 1000;
    }

    #region 应用性能追踪告警
    private void ProcessAppTracer(AppTracer app)
    {
        // 应用是否需要告警
        if (app == null || !app.Enable) return;
        if (app.AlarmThreshold <= 0 && app.AlarmErrorRate <= 0) return;

        var appId = app.ID;
        var webhook = RobotHelper.GetAlarm(app.Project, app.Category, app.AlarmRobot);
        if (webhook.IsNullOrEmpty()) return;

        using var span = _tracer?.NewSpan($"alarm:{nameof(AppTracer)}");

        // 最近一段时间的5分钟级数据
        var time = DateTime.Now;
        var minute = time.Date.AddHours(time.Hour).AddMinutes(time.Minute / 5 * 5);
        span?.AppendTag(new { time, minute });

        var st = AppMinuteStat.FindByAppIdAndTime(appId, minute);
        if (st != null)
        {
            // 判断告警
            if (app.AlarmThreshold > 0 && st.Errors >= app.AlarmThreshold ||
                app.AlarmErrorRate > 0 && st.ErrorRate >= app.AlarmErrorRate)
            {
                span?.AppendTag(new { st.Errors, st.ErrorRate });

                // 一定时间内不要重复报错，除非错误翻倍
                var error2 = _cache.Get<Int32>("alarm:AppTracer:" + appId);
                if (error2 == 0 || st.Errors > error2 * 2)
                {
                    _cache.Set("alarm:AppTracer:" + appId, st.Errors, 5 * 60);

                    var msg = GetMarkdown(app, st, true);
                    RobotHelper.SendAlarm(app.Category ?? app.ProjectName, webhook, "应用告警", msg);
                }
            }
        }
    }

    private String GetMarkdown(AppTracer app, AppMinuteStat st, Boolean includeTitle)
    {
        var sb = new StringBuilder();
        if (includeTitle) sb.AppendLine($"### [{app}]应用告警");
        sb.AppendLine($">**时间：**<font color=\"blue\">{st.StatTime:yyyy-MM-dd HH:mm:ss}</font>");
        sb.AppendLine($">**总数：**<font color=\"red\">{st.Errors}</font>");
        sb.AppendLine($">**错误率：**<font color=\"red\">{st.ErrorRate:p2}</font>");

        var url = _setting.WebUrl;
        var appUrl = "";
        var traceUrl = "";
        if (!url.IsNullOrEmpty())
        {
            appUrl = url.EnsureEnd("/") + "Monitors/appMinuteStat?appId=" + st.AppId + "&minError=1";
            traceUrl = url.EnsureEnd("/") + "Monitors/traceMinuteStat?appId=" + st.AppId + "&minError=1";
        }

        // 找找具体接口错误
        var names = new List<String>();
        var sts = TraceMinuteStat.FindAllByAppIdAndTime(st.AppId, st.StatTime).OrderByDescending(e => e.Errors).ToList();
        foreach (var item in sts)
        {
            if (item.Errors > 0)
            {
                sb.AppendLine($">**错误：**<font color=\"red\">埋点[{item.Name}]报错[{item.Errors:n0}]次</font>[更多]({traceUrl}&itemId={item.ItemId})");

                // 相同接口的错误，不要报多次
                if (!names.Contains(item.Name))
                {
                    var ds = TraceData.Search(st.AppId, item.ItemId, "minute", item.StatTime, 20);
                    if (ds.Count > 0)
                    {
                        // 应用节点
                        var nodes = new Dictionary<String, Node>();
                        foreach (var traceData in ds.Where(e => e.Errors > 0).OrderByDescending(e => e.Errors))
                        {
                            if (!nodes.ContainsKey(traceData.ClientId))
                            {
                                var online = AppOnline.FindByClient(traceData.ClientId);
                                var node = online?.Node;
                                if (node != null) nodes[traceData.ClientId] = node;
                            }
                        }
                        if (nodes.Count > 0) sb.AppendLine($">**节点：**<font color=\"greed\">{nodes.Join(",", e => e.Value.Name)}</font>");

                        var sms = SampleData.FindAllByDataIds(ds.Select(e => e.Id).ToArray(), item.StatTime).Where(e => !e.Error.IsNullOrEmpty()).ToList();
                        if (sms.Count > 0)
                        {
                            var msg = sms[0].Error?.Trim();
                            if (!msg.IsNullOrEmpty())
                            {
                                // 错误内容取第一行，详情看更多
                                var p = msg.IndexOfAny(new[] { '\r', '\n' });
                                if (p > 0) msg = msg[..p];

                                sb.AppendLine($">内容：{msg}");

                                names.Add(item.Name);
                            }
                        }
                    }
                }
            }
        }

        var str = sb.ToString();
        if (str.Length > 1600) str = str[..1600];

        // 构造网址
        if (!appUrl.IsNullOrEmpty())
        {
            str += Environment.NewLine + $"[更多信息]({appUrl})";
        }

        return str;
    }

    private void ProcessTraceItem(AppTracer app)
    {
        if (app == null || !app.Enable) return;

        // 应用是否配置了全局跟踪项告警
        var flag = app.ItemAlarmThreshold > 0 || app.ItemAlarmErrorRate > 0;

        // 监控项单独告警
        var tis = app.TraceItems;
        if (!flag) tis = tis.Where(e => e.AlarmThreshold > 0 || e.AlarmErrorRate > 0).ToList();
        if (tis.Count > 0)
        {
            // 最近一段时间的5分钟级数据
            var time = DateTime.Now;
            var minute = time.Date.AddHours(time.Hour).AddMinutes(time.Minute / 5 * 5);

            using var span = _tracer?.NewSpan($"alarm:{nameof(TraceItem)}", new { appId = app.ID, time, minute });

            var list = TraceMinuteStat.Search(app.ID, minute, tis.Select(e => e.Id).ToArray());
            foreach (var st in list)
            {
                var ti = tis.FirstOrDefault(e => e.Id == st.ItemId);
                if (ti != null)
                {
                    var max = ti.AlarmThreshold;
                    var rate = ti.AlarmErrorRate;
                    if (max <= 0 && rate <= 0)
                    {
                        max = app.ItemAlarmThreshold;
                        rate = app.ItemAlarmErrorRate;
                    }

                    // 必须两个条件同时满足，才能告警
                    if (max > 0 && st.Errors >= max &&
                        rate > 0 && st.ErrorRate >= rate)
                    {
                        span?.AppendTag(new { st.Errors, st.ErrorRate, rate });

                        // 一定时间内不要重复报错，除非错误翻倍
                        var error2 = _cache.Get<Int32>("alarm:TraceMinuteStat:" + ti.Id);
                        if (error2 == 0 || st.Errors > error2 * 2)
                        {
                            _cache.Set("alarm:TraceMinuteStat:" + ti.Id, st.Errors, 5 * 60);

                            // 优先本地跟踪项，其次应用，最后是告警分组
                            var webhook = ti.AlarmRobot;
                            if (webhook.IsNullOrEmpty()) webhook = app.AlarmRobot;

                            var group = ti.AlarmGroup;
                            if (group.IsNullOrEmpty()) group = app.Category;

                            var msg = GetMarkdown(app, st, true);
                            RobotHelper.SendAlarm(group, webhook, "埋点告警", msg);
                        }
                    }
                }
            }
        }
    }

    private String GetMarkdown(AppTracer app, TraceMinuteStat st, Boolean includeTitle)
    {
        var sb = new StringBuilder();
        if (includeTitle) sb.AppendLine($"### [{app}]埋点告警");
        sb.AppendLine($">**时间：**<font color=\"blue\">{st.StatTime:yyyy-MM-dd HH:mm:ss}</font>");
        sb.AppendLine($">**总数：**<font color=\"red\">{st.Errors}</font>");
        sb.AppendLine($">**错误率：**<font color=\"red\">{st.ErrorRate:p2}</font>");

        var url = _setting.WebUrl;
        var traceUrl = "";
        if (!url.IsNullOrEmpty())
        {
            traceUrl = url.EnsureEnd("/") + $"Monitors/traceData?appId={st.AppId}&kind=minute&time={HttpUtility.UrlEncode(st.StatTime.ToFullString())}&itemId={st.ItemId}&minError=1";
        }

        // 找找具体接口错误
        var item = st;
        sb.AppendLine($">**错误：**<font color=\"red\">埋点[{item.Name}]报错[{item.Errors:n0}]次</font>");

        var ds = TraceData.Search(st.AppId, item.ItemId, "minute", item.StatTime, 100);
        if (ds.Count > 0)
        {
            // 应用节点
            var nodes = new Dictionary<String, Node>();
            foreach (var traceData in ds.Where(e => e.Errors > 0).OrderByDescending(e => e.Errors))
            {
                if (!nodes.ContainsKey(traceData.ClientId))
                {
                    var online = AppOnline.FindByClient(traceData.ClientId);
                    var node = online?.Node;
                    if (node != null) nodes[traceData.ClientId] = node;
                }
            }
            if (nodes.Count > 0)
            {
                var names = nodes.Select(e => e.Value.Name).Distinct().ToArray();
                sb.AppendLine($">**节点：**<font color=\"greed\">{names.Join(",")}</font>");
            }

            var sms = SampleData.FindAllByDataIds(ds.Select(e => e.Id).ToArray(), item.StatTime).Where(e => !e.Error.IsNullOrEmpty()).ToList();
            if (sms.Count > 0)
            {
                var msg = sms[0].Error?.Trim();
                if (!msg.IsNullOrEmpty())
                {
                    // 错误内容取第一行，详情看更多
                    var p = msg.IndexOfAny(new[] { '\r', '\n' });
                    if (p > 0) msg = msg[..p];

                    sb.AppendLine($">内容：{msg}");
                }
            }
        }

        var str = sb.ToString();
        if (str.Length > 1600) str = str[..1600];

        // 构造网址
        if (!traceUrl.IsNullOrEmpty())
        {
            str += Environment.NewLine + $"[更多信息]({traceUrl})";
        }

        return str;
    }

    private void ProcessRingRate(AppTracer app)
    {
        if (app == null || !app.Enable) return;

        // 监控项单独告警
        var tis = app.TraceItems.Where(e => e.MaxRingRate > 0 || e.MinRingRate > 0).ToList();
        if (tis.Count <= 0) return;

        // 最近一段时间的小时级数据
        var time = DateTime.Now;
        var hour = time.Date.AddHours(time.Hour);
        if (time.Minute < 5) return;

        using var span = _tracer?.NewSpan($"alarm:RingRate", new { app.ID, app.Name, app.DisplayName, time, hour });

        var list = TraceHourStat.Search(app.ID, -1, null, hour, hour.AddHours(1), null, null);
        foreach (var st in list)
        {
            var ti = tis.FirstOrDefault(e => e.Id == st.ItemId);
            if (ti != null && st.RingRate > 0)
            {
                var max = ti.MaxRingRate;
                var min = ti.MinRingRate;

                // 昨日基数必须大于一定值，避免分母过小导致误报
                var st2 = TraceHourStat.FindAllByStatTimeAndAppIdAndItemId(st.StatTime.AddDays(-1), st.AppId, st.ItemId).FirstOrDefault();
                var yesterday = st2 != null ? st2.Total : (st.Total / st.RingRate);
                if (yesterday > 10)
                {
                    // 根据当前小时已过去时间，折算得到新的环比率
                    var seconds = time.Minute * 60 + time.Second;
                    var rate = st.Total / (yesterday * seconds / 3600);

                    // 满足任意一个条件，都要告警
                    if (max > 0 && rate >= max ||
                        min > 0 && rate <= min)
                    {
                        span?.AppendTag(new { seconds, yesterday, rate });

                        // 一定时间内不要重复报错，除非错误翻倍
                        var error2 = _cache.Get<Double>("alarm:RingRate:" + ti.Id);
                        if (error2 == 0 || rate > error2 * 2 || rate < error2 / 2)
                        {
                            _cache.Set("alarm:RingRate:" + ti.Id, rate, 60 * 60);

                            // 优先本地跟踪项，其次应用，最后是告警分组
                            var webhook = ti.AlarmRobot;
                            if (webhook.IsNullOrEmpty()) webhook = app.AlarmRobot;

                            var group = ti.AlarmGroup;
                            if (group.IsNullOrEmpty()) group = app.Category;

                            var msg = GetMarkdown(app, st, (Int32)yesterday, rate, true);
                            RobotHelper.SendAlarm(group, webhook, "埋点告警", msg);
                        }
                    }
                }
            }
        }
    }

    private String GetMarkdown(AppTracer app, TraceHourStat st, Int32 yesterday, Double rate, Boolean includeTitle)
    {
        var sb = new StringBuilder();
        if (includeTitle) sb.AppendLine($"### [{app}]环比{(rate >= 1 ? "高调用" : "调用量下滑")}告警");
        sb.AppendLine($">**埋点：**<font color=\"blue\">{st.Name}</font>");
        sb.AppendLine($">**时间：**<font color=\"blue\">{st.StatTime:yyyy-MM-dd HH:mm:ss}</font>");
        sb.AppendLine($">**今日：**<font color=\"red\">{st.Total}</font>");
        sb.AppendLine($">**昨日：**<font color=\"red\">{yesterday}</font>");
        sb.AppendLine($">**环比：**<font color=\"red\">{st.RingRate:p2}</font>");
        sb.AppendLine($">**折算环比：**<font color=\"red\">{rate:p2}</font>");

        var url = _setting.WebUrl;
        var traceUrl = "";
        if (!url.IsNullOrEmpty())
        {
            traceUrl = url.EnsureEnd("/") + $"Monitors/traceHourStat?appId={st.AppId}&itemId={st.ItemId}";
        }

        var str = sb.ToString();
        if (str.Length > 1600) str = str[..1600];

        // 构造网址
        if (!traceUrl.IsNullOrEmpty())
        {
            str += Environment.NewLine + $"[更多信息]({traceUrl})";
        }

        return str;
    }
    #endregion

    #region 节点告警
    private void ProcessNode(Node node)
    {
        if (node == null || !node.Enable) return;

        var webhook = RobotHelper.GetAlarm(node.Project, node.Category, node.WebHook);
        if (webhook.IsNullOrEmpty()) return;

        if (node.AlarmCpuRate <= 0 && node.AlarmMemoryRate <= 0 && node.AlarmDiskRate <= 0 && node.AlarmProcesses.IsNullOrEmpty()) return;

        using var span = _tracer?.NewSpan($"alarm:{nameof(Node)}");

        // 最新数据
        var data = NodeData.FindLast(node.ID);
        if (data == null) return;

        // CPU告警
        if (node.AlarmCpuRate > 0)
        {
            var rate = data.CpuRate * 100;
            if (rate >= node.AlarmCpuRate)
            {
                // 一定时间内不要重复报错，除非错误翻倍
                var error2 = _cache.Get<Double>("alarm:CpuRate:" + node.ID);
                if (error2 == 0 || rate > error2 * 2)
                {
                    _cache.Set("alarm:CpuRate:" + node.ID, rate, 5 * 60);

                    SendAlarm("cpu", node, data, $"[{node.Name}]CPU告警");
                }
            }
        }

        // 内存告警
        if (node.AlarmMemoryRate > 0 && node.Memory > 0)
        {
            var rate = (node.Memory - data.AvailableMemory) * 100d / node.Memory;
            if (rate >= node.AlarmMemoryRate)
            {
                // 一定时间内不要重复报错，除非错误翻倍
                var error2 = _cache.Get<Double>("alarm:MemoryRate:" + node.ID);
                if (error2 == 0 || rate > error2 * 2)
                {
                    _cache.Set("alarm:MemoryRate:" + node.ID, rate, 5 * 60);

                    SendAlarm("memory", node, data, $"[{node.Name}]内存告警");
                }
            }
        }

        // 磁盘告警
        if (node.AlarmDiskRate > 0 && node.TotalSize > 0)
        {
            var rate = (node.TotalSize - data.AvailableFreeSpace) * 100d / node.TotalSize;
            if (rate >= node.AlarmDiskRate)
            {
                // 一定时间内不要重复报错，除非错误翻倍
                var error2 = _cache.Get<Double>("alarm:DiskRate:" + node.ID);
                if (error2 == 0 || rate > error2 * 2)
                {
                    _cache.Set("alarm:DiskRate:" + node.ID, rate, 5 * 60);

                    SendAlarm("disk", node, data, $"[{node.Name}]磁盘告警");
                }
            }
        }

        // TCP告警
        if (node.AlarmTcp > 0)
        {
            var tcp = data.TcpConnections;
            if (tcp < data.TcpTimeWait) tcp = data.TcpTimeWait;
            if (tcp < data.TcpCloseWait) tcp = data.TcpCloseWait;
            if (tcp >= node.AlarmTcp)
            {
                // 一定时间内不要重复报错，除非错误翻倍
                var error2 = _cache.Get<Int32>("alarm:Tcp:" + node.ID);
                if (error2 == 0 || tcp > error2 * 2)
                {
                    _cache.Set("alarm:Tcp:" + node.ID, tcp, 5 * 60);

                    SendAlarm("tcp", node, data, $"[{node.Name}]Tcp告警");
                }
            }
        }

        // 进程告警
        if (!node.AlarmProcesses.IsNullOrEmpty())
        {
            var olt = NodeOnline.FindByNodeId(node.ID);
            if (olt != null && !olt.Processes.IsNullOrEmpty())
            {
                var alarms = node.AlarmProcesses.Split(",", StringSplitOptions.RemoveEmptyEntries);
                var ps = olt.Processes?.Split(",", StringSplitOptions.RemoveEmptyEntries);
                if (alarms != null && alarms.Length > 0 && ps != null && ps.Length > 0)
                {
                    // 查找丢失的进程
                    var ps2 = alarms.Where(e => !ps.Contains(e)).ToList();
                    if (ps2.Count > 0)
                    {
                        // 一定时间内不要重复报错
                        var error2 = _cache.Get<Int32>("alarm:Process:" + node.ID);
                        if (error2 == 0 || ps2.Count > error2)
                        {
                            _cache.Set("alarm:Process:" + node.ID, ps2.Count, 5 * 60);

                            SendAlarm("process", node, data, $"[{node.Name}]进程守护告警", ps2.Join());
                        }
                    }
                }
            }
        }
    }

    private void SendAlarm(String kind, Node node, NodeData data, String title, String info = null)
    {
        var msg = GetMarkdown(kind, node, data, title, info);
        RobotHelper.SendAlarm(node.Category, node.WebHook, title, msg);
    }

    private String GetMarkdown(String kind, Node node, NodeData data, String title, String msg = null)
    {
        var sb = new StringBuilder();
        if (!title.IsNullOrEmpty()) sb.AppendLine($"### {title}");
        sb.AppendLine($">**时间：**<font color=\"blue\">{data.CreateTime:yyyy-MM-dd HH:mm:ss}</font>");
        sb.AppendLine($">**节点：**<font color=\"gray\">{node} / {node.IP}</font>");
        sb.AppendLine($">**分类：**<font color=\"gray\">{node.Category}</font>");
        sb.AppendLine($">**系统：**<font color=\"gray\">{node.OS}</font>");
        sb.AppendLine($">**CPU核心：**<font color=\"gray\">{node.Cpu}</font>");
        sb.AppendLine($">**内存容量：**<font color=\"gray\">{node.Memory:n0}M，可用 {data.AvailableMemory:n0}M</font>");
        sb.AppendLine($">**磁盘容量：**<font color=\"gray\">{node.TotalSize:n0}M，可用 {data.AvailableFreeSpace:n0}M</font>");

        switch (kind)
        {
            case "cpu":
                sb.AppendLine($">**CPU使用率：**<font color=\"red\">{data.CpuRate:p0} >= {node.AlarmCpuRate / 100d:p0}</font>");
                break;
            case "memory":
                var rate1 = 1 - (node.Memory == 0 ? 0 : ((Double)data.AvailableMemory / node.Memory));
                sb.AppendLine($">**内存使用率：**<font color=\"red\">{rate1:p0} >= {node.AlarmMemoryRate / 100d:p0}</font>");
                break;
            case "disk":
                var rate2 = 1 - (node.TotalSize == 0 ? 0 : ((Double)data.AvailableFreeSpace / node.TotalSize));
                sb.AppendLine($">**磁盘使用率：**<font color=\"red\"> {rate2:p0} >= {node.AlarmDiskRate / 100d:p0}</font>");
                break;
            case "tcp":
                if (data.TcpConnections >= node.AlarmTcp)
                    sb.AppendLine($">**TCP连接数：**<font color=\"red\">{data.TcpConnections:n0} >= {node.AlarmTcp:n0}</font>");
                if (data.TcpTimeWait >= node.AlarmTcp)
                    sb.AppendLine($">**TCP主动关闭：**<font color=\"red\">{data.TcpTimeWait:n0} >= {node.AlarmTcp:n0}</font>");
                if (data.TcpCloseWait >= node.AlarmTcp)
                    sb.AppendLine($">**TCP被动关闭：**<font color=\"red\">{data.TcpCloseWait:n0} >= {node.AlarmTcp:n0}</font>");
                break;
            case "process":
                sb.AppendLine($">**进程已退出：**<font color=\"red\">{msg}</font>");
                break;
        }

        var str = sb.ToString();
        if (str.Length > 2000) str = str[..2000];

        // 构造网址
        var url = _setting.WebUrl;
        if (!url.IsNullOrEmpty())
        {
            url = url.EnsureEnd("/") + "Nodes/NodeData?nodeId=" + node.ID;
            str += Environment.NewLine + $"[更多信息]({url})";
        }

        return str;
    }
    #endregion

    #region Redis告警
    private void ProcessRedisNode(RedisNode node)
    {
        if (node == null || !node.Enable) return;

        ProcessRedisData(node);
        ProcessRedisQueue(node);
    }

    private void ProcessRedisData(RedisNode node)
    {
        //if (!RobotHelper.CanAlarm(node.Category, node.WebHook)) return;
        if (node.AlarmMemoryRate <= 0 || node.AlarmConnections == 0) return;

        var webhook = RobotHelper.GetAlarm(node.Project, node.Category, node.WebHook);
        if (webhook.IsNullOrEmpty()) return;

        // 最新数据
        var data = RedisData.FindLast(node.Id);
        if (data == null) return;

        using var span = _tracer?.NewSpan($"alarm:{nameof(RedisNode)}");

        var actions = new List<Action<StringBuilder>>();

        // 内存告警
        var rate = data.UsedMemory * 100d / node.MaxMemory;
        if (rate >= node.AlarmMemoryRate)
        {
            // 一定时间内不要重复报错，除非错误翻倍
            var error2 = _cache.Get<Double>("alarm:RedisMemory:" + node.Id);
            if (error2 == 0 || rate > error2 * 2)
            {
                _cache.Set("alarm:RedisMemory:" + node.Id, rate, 5 * 60);

                actions.Add(sb => sb.AppendLine($">**内存告警：**<font color=\"red\">{rate / 100:p0} >= {node.AlarmMemoryRate / 100:p0}</font>"));
            }
        }

        // 连接数告警
        var cs = data.ConnectedClients;
        if (node.AlarmConnections > 0 && cs >= node.AlarmConnections)
        {
            // 一定时间内不要重复报错，除非错误翻倍
            var error2 = _cache.Get<Int32>("alarm:RedisConnections:" + node.Id);
            if (error2 == 0 || cs > error2 * 2)
            {
                _cache.Set("alarm:RedisConnections:" + node.Id, cs, 5 * 60);

                actions.Add(sb => sb.AppendLine($">**连接数告警：**<font color=\"red\">{cs:n0} >= {node.AlarmConnections:n0}</font>"));
            }
        }

        // 速度告警
        var speed = data.Speed;
        if (node.AlarmSpeed > 0 && speed >= node.AlarmSpeed)
        {
            // 一定时间内不要重复报错，除非错误翻倍
            var error2 = _cache.Get<Int32>("alarm:RedisSpeed:" + node.Id);
            if (error2 == 0 || speed > error2 * 2)
            {
                _cache.Set("alarm:RedisSpeed:" + node.Id, speed, 5 * 60);

                actions.Add(sb => sb.AppendLine($">**速度告警：**<font color=\"red\">{speed:n0} >= {node.AlarmSpeed:n0}</font>"));
            }
        }

        // 入流量告警
        var input = data.InputKbps;
        if (node.AlarmInputKbps > 0 && input >= node.AlarmInputKbps)
        {
            // 一定时间内不要重复报错，除非错误翻倍
            var error2 = _cache.Get<Double>("alarm:RedisInputKbps:" + node.Id);
            if (error2 == 0 || input > error2 * 2)
            {
                _cache.Set("alarm:RedisInputKbps:" + node.Id, input, 5 * 60);

                actions.Add(sb => sb.AppendLine($">**入流量告警：**<font color=\"red\">{input:n0} >= {node.AlarmInputKbps:n0}</font>"));
            }
        }

        // 出流量告警
        var output = data.OutputKbps;
        if (node.AlarmOutputKbps > 0 && output >= node.AlarmOutputKbps)
        {
            // 一定时间内不要重复报错，除非错误翻倍
            var error2 = _cache.Get<Double>("alarm:RedisOutputKbps:" + node.Id);
            if (error2 == 0 || output > error2 * 2)
            {
                _cache.Set("alarm:RedisOutputKbps:" + node.Id, output, 5 * 60);

                actions.Add(sb => sb.AppendLine($">**出流量告警：**<font color=\"red\">{output:n0} >= {node.AlarmOutputKbps:n0}</font>"));
            }
        }

        if (actions.Count > 0)
        {
            var msg = GetMarkdown(node, data, "Redis告警", actions);
            RobotHelper.SendAlarm(node.Category, node.WebHook, "Redis告警", msg);
        }
    }

    private String GetMarkdown(RedisNode node, RedisData data, String title, IList<Action<StringBuilder>> actions)
    {
        var sb = new StringBuilder();
        if (!title.IsNullOrEmpty()) sb.AppendLine($"### [{node}]{title}");
        sb.AppendLine($">**时间：**<font color=\"blue\">{data.CreateTime:yyyy-MM-dd HH:mm:ss}</font>");
        sb.AppendLine($">**分类：**<font color=\"gray\">{node.Category}</font>");
        sb.AppendLine($">**版本：**<font color=\"gray\">{node.Version}</font>");
        sb.AppendLine($">**已用内存：**<font color=\"gray\">{data.UsedMemory:n0}</font>");
        sb.AppendLine($">**内存容量：**<font color=\"gray\">{node.MaxMemory:n0}</font>");
        sb.AppendLine($">**连接数：**<font color=\"gray\">{data.ConnectedClients:n0}</font>");
        sb.AppendLine($">**服务器：**<font color=\"gray\">{node.Server}</font>");

        //var rate = node.MaxMemory == 0 ? 0 : (data.UsedMemory * 100 / node.MaxMemory);
        //if (rate >= node.AlarmMemoryRate && node.AlarmMemoryRate > 0)
        //{
        //    sb.AppendLine($">**内存告警：**<font color=\"info\">{data.UsedMemory}/{node.MaxMemory} >= {node.AlarmMemoryRate:p0}</font>");
        //}

        //if (node.AlarmConnections > 0 && data.ConnectedClients >= node.AlarmConnections)
        //{
        //    sb.AppendLine($">**连接告警：**<font color=\"info\">{data.ConnectedClients:n0} >= {node.AlarmConnections:n0}</font>");
        //}
        foreach (var item in actions)
        {
            item(sb);
        }

        var str = sb.ToString();
        if (str.Length > 2000) str = str[..2000];

        // 构造网址
        var url = _setting.WebUrl;
        if (!url.IsNullOrEmpty())
        {
            url = url.EnsureEnd("/") + "Nodes/RedisNode?id=" + node.Id;
            str += Environment.NewLine + $"[更多信息]({url})";
        }

        return str;
    }
    #endregion

    #region Redis队列告警
    private void ProcessRedisQueue(RedisNode node)
    {
        using var span = _tracer?.NewSpan($"alarm:{nameof(RedisMessageQueue)}");

        // 所有队列
        var list = RedisMessageQueue.FindAllByRedisId(node.Id);
        foreach (var queue in list)
        {
            var groupName = !queue.Category.IsNullOrEmpty() ? queue.Category : node.Category;
            var webhook = !queue.WebHook.IsNullOrEmpty() ? queue.WebHook : node.WebHook;

            // 判断告警
            if (queue.Enable && queue.MaxMessages > 0 && queue.Messages >= queue.MaxMessages)
            {
                webhook = RobotHelper.GetAlarm(node.Project, groupName, webhook);
                if (webhook.IsNullOrEmpty()) continue;

                // 一定时间内不要重复报错，除非错误翻倍
                var error2 = _cache.Get<Int32>("alarm:RedisMessageQueue:" + queue.Id);
                if (error2 == 0 || queue.Messages > error2 * 2)
                {
                    _cache.Set("alarm:RedisMessageQueue:" + queue.Id, queue.Messages, 5 * 60);

                    var msg = GetMarkdown(node, queue, true);
                    RobotHelper.SendAlarm(groupName, webhook, "消息队列告警", msg);
                }
            }
        }
    }

    private String GetMarkdown(RedisNode node, RedisMessageQueue queue, Boolean includeTitle)
    {
        var sb = new StringBuilder();
        if (includeTitle) sb.AppendLine($"### [{queue.Name}/{node}]消息队列告警");
        sb.AppendLine($">**时间：**<font color=\"blue\">{queue.UpdateTime:yyyy-MM-dd HH:mm:ss}</font>");
        sb.AppendLine($">**主题：**<font color=\"gray\">{queue.Topic}</font>");
        sb.AppendLine($">**积压：**<font color=\"red\">{queue.Messages:n0} > {queue.MaxMessages:n0}</font>");
        sb.AppendLine($">**消费者：**<font color=\"green\">{queue.Consumers}</font>");
        sb.AppendLine($">**总消费：**<font color=\"green\">{queue.Total:n0}</font>");
        sb.AppendLine($">**服务器：**<font color=\"gray\">{node.Server}</font>");

        var str = sb.ToString();
        if (str.Length > 2000) str = str[..2000];

        // 构造网址
        var url = _setting.WebUrl;
        if (!url.IsNullOrEmpty())
        {
            url = url.EnsureEnd("/") + "Nodes/RedisMessageQueue?redisId=" + queue.RedisId + "&q=" + queue.Name;
            str += Environment.NewLine + $"[更多信息]({url})";
        }

        return str;
    }
    #endregion
}