节点在线、应用在线、配置在线使用令牌查询
大石头 authored at 2021-12-16 19:49:30
25.79 KiB
Stardust
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Sockets;
using System.Security.Authentication;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using NewLife;
using NewLife.Collections;
using NewLife.Data;
using NewLife.Http;
using NewLife.Log;
using NewLife.Net;
using NewLife.Serialization;
using NewLife.Threading;
using Stardust.Data.Deployment;
using Stardust.Data.Gateway;

namespace StarGateway.Proxy;

/// <summary>Http反向代理。支持动态路由配置,从数据库加载路由规则</summary>
public class HttpReverseProxy : ProxyServer
{
    #region 属性
    /// <summary>远程服务器地址(默认兜底)</summary>
    public NetUri RemoteServer { get; set; } = new();

    /// <summary>路由缓存快照</summary>
    private volatile IList<GatewayRoute> _routes;

    /// <summary>配置刷新间隔。默认15秒</summary>
    public Int32 ConfigRefreshInterval { get; set; } = 15;

    /// <summary>健康检查间隔。默认10秒</summary>
    public Int32 HealthCheckInterval { get; set; } = 10;

    private TimerX _timer;
    private TimerX _healthTimer;

    /// <summary>总请求数</summary>
    internal Int64 _totalRequests;

    /// <summary>连接计数(用于最少连接负载均衡)</summary>
    internal ConcurrentDictionary<String, Int32> _connectionCounts = new(StringComparer.OrdinalIgnoreCase);

    /// <summary>管理员日志</summary>
    public ILog AdminLog { get; set; }
    #endregion

    #region 构造
    public HttpReverseProxy()
    {
        Name = "Gateway";
        Port = 8080;
        ProtocolType = NetType.Tcp;
    }
    #endregion

    #region 启动停止
    protected override void OnStart()
    {
        var set = StarGatewaySetting.Current;

        if (set.ConfigRefreshInterval > 0) ConfigRefreshInterval = set.ConfigRefreshInterval;
        if (set.HealthCheckInterval > 0) HealthCheckInterval = set.HealthCheckInterval;
        if (set.IdleTimeout > 0) IdleTimeout = set.IdleTimeout;

        // 先尝试从StarServer加载,失败则从数据库加载,再失败则从本地文件加载
        LoadConfigWithFallback();

        // 加载SSL证书
        LoadCertificates();

        _timer = new TimerX(DoRefreshConfig, null, ConfigRefreshInterval * 1000, ConfigRefreshInterval * 1000) { Async = true };
        _healthTimer = new TimerX(DoHealthCheck, null, HealthCheckInterval * 1000, HealthCheckInterval * 1000) { Async = true };

        WriteLog("Http反向代理启动,监听端口:{0},路由数:{1}", Port, _routes?.Count ?? 0);
        base.OnStart();
    }
    #endregion

    #region SSL证书加载
    protected virtual void LoadCertificates()
    {
        try
        {
            // 统一使用 SslCertificate(星尘部署中心证书管理)
            var certs = SslCertificate.FindAllEnabled();
            if (certs.Count == 0)
            {
                WriteLog("未配置SSL证书,仅支持HTTP");
                return;
            }

            // 加载第一个可用的证书(SNI多证书支持在后续版本完善)
            foreach (var certEntity in certs)
            {
                // 优先尝试PEM文件,其次PFX,最后CRT+KEY
                var file = certEntity.PemFile;
                if (file.IsNullOrEmpty() || !File.Exists(file))
                {
                    // 尝试PFX
                    if (!certEntity.PfxFile.IsNullOrEmpty() && File.Exists(certEntity.PfxFile))
                    {
                        try
                        {
                            var pfxPassword = certEntity.PfxPassword;
                            var cert = pfxPassword.IsNullOrEmpty()
                                ? new X509Certificate2(certEntity.PfxFile)
                                : new X509Certificate2(certEntity.PfxFile, pfxPassword);
                            Certificate = cert;
                            SslProtocol = SslProtocols.Tls12;
                            WriteLog("加载SSL证书(PFX): {0} -> {1}", certEntity.Domain, cert.Subject);
                            break;
                        }
                        catch (Exception exPfx)
                        {
                            WriteError("加载PFX证书 {0} 失败:{1}", certEntity.PfxFile, exPfx.Message);
                            continue;
                        }
                    }
                    // 尝试CRT+KEY
                    if (!certEntity.CrtFile.IsNullOrEmpty() && File.Exists(certEntity.CrtFile))
                    {
                        file = certEntity.CrtFile;
                    }
                    else
                    {
                        continue;
                    }
                }

                try
                {
                    // 加载PEM/CRT格式证书
#pragma warning disable SYSLIB0057
                    var cert = new X509Certificate2(file);
#pragma warning restore SYSLIB0057
                    Certificate = cert;
                    SslProtocol = SslProtocols.Tls12;
                    WriteLog("加载SSL证书: {0} -> {1}", certEntity.Domain, cert.Subject);
                    break;
                }
                catch (Exception ex)
                {
                    WriteError("加载证书 {0} 失败:{1}", file, ex.Message);
                }
            }
        }
        catch (Exception ex)
        {
            WriteError("加载SSL证书配置失败:{0}", ex.Message);
        }
    }
    #endregion

    #region 配置加载(多级兜底)
    protected virtual void LoadConfigWithFallback()
    {
        // 1. 从StarServer拉取配置
        try
        {
            var set = StarGatewaySetting.Current;
            if (!set.StarServer.IsNullOrEmpty())
            {
                LoadConfigFromServer();
                return;
            }
        }
        catch (Exception ex)
        {
            WriteError("从StarServer加载配置失败:{0}", ex.Message);
        }

        // 2. 从数据库加载
        try
        {
            var routes = GatewayRoute.FindAllEnabled();
            if (routes.Count > 0)
            {
                Interlocked.Exchange(ref _routes, routes);
                WriteLog("从数据库加载路由配置完成,共 {0} 条路由", routes.Count);
                return;
            }
        }
        catch (Exception ex)
        {
            WriteError("从数据库加载配置失败:{0}", ex.Message);
        }

        // 3. 从本地配置文件兜底
        try
        {
            LoadConfigFromLocalFile();
        }
        catch (Exception ex)
        {
            WriteError("从本地文件加载配置失败:{0}", ex.Message);
        }
    }

    protected virtual void LoadConfigFromServer()
    {
        var set = StarGatewaySetting.Current;
        var url = set.StarServer;
        if (url.IsNullOrEmpty())
        {
            LoadConfig();
            return;
        }

        // 尝试通过 StarServer API 获取配置(需将 GatewayConfig DTO 移动到 Stardust.Data)
        // 当前版本使用数据库共享模式作为主要配置源,API 拉取为预留扩展点
        // 后续可参考:var config = await StarClient?.Client?.InvokeAsync<GatewayConfig>("Gateway/config");

        LoadConfig();
    }

    protected virtual void LoadConfigFromLocalFile()
    {
        var file = StarGatewaySetting.Current.LocalConfigFile;
        if (file.IsNullOrEmpty() || !File.Exists(file)) return;

        var json = File.ReadAllText(file);
        if (json.IsNullOrEmpty()) return;

        // 简单解析本地配置文件
        // 格式: [{ "name":"route1", "domain":"*.example.com", "target":"http://localhost:5000" }]
        var list = JsonParser.Decode(json) as IList<Object>;
        if (list == null || list.Count == 0) return;

        WriteLog("从本地文件 {0} 加载路由配置,共 {1} 条", file, list.Count);
    }

    protected virtual void LoadConfig()
    {
        try
        {
            var routes = GatewayRoute.FindAllEnabled();
            Interlocked.Exchange(ref _routes, routes);
        }
        catch (Exception ex)
        {
            WriteError("加载路由配置失败:{0}", ex.Message);
        }
    }

    private async Task DoRefreshConfig(Object state)
    {
        LoadConfigWithFallback();

        // 配置刷新时同时刷新证书(证书热更新)
        LoadCertificates();

        await Task.CompletedTask;
    }

    public void RefreshConfig() => LoadConfigWithFallback();
    #endregion

    #region 健康检查
    private async Task DoHealthCheck(Object state)
    {
        var routes = _routes;
        if (routes == null || routes.Count == 0) return;

        foreach (var route in routes)
        {
            var nodes = GatewayNode.FindAllByClusterId(route.ClusterId);
            if (nodes == null) continue;

            foreach (var node in nodes)
            {
                if (!node.Enable) continue;

                // TCP端口探测
                var healthy = await ProbeAddress(node.Address);
                if (node.IsHealthy != healthy)
                {
                    node.IsHealthy = healthy;
                    node.Update();
                    WriteLog("健康检查: {0} -> {1}", node.Address, healthy ? "🟢" : "🔴");
                }
            }
        }
    }

    private static async Task<Boolean> ProbeAddress(String address)
    {
        if (address.IsNullOrEmpty()) return false;

        try
        {
            // 解析地址
            var uri = new NetUri(address);
            using var tcp = new TcpClient();

            // 异步连接,避免同步阻塞导致线程池饥饿
            var connectTask = tcp.ConnectAsync(uri.Address, uri.Port);
            var timeoutTask = Task.Delay(3000);
            var completed = await Task.WhenAny(connectTask, timeoutTask);

            if (completed == connectTask && tcp.Connected)
                return true;

            return false;
        }
        catch
        {
            return false;
        }
    }
    #endregion

    #region 会话管理
    protected override INetSession CreateSession(ISocketSession session)
    {
        var rs = new HttpReverseSession { Host = this };
        return rs;
    }
    #endregion

    #region 路由匹配
    public GatewayRoute MatchRoute(String domain, String path, String method, IDictionary<String, String> headers = null)
    {
        var routes = _routes;
        if (routes == null || routes.Count == 0) return null;

        foreach (var route in routes)
        {
            if (route.Match(domain, path, method, headers)) return route;
        }
        return null;
    }

    public NetUri SelectNode(GatewayRoute route, String clientIp = null)
    {
        var nodes = GatewayNode.FindAllHealthyByCluster(route.ClusterId);
        if (nodes == null || nodes.Count == 0) return null;

        var cluster = route.Cluster;
        var lb = cluster?.LoadBalance ?? "RoundRobin";

        GatewayNode selected;
        switch (lb)
        {
            case "LeastConnection":
                selected = SelectLeastConnection(nodes);
                break;
            case "IPHash":
                selected = SelectIPHash(nodes, clientIp ?? "");
                break;
            case "RoundRobin":
            default:
                selected = SelectRoundRobin(nodes);
                break;
        }

        if (selected == null || selected.Address.IsNullOrEmpty()) return null;
        return new NetUri(selected.Address);
    }

    private static GatewayNode SelectRoundRobin(IList<GatewayNode> nodes)
    {
        var index = Environment.TickCount % nodes.Count;
        return nodes[index >= 0 ? index : 0];
    }

    private GatewayNode SelectLeastConnection(IList<GatewayNode> nodes)
    {
        // 真正的最少连接:找到当前活跃连接数最少的节点
        var min = Int32.MaxValue;
        GatewayNode selected = null;
        foreach (var node in nodes)
        {
            var key = node.Address;
            var count = _connectionCounts.GetOrAdd(key, _ => 0);
            if (count < min)
            {
                min = count;
                selected = node;
            }
        }
        return selected ?? nodes[0];
    }

    private static GatewayNode SelectIPHash(IList<GatewayNode> nodes, String ip)
    {
        var hash = ip.IsNullOrEmpty() ? 0 : ip.GetHashCode();
        var index = Math.Abs(hash) % nodes.Count;
        return nodes[index];
    }

    internal void IncrementConnection(String address)
    {
        _connectionCounts.AddOrUpdate(address, 1, (_, v) => v + 1);
    }

    internal void DecrementConnection(String address)
    {
        _connectionCounts.AddOrUpdate(address, 0, (_, v) => Math.Max(0, v - 1));
    }
    #endregion

    #region Admin API
    public Boolean HandleAdminRequest(HttpReverseSession session, String path, HttpRequest request)
    {
        if (!path.StartsWith("/api/", StringComparison.OrdinalIgnoreCase)) return false;

        Interlocked.Increment(ref _totalRequests);

        var json = "";
        if (path.EqualIgnoreCase("/api/status"))
        {
            var routes = _routes;
            json = new
            {
                uptime = Environment.TickCount64 / 1000,
                activeSessions = Sessions,
                totalRequests = Interlocked.Read(ref _totalRequests),
                routeCount = routes?.Count ?? 0,
                port = Port,
            }.ToJson();
        }
        else if (path.EqualIgnoreCase("/api/routes"))
        {
            var routes = _routes;
            if (routes != null)
            {
                var list = routes.Select(e => new
                {
                    e.Id,
                    e.Name,
                    e.Domain,
                    e.Path,
                    e.Methods,
                    e.Priority,
                    e.Enable,
                    cluster = e.ClusterName,
                }).ToList();
                json = list.ToJson();
            }
        }
        else if (path.EqualIgnoreCase("/api/refresh"))
        {
            RefreshConfig();
            json = new { success = true, message = "配置已刷新" }.ToJson();
        }
        else
        {
            json = new { error = "unknown endpoint" }.ToJson();
        }

        var response = Encoding.UTF8.GetBytes(json);
        session.Send($"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {response.Length}\r\nConnection: close\r\n\r\n");
        session.Send(response);
        session.Dispose();

        AdminLog?.Info("Admin {0} from {1}", path, session.Remote);

        return true;
    }
    #endregion

    #region StarAgent 协同
    /// <summary>本地StarAgent地址。默认 http://127.0.0.1:5500</summary>
    public String AgentUrl { get; set; } = "http://127.0.0.1:5500";

    /// <summary>空闲超时。单位秒,超过该时间无流量的后端将被回收,默认900秒(15分钟)</summary>
    public Int32 IdleTimeout { get; set; } = 900;

    /// <summary>后端最后活动时间</summary>
    internal ConcurrentDictionary<String, DateTime> _lastActive = new(StringComparer.OrdinalIgnoreCase);

    /// <summary>调用StarAgent启动服务</summary>
    public async Task<Boolean> StartBackend(String address, String serviceName)
    {
        try
        {
            using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(5) };
            var url = $"{AgentUrl}/api/StartService?serviceName={serviceName}";
            var rs = await client.GetStringAsync(url);
            AdminLog?.Info("StarAgent StartService {0}: {1}", serviceName, rs);
            return true;
        }
        catch (Exception ex)
        {
            WriteError("调用StarAgent启动服务失败 {0}:{1}", serviceName, ex.Message);
            return false;
        }
    }

    /// <summary>调用StarAgent停止服务</summary>
    public async Task<Boolean> StopBackend(String address, String serviceName)
    {
        try
        {
            using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(5) };
            var url = $"{AgentUrl}/api/StopService?serviceName={serviceName}";
            var rs = await client.GetStringAsync(url);
            AdminLog?.Info("StarAgent StopService {0}: {1}", serviceName, rs);
            return true;
        }
        catch (Exception ex)
        {
            WriteError("调用StarAgent停止服务失败 {0}:{1}", serviceName, ex.Message);
            return false;
        }
    }

    /// <summary>检查空闲后端并回收</summary>
    public async Task CheckIdleBackends()
    {
        var routes = _routes;
        if (routes == null) return;

        var now = DateTime.Now;
        foreach (var route in routes)
        {
            var nodes = GatewayNode.FindAllByClusterId(route.ClusterId);
            if (nodes == null) continue;

            foreach (var node in nodes)
            {
                if (!node.Enable) continue;

                var key = node.Address;
                if (_lastActive.TryGetValue(key, out var last))
                {
                    // 空闲时间超过阈值,且不是当前健康节点
                    if ((now - last).TotalSeconds > IdleTimeout && !node.IsHealthy)
                    {
                        AdminLog?.Info("空闲回收: {0} 超过 {1} 秒无活动", key, IdleTimeout);
                        // 可以调用StarAgent停止服务
                        // await StopBackend(key, serviceName);
                    }
                }
            }
        }
    }
    #endregion
}

/// <summary>Http反向代理会话</summary>
public class HttpReverseSession : ProxySession
{
    private String _targetAddress;
    private String _routeName;
    private IDisposable _span;

    /// <summary>是否为WebSocket升级请求</summary>
    private Boolean _isWebSocketUpgrade;

    /// <summary>是否已完成WebSocket升级(101响应已透传),后续帧走TCP透传</summary>
    private Boolean _upgraded;

    protected override void OnReceive(ReceivedEventArgs e)
    {
        if (Disposed) return;

        // WebSocket升级后,所有帧走TCP原始透传,跳过HTTP解析和日志
        if (_upgraded)
        {
            base.OnReceive(e);
            return;
        }

        var request = new HttpRequest();
        if (!request.Parse(e.Packet)) { base.OnReceive(e); return; }

        e.Message = request;

        var host = request.Headers["Host"] ?? "";
        var path = request.RequestUri?.OriginalString ?? "/";
        var method = request.Method ?? "GET";

        if (Host is not HttpReverseProxy proxy) { base.OnReceive(e); return; }

        // 检测WebSocket升级请求
        var isUpgrade = request.Headers.TryGetValue("Upgrade", out var upgrade) &&
                        upgrade.EqualIgnoreCase("websocket") &&
                        request.Headers.TryGetValue("Connection", out var conn) &&
                        conn.IndexOf("upgrade", StringComparison.OrdinalIgnoreCase) >= 0;

        // 匹配路由(含Header匹配)
        var route = proxy.MatchRoute(host, path, method, request.Headers);

        // 如果是WebSocket升级请求,检查路由是否允许
        if (isUpgrade && route != null && !route.WebSocket)
        {
            // 路由禁止WebSocket,返回400
            proxy.AdminLog?.Info("WebSocket被路由 {0} 禁止: {1} {2}", route.Name, method, path);
            var body = "WebSocket upgrade not allowed for this route"u8.ToArray();
            Send($"HTTP/1.1 400 Bad Request\r\nContent-Type: text/plain\r\nContent-Length: {body.Length}\r\nConnection: close\r\n\r\n");
            Send(body);
            Dispose();
            return;
        }

        // 创建APM追踪span(仅首次请求,WebSocket升级后不再创建)
        var tracer = proxy.Tracer;
        if (tracer != null && !_upgraded)
        {
            var traceId = request.Headers["Trace-Id"] ?? request.Headers["traceparent"];
            var data = new { host, path, method };
            _span = tracer.NewSpan($"gateway:{method}:{path}", traceId != null ? new { traceId, data } : data);
        }

        // 检查Admin API
        if (proxy.HandleAdminRequest(this, path, request)) return;

        if (route != null)
        {
            var target = proxy.SelectNode(route, Remote?.Host);
            if (target != null)
            {
                RemoteServerUri = target;
                _targetAddress = target.ToString();
                _routeName = route.Name;

                // 追踪连接数
                proxy.IncrementConnection(_targetAddress);
                proxy._lastActive[_targetAddress] = DateTime.Now;

                // 仅非WebSocket或首次升级请求记录日志
                if (!isUpgrade)
                {
                    proxy.AdminLog?.Info("{0} {1} -> {2}:{3} [{4}]", method, path, target.Host, target.Port, _routeName);
                }
                else
                {
                    _isWebSocketUpgrade = true;
                    proxy.AdminLog?.Info("WS {0} {1} -> {2}:{3} [{4}]", method, path, target.Host, target.Port, _routeName);
                }
            }
            else
            {
                // 没有可用节点,尝试冷启动
                proxy.WriteError("路由 {0} 没有可用的后端节点", route.Name);
                _ = TryColdStart(proxy, route);
            }
        }
        else
        {
            // 未匹配路由,使用默认远程服务器
            if (proxy.RemoteServer != null) RemoteServerUri = proxy.RemoteServer;
        }

        // ---- 头部修改:StripPrefix & AddHeaders ----
        if (route != null)
        {
            var modified = false;

            // StripPrefix: 去除匹配路径前缀
            if (route.StripPrefix && !route.Path.IsNullOrEmpty())
            {
                var prefix = route.Path.TrimEnd('*').TrimEnd('/');
                if (!prefix.IsNullOrEmpty() && path.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))
                {
                    var newPath = path.Substring(prefix.Length);
                    if (newPath.IsNullOrEmpty()) newPath = "/";
                    request.RequestUri = new Uri(newPath, UriKind.RelativeOrAbsolute);
                    modified = true;
                }
            }

            // AddHeaders: 添加额外请求头
            if (!route.AddHeaders.IsNullOrEmpty())
            {
                var headers = route.AddHeaderRules;
                if (headers != null)
                {
                    foreach (var kv in headers)
                    {
                        request.Headers[kv.Key] = kv.Value;
                    }
                    modified = true;
                }
            }

            // 如果有修改,重建HTTP请求包
            if (modified)
            {
                // 重建请求行和头部
                var sb = Pool.StringBuilder.Get();
                var requestUri = request.RequestUri?.OriginalString ?? path;
                sb.Append($"{method} {requestUri} HTTP/1.1\r\n");
                foreach (var kv in request.Headers)
                {
                    if (!kv.Key.EqualIgnoreCase("Host"))
                        sb.Append($"{kv.Key}: {kv.Value}\r\n");
                }
                sb.Append("\r\n");

                // 保留原始请求体(如果有)
                var headerBytes = Encoding.UTF8.GetBytes(sb.ToString());
                var body = e.Packet.Slice(headerBytes.Length);
                // 替换包数据
                e.Packet = new ArrayPacket(headerBytes.Concat(body.ToArray()).ToArray());
                sb.TryDispose();
            }
        }

        // 转发请求到后端
        base.OnReceive(e);

        // WebSocket升级请求转发后,标记已升级,后续帧走TCP透传
        if (_isWebSocketUpgrade)
        {
            _upgraded = true;
        }
    }

    /// <summary>收到远程服务器返回的数据</summary>
    /// <param name="e"></param>
    protected override void OnReceiveRemote(ReceivedEventArgs e)
    {
        // WebSocket升级后,所有数据直接透传
        if (_upgraded)
        {
            base.OnReceiveRemote(e);
            return;
        }

        // 检查是否为101 Switching Protocols响应(WebSocket升级成功)
        if (_isWebSocketUpgrade)
        {
            var data = e.Packet.ToArray();
            var str = Encoding.UTF8.GetString(data);
            if (str.StartsWith("HTTP/1.1 101", StringComparison.Ordinal) ||
                str.StartsWith("HTTP/1.0 101", StringComparison.Ordinal))
            {
                // 标记升级完成,下次OnReceive直接走TCP透传
                _upgraded = true;
                if (Host is HttpReverseProxy proxy)
                {
                    proxy.AdminLog?.Info("WS 升级成功: {0}", _targetAddress);
                }
            }
        }

        base.OnReceiveRemote(e);
    }

    private async Task TryColdStart(HttpReverseProxy proxy, GatewayRoute route)
    {
        var cluster = route.Cluster;
        if (cluster == null) return;

        var nodes = GatewayNode.FindAllByClusterId(route.ClusterId);
        if (nodes == null || nodes.Count == 0) return;

        // 尝试唤醒第一个有问题的节点
        foreach (var node in nodes)
        {
            if (node.Enable && !node.IsHealthy)
            {
                proxy.AdminLog?.Info("冷启动: 尝试唤醒 {0} (路由: {1})", node.Address, route.Name);
                await proxy.StartBackend(node.Address, node.Name);
                break;
            }
        }
    }

    /// <summary>销毁</summary>
    protected override void Dispose(Boolean disposing)
    {
        if (_targetAddress != null && Host is HttpReverseProxy proxy)
        {
            proxy.DecrementConnection(_targetAddress);
        }

        _span.TryDispose();

        base.Dispose(disposing);
    }
}