diff --git a/bbit_ai/app/models/CubeReportRequest.py b/bbit_ai/app/models/CubeReportRequest.py index 627766a..6d2a061 100644 --- a/bbit_ai/app/models/CubeReportRequest.py +++ b/bbit_ai/app/models/CubeReportRequest.py @@ -25,3 +25,7 @@ class SendCubeReportMessageRequest(BaseModel): class RenameCubeReportConversationRequest(BaseModel): name: str = Field(min_length=1, max_length=200) + + +class ExportCubeReportRequest(BaseModel): + columnKeys: list[str] = Field(min_length=1, max_length=16384) diff --git a/bbit_ai/app/routers/CubeReport.py b/bbit_ai/app/routers/CubeReport.py index 8a9ca23..4c1d9d7 100644 --- a/bbit_ai/app/routers/CubeReport.py +++ b/bbit_ai/app/routers/CubeReport.py @@ -25,6 +25,7 @@ from db.postgres.cube_report import ( ) from models.BaseResponse import BaseResponse from models.CubeReportRequest import ( + ExportCubeReportRequest, RenameCubeReportConversationRequest, SendCubeReportMessageRequest, ) @@ -352,11 +353,13 @@ async def _export_cube_result( tenant_id: str, limit_source: str, title: str, + column_keys: list[str] | None = None, ) -> BaseResponse: export_data = await query_cube_export( base_load=query, tenant_id=tenant_id, limit_source=limit_source, + column_keys=column_keys, ) filename = sanitize_filename(title or "Cube查询结果") contents, sheet_count = await asyncio.to_thread( @@ -378,6 +381,7 @@ async def _export_cube_result( @cubeReportRouter.post("/reports/{report_id}/export") async def export_report_data( report_id: str, + request: ExportCubeReportRequest | None = None, user_id: UUID = Depends(get_user_id_from_token), ): report, tenant_id = await _resolve_saved_report(report_id, user_id) @@ -386,6 +390,7 @@ async def export_report_data( tenant_id=tenant_id, limit_source=report["limitSource"], title=report["title"], + column_keys=request.columnKeys if request else None, ) @@ -440,6 +445,7 @@ async def delete_report( @cubeReportRouter.post("/sessions/{conversation_id}/export") async def export_session_data( conversation_id: str, + request: ExportCubeReportRequest | None = None, user_id: UUID = Depends(get_user_id_from_token), ): session = await get_dify_conversation(conversation_id, str(user_id)) @@ -453,6 +459,7 @@ async def export_session_data( tenant_id=tenant_id, limit_source=state["limitSource"], title=str(session.get("title") or state.get("title") or "Cube查询结果"), + column_keys=request.columnKeys if request else None, ) diff --git a/bbit_ai/app/service/cube_report.py b/bbit_ai/app/service/cube_report.py index 8b90889..156b457 100644 --- a/bbit_ai/app/service/cube_report.py +++ b/bbit_ai/app/service/cube_report.py @@ -903,6 +903,7 @@ async def query_cube_export( base_load: dict[str, Any], tenant_id: str | None, limit_source: str | None, + column_keys: list[str] | None = None, ) -> dict[str, Any]: query = deepcopy(base_load) if not tenant_id: @@ -925,6 +926,19 @@ async def query_cube_export( key: _column_title(key, sql_info["aliases"], member_titles) for key in keys } + if column_keys: + selected_keys = list(dict.fromkeys(str(key).strip() for key in column_keys)) + invalid_keys = [key for key in selected_keys if key not in headers] + if invalid_keys: + raise HTTPException( + status_code=422, + detail=f"导出列不存在或已失效：{', '.join(invalid_keys[:5])}", + ) + headers = {key: headers[key] for key in selected_keys} + rows = [ + {key: row.get(key) for key in selected_keys} + for row in rows + ] return { "headers": headers, "rows": rows, diff --git a/docs/open-api-kong-setup.md b/docs/open-api-kong-setup.md new file mode 100644 index 0000000..521f70d --- /dev/null +++ b/docs/open-api-kong-setup.md @@ -0,0 +1,99 @@ +# 开放接口统计：Kong 3.9.1 配置清单 + +Ktor 统计接收地址： + +```text +http://:8089/internal/open-api/statistics/ingest +``` + +生产环境部署 Ktor 时必须设置： + +```text +OPEN_API_LOG_TOKEN=<高强度随机Token> +``` + +Kong HTTP Log 中配置的 Bearer Token 必须与该环境变量一致。 + +## 1. Key Auth + +先查询目标 Service 或 Route 上的 Key Auth 插件： + +```bash +curl http://127.0.0.1:8001/services//plugins +``` + +更新插件： + +```bash +curl -X PATCH http://127.0.0.1:8001/plugins/ \ + --data "config.hide_credentials=true" \ + --data "config.key_in_header=true" \ + --data "config.key_in_query=false" \ + --data "config.key_in_body=false" +``` + +确认返回配置中： + +```json +{ + "hide_credentials": true, + "key_in_header": true, + "key_in_query": false, + "key_in_body": false +} +``` + +## 2. HTTP Log + +推荐挂载到开放接口对应的 Route。若一个 Service 只有开放接口，也可挂载到 Service。 + +```bash +curl -X POST http://127.0.0.1:8001/routes//plugins \ + -H "Content-Type: application/json" \ + -d '{ + "name": "http-log", + "config": { + "http_endpoint": "http://:8089/internal/open-api/statistics/ingest", + "method": "POST", + "content_type": "application/json", + "headers": { + "Authorization": "Bearer " + }, + "timeout": 3000, + "keepalive": 60000, + "queue": { + "max_entries": 10000, + "max_batch_size": 50, + "max_coalescing_delay": 1, + "max_retry_time": 60 + } + } + }' +``` + +建议配置范围： + +- 票通 `/api/open/v1` +- AI 后端 `/api/public` +- 核心 Ktor `/traceability/public` +- 其他需要统计的开放 Route + +不要将回调地址配置为经过 Kong 的公网域名，避免日志回调再次触发 HTTP Log。 + +## 3. 验收 + +每类 Route 至少执行以下测试： + +1. 有效 API Key 调用成功。 +2. 不带 API Key，Kong 返回 401。 +3. 使用错误 API Key，Kong 返回 401。 +4. 同一个 Consumer 从不同 IP 调用。 +5. 不同 Consumer 调用同一个接口。 +6. 动态发票号被归入同一个路径模板。 + +在前端“开放接口中心”检查： + +- 有效 Key 显示 Kong Consumer。 +- 无效或缺失 Key 显示匿名调用方和来源 IP。 +- 小时调用量与测试次数一致。 +- 页面中不出现 API Key、请求体、发票号或图片内容。 diff --git a/ktor/src/main/kotlin/ink/snowflake/server/Application.kt b/ktor/src/main/kotlin/ink/snowflake/server/Application.kt index c647aa0..a5604ce 100644 --- a/ktor/src/main/kotlin/ink/snowflake/server/Application.kt +++ b/ktor/src/main/kotlin/ink/snowflake/server/Application.kt @@ -5,6 +5,7 @@ import ink.snowflake.server.controller.User import ink.snowflake.server.controller.chat import ink.snowflake.server.utils.plugins.configureSockets import ink.snowflake.server.controller.ImageAnalytics +import ink.snowflake.server.controller.OpenApiStatistics import ink.snowflake.server.controller.Public import ink.snowflake.server.controller.RemoteDebug import ink.snowflake.server.controller.Traceability @@ -78,6 +79,8 @@ fun Application.module() { // 业务-图片分析 ImageAnalytics() Traceability(appConfig) + // 开放接口资产与 Kong 调用统计 + OpenApiStatistics(appConfig) // 业务-公开接口 Public() } diff --git a/ktor/src/main/kotlin/ink/snowflake/server/controller/OpenApiStatistics.kt b/ktor/src/main/kotlin/ink/snowflake/server/controller/OpenApiStatistics.kt new file mode 100644 index 0000000..123b421 --- /dev/null +++ b/ktor/src/main/kotlin/ink/snowflake/server/controller/OpenApiStatistics.kt @@ -0,0 +1,148 @@ +package ink.snowflake.server.controller + +import ink.snowflake.server.model.request.SaveOpenApiEndpointRequest +import ink.snowflake.server.model.request.UpdateOpenApiConsumerRequest +import ink.snowflake.server.model.response.BaseResponse +import ink.snowflake.server.utils.AppConfig +import ink.snowflake.server.utils.dao.OpenApiStatisticsDao +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.Application +import io.ktor.server.auth.authenticate +import io.ktor.server.request.receive +import io.ktor.server.request.receiveText +import io.ktor.server.response.respond +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import io.ktor.server.routing.put +import io.ktor.server.routing.route +import io.ktor.server.routing.routing +import kotlinx.datetime.Clock +import kotlinx.datetime.Instant +import java.security.MessageDigest +import java.util.UUID + +fun Application.OpenApiStatistics(config: AppConfig) { + OpenApiStatisticsDao.initSchemaAndSeed() + + routing { + post("/internal/open-api/statistics/ingest") { + if (!config.openApiStatisticsEnabled) { + call.respond(HttpStatusCode.ServiceUnavailable) + return@post + } + val expected = "Bearer ${config.openApiStatisticsIngestToken}" + val actual = call.request.headers[HttpHeaders.Authorization].orEmpty() + if ( + config.openApiStatisticsIngestToken.isBlank() || + !MessageDigest.isEqual(expected.toByteArray(Charsets.UTF_8), actual.toByteArray(Charsets.UTF_8)) + ) { + call.respond(HttpStatusCode.Unauthorized) + return@post + } + OpenApiStatisticsDao.ingest(call.receiveText()) + call.respond(HttpStatusCode.NoContent) + } + + authenticate { + route("/open-api") { + get("/overview") { + val (from, to) = call.statisticsRange() + call.respond( + BaseResponse( + data = OpenApiStatisticsDao.overview( + from = from, + to = to, + serviceCode = call.request.queryParameters["serviceCode"], + category = call.request.queryParameters["category"], + ) + ) + ) + } + + get("/endpoints") { + call.respond(BaseResponse(data = OpenApiStatisticsDao.listEndpoints())) + } + + post("/endpoints") { + val request = call.receive() + if (request.name.isBlank() || request.serviceCode.isBlank() || request.pathTemplate.isBlank()) { + call.respond( + HttpStatusCode.BadRequest, + BaseResponse(status = false, message = "接口名称、所属服务和路径不能为空") + ) + return@post + } + call.respond(BaseResponse(data = OpenApiStatisticsDao.saveEndpoint(null, request))) + } + + put("/endpoints/{id}") { + val id = call.parameters["id"]?.let { runCatching { UUID.fromString(it) }.getOrNull() } + if (id == null) { + call.respond( + HttpStatusCode.BadRequest, + BaseResponse(status = false, message = "接口 ID 无效") + ) + return@put + } + val request = call.receive() + call.respond(BaseResponse(data = OpenApiStatisticsDao.saveEndpoint(id, request))) + } + + get("/usage") { + val (from, to) = call.statisticsRange() + val endpointId = call.request.queryParameters["endpointId"] + ?.let { runCatching { UUID.fromString(it) }.getOrNull() } + call.respond( + BaseResponse( + data = OpenApiStatisticsDao.listUsage( + from = from, + to = to, + serviceCode = call.request.queryParameters["serviceCode"], + category = call.request.queryParameters["category"], + endpointId = endpointId, + callerType = call.request.queryParameters["callerType"], + ) + ) + ) + } + + get("/consumers") { + call.respond(BaseResponse(data = OpenApiStatisticsDao.listConsumers())) + } + + put("/consumers/{id}") { + val id = call.parameters["id"]?.let { runCatching { UUID.fromString(it) }.getOrNull() } + if (id == null) { + call.respond( + HttpStatusCode.BadRequest, + BaseResponse(status = false, message = "调用方 ID 无效") + ) + return@put + } + val result = OpenApiStatisticsDao.updateConsumer(id, call.receive()) + if (result == null) { + call.respond( + HttpStatusCode.NotFound, + BaseResponse(status = false, message = "调用方不存在") + ) + return@put + } + call.respond(BaseResponse(data = result)) + } + } + } + } +} + +private fun io.ktor.server.application.ApplicationCall.statisticsRange(): Pair { + val now = Clock.System.now() + val defaultFrom = Instant.fromEpochMilliseconds(now.toEpochMilliseconds() - 7L * 24 * 3_600_000) + val from = request.queryParameters["from"] + ?.let { runCatching { Instant.parse(it) }.getOrNull() } + ?: defaultFrom + val to = request.queryParameters["to"] + ?.let { runCatching { Instant.parse(it) }.getOrNull() } + ?: now + return if (from <= to) from to to else to to from +} diff --git a/ktor/src/main/kotlin/ink/snowflake/server/model/database/OpenApiTables.kt b/ktor/src/main/kotlin/ink/snowflake/server/model/database/OpenApiTables.kt new file mode 100644 index 0000000..46107cf --- /dev/null +++ b/ktor/src/main/kotlin/ink/snowflake/server/model/database/OpenApiTables.kt @@ -0,0 +1,64 @@ +package ink.snowflake.server.model.database + +import org.jetbrains.exposed.v1.core.Table +import org.jetbrains.exposed.v1.core.dao.id.UUIDTable +import org.jetbrains.exposed.v1.datetime.timestamp + +object OpenApiEndpointsTable : UUIDTable("open_api_endpoints") { + val name = varchar("name", 160) + val serviceCode = varchar("service_code", 64) + val category = varchar("category", 80) + val httpMethod = varchar("http_method", 16).default("*") + val pathTemplate = varchar("path_template", 500) + val kongServiceId = varchar("kong_service_id", 80).default("") + val kongRouteId = varchar("kong_route_id", 80).default("") + val description = text("description").default("") + val owner = varchar("owner", 120).default("") + val monitorEnabled = bool("monitor_enabled").default(true) + val status = varchar("status", 32).default("enabled") + val createdAt = timestamp("created_at") + val updatedAt = timestamp("updated_at") + + init { + uniqueIndex(serviceCode, httpMethod, pathTemplate) + } +} + +object OpenApiConsumersTable : UUIDTable("open_api_consumers") { + val kongConsumerId = varchar("kong_consumer_id", 80).uniqueIndex() + val username = varchar("username", 160).default("") + val customId = varchar("custom_id", 160).default("") + val displayName = varchar("display_name", 160).default("") + val description = text("description").default("") + val firstSeenAt = timestamp("first_seen_at") + val lastSeenAt = timestamp("last_seen_at") + val status = varchar("status", 32).default("active") +} + +object OpenApiUsageHourlyTable : UUIDTable("open_api_usage_hourly") { + val bucketTime = timestamp("bucket_time") + val endpointId = reference("endpoint_id", OpenApiEndpointsTable) + val callerType = varchar("caller_type", 24) + val callerKey = varchar("caller_key", 160) + val consumerId = varchar("consumer_id", 80).default("") + val consumerUsername = varchar("consumer_username", 160).default("") + val clientIp = varchar("client_ip", 80).default("") + val callCount = long("call_count").default(0) + val successCount = long("success_count").default(0) + val failureCount = long("failure_count").default(0) + val unauthorizedCount = long("unauthorized_count").default(0) + val firstCalledAt = timestamp("first_called_at") + val lastCalledAt = timestamp("last_called_at") + val updatedAt = timestamp("updated_at") + + init { + uniqueIndex(bucketTime, endpointId, callerType, callerKey) + } +} + +object OpenApiIngestDedupTable : Table("open_api_ingest_dedup") { + val requestId = varchar("request_id", 100) + val receivedAt = timestamp("received_at") + + override val primaryKey = PrimaryKey(requestId) +} diff --git a/ktor/src/main/kotlin/ink/snowflake/server/model/request/OpenApiStatisticsRequest.kt b/ktor/src/main/kotlin/ink/snowflake/server/model/request/OpenApiStatisticsRequest.kt new file mode 100644 index 0000000..971d108 --- /dev/null +++ b/ktor/src/main/kotlin/ink/snowflake/server/model/request/OpenApiStatisticsRequest.kt @@ -0,0 +1,25 @@ +package ink.snowflake.server.model.request + +import kotlinx.serialization.Serializable + +@Serializable +data class SaveOpenApiEndpointRequest( + val name: String, + val serviceCode: String, + val category: String, + val httpMethod: String = "*", + val pathTemplate: String, + val kongServiceId: String = "", + val kongRouteId: String = "", + val description: String = "", + val owner: String = "", + val monitorEnabled: Boolean = true, + val status: String = "enabled", +) + +@Serializable +data class UpdateOpenApiConsumerRequest( + val displayName: String = "", + val description: String = "", + val status: String = "active", +) diff --git a/ktor/src/main/kotlin/ink/snowflake/server/model/response/OpenApiStatisticsResponse.kt b/ktor/src/main/kotlin/ink/snowflake/server/model/response/OpenApiStatisticsResponse.kt new file mode 100644 index 0000000..ee2cd46 --- /dev/null +++ b/ktor/src/main/kotlin/ink/snowflake/server/model/response/OpenApiStatisticsResponse.kt @@ -0,0 +1,89 @@ +package ink.snowflake.server.model.response + +import kotlinx.serialization.Serializable + +@Serializable +data class OpenApiEndpointResponse( + val id: String, + val name: String, + val serviceCode: String, + val category: String, + val httpMethod: String, + val pathTemplate: String, + val kongServiceId: String, + val kongRouteId: String, + val description: String, + val owner: String, + val monitorEnabled: Boolean, + val status: String, + val recentCalls: Long, + val recentConsumers: Int, + val lastCalledAt: String, + val updatedAt: String, +) + +@Serializable +data class OpenApiTrendPointResponse( + val bucket: String, + val calls: Long, + val authenticatedCalls: Long, + val anonymousCalls: Long, + val failures: Long, +) + +@Serializable +data class OpenApiRankItemResponse( + val key: String, + val name: String, + val value: Long, + val secondary: String = "", +) + +@Serializable +data class OpenApiOverviewResponse( + val totalCalls: Long, + val authenticatedCalls: Long, + val anonymousCalls: Long, + val unauthorizedCalls: Long, + val activeConsumers: Int, + val activeIps: Int, + val successRate: Double, + val trend: List, + val topEndpoints: List, + val topConsumers: List, + val categoryDistribution: List, +) + +@Serializable +data class OpenApiConsumerResponse( + val id: String, + val kongConsumerId: String, + val username: String, + val customId: String, + val displayName: String, + val description: String, + val firstSeenAt: String, + val lastSeenAt: String, + val status: String, + val callCount: Long, + val endpointCount: Int, + val recentIp: String, +) + +@Serializable +data class OpenApiUsageResponse( + val bucket: String, + val endpointId: String, + val endpointName: String, + val serviceCode: String, + val category: String, + val callerType: String, + val callerName: String, + val consumerId: String, + val clientIp: String, + val callCount: Long, + val successCount: Long, + val failureCount: Long, + val unauthorizedCount: Long, + val lastCalledAt: String, +) diff --git a/ktor/src/main/kotlin/ink/snowflake/server/utils/AppConfig.kt b/ktor/src/main/kotlin/ink/snowflake/server/utils/AppConfig.kt index 62c188e..800e32c 100644 --- a/ktor/src/main/kotlin/ink/snowflake/server/utils/AppConfig.kt +++ b/ktor/src/main/kotlin/ink/snowflake/server/utils/AppConfig.kt @@ -33,4 +33,11 @@ class AppConfig(config: ApplicationConfig) { ?.trimEnd('/') ?.takeIf { it.isNotBlank() } ?: "http://127.0.0.1:8081" + val openApiStatisticsEnabled: Boolean = + config.propertyOrNull("ktor.open-api-statistics.enabled")?.getString()?.toBooleanStrictOrNull() ?: true + val openApiStatisticsIngestToken: String = + System.getenv("OPEN_API_LOG_TOKEN") + ?.trim() + ?.takeIf { it.isNotBlank() } + ?: config.propertyOrNull("ktor.open-api-statistics.ingest-token")?.getString()?.trim().orEmpty() } diff --git a/ktor/src/main/kotlin/ink/snowflake/server/utils/dao/OpenApiStatisticsDao.kt b/ktor/src/main/kotlin/ink/snowflake/server/utils/dao/OpenApiStatisticsDao.kt new file mode 100644 index 0000000..22754b2 --- /dev/null +++ b/ktor/src/main/kotlin/ink/snowflake/server/utils/dao/OpenApiStatisticsDao.kt @@ -0,0 +1,553 @@ +package ink.snowflake.server.utils.dao + +import com.google.gson.JsonElement +import com.google.gson.JsonObject +import com.google.gson.JsonParser +import ink.snowflake.server.model.database.OpenApiConsumersTable +import ink.snowflake.server.model.database.OpenApiEndpointsTable +import ink.snowflake.server.model.database.OpenApiIngestDedupTable +import ink.snowflake.server.model.database.OpenApiUsageHourlyTable +import ink.snowflake.server.model.request.SaveOpenApiEndpointRequest +import ink.snowflake.server.model.request.UpdateOpenApiConsumerRequest +import ink.snowflake.server.model.response.OpenApiConsumerResponse +import ink.snowflake.server.model.response.OpenApiEndpointResponse +import ink.snowflake.server.model.response.OpenApiOverviewResponse +import ink.snowflake.server.model.response.OpenApiRankItemResponse +import ink.snowflake.server.model.response.OpenApiTrendPointResponse +import ink.snowflake.server.model.response.OpenApiUsageResponse +import kotlinx.datetime.Clock +import kotlinx.datetime.Instant +import org.jetbrains.exposed.v1.core.ResultRow +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.SqlExpressionBuilder.eq +import org.jetbrains.exposed.v1.core.SqlExpressionBuilder.greaterEq +import org.jetbrains.exposed.v1.core.SqlExpressionBuilder.less +import org.jetbrains.exposed.v1.core.SqlExpressionBuilder.lessEq +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.datetime.timestampLiteral +import org.jetbrains.exposed.v1.jdbc.SchemaUtils +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.insertAndGetId +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.jetbrains.exposed.v1.jdbc.update +import java.util.UUID +import kotlin.math.round + +object OpenApiStatisticsDao { + private const val HOUR_MILLIS = 3_600_000L + private const val DEDUP_RETENTION_MILLIS = 7L * 24 * HOUR_MILLIS + + private data class SeedEndpoint( + val name: String, + val serviceCode: String, + val category: String, + val method: String, + val path: String, + val description: String, + ) + + private data class IngestEvent( + val requestId: String, + val method: String, + val uri: String, + val startedAt: Instant, + val serviceId: String, + val serviceName: String, + val routeId: String, + val consumerId: String, + val consumerUsername: String, + val consumerCustomId: String, + val clientIp: String, + val statusCode: Int, + ) + + private val seedEndpoints = listOf( + SeedEndpoint("蓝字发票开具", "invoice", "票通开票", "*", "/api/open/v1/blue-invoices", "创建蓝字发票"), + SeedEndpoint("蓝字发票详情", "invoice", "票通开票", "*", "/api/open/v1/blue-invoices/{invoiceCode}", "查询蓝字发票详情"), + SeedEndpoint("蓝字发票样例", "invoice", "票通开票", "*", "/api/open/v1/blue-invoices/sample/{invoiceCode}", "查询蓝字发票样例"), + SeedEndpoint("蓝字发票批次", "invoice", "票通开票", "*", "/api/open/v1/blue-invoices/batches", "创建或查询发票批次"), + SeedEndpoint("蓝字发票批次详情", "invoice", "票通开票", "*", "/api/open/v1/blue-invoices/batches/{batchCode}", "查询发票批次详情"), + SeedEndpoint("生产开票任务", "invoice", "票通开票", "*", "/api/open/v1/blue-invoice-tasks/production", "提交生产开票任务"), + SeedEndpoint("蚕茧票据指标识别", "ai-fastapi", "AI识别", "POST", "/api/public/recognize-cocoon-metrics", "识别蚕茧票据指标"), + SeedEndpoint("许可证识别", "ai-fastapi", "AI识别", "POST", "/api/public/recognize-license", "识别许可证图片"), + SeedEndpoint("蚕茧图片识别", "ai-fastapi", "AI识别", "POST", "/api/public/recognize-silkworm-cocoon", "识别蚕茧图片"), + SeedEndpoint("云哨记录分析", "ai-fastapi", "牧安云哨", "POST", "/api/public/sentinel-record-analytics", "提交牧安云哨分析任务"), + SeedEndpoint("开放批次创建", "ktor-core", "溯源开放", "POST", "/traceability/public/integration/batches", "通过开放接口创建溯源批次"), + SeedEndpoint("批次公开地址", "ktor-core", "溯源开放", "GET", "/traceability/public/integration/batches/{id}/public-url", "查询批次公开地址"), + SeedEndpoint("溯源详情查询", "ktor-core", "溯源开放", "GET", "/traceability/public/by-code/{code}", "通过批次编码查询公开信息"), + SeedEndpoint("溯源预览查询", "ktor-core", "溯源开放", "GET", "/traceability/public/preview/by-code/{code}", "查询未发布批次预览"), + SeedEndpoint("预演页查询", "ktor-core", "溯源开放", "GET", "/traceability/public/preview-page/by-code/{code}", "查询溯源预演页"), + SeedEndpoint("溯源意见反馈", "ktor-core", "溯源开放", "POST", "/traceability/public/feedback", "提交消费者反馈"), + SeedEndpoint("溯源公开页面", "ktor-core", "溯源开放", "GET", "/traceability/public/page/{code}", "访问公开溯源页面"), + SeedEndpoint("远程设备连接", "ktor-core", "运维开放", "GET", "/silk-remote/connectLocalDevice", "连接远程 ADB 设备"), + SeedEndpoint("远程设备断开", "ktor-core", "运维开放", "GET", "/silk-remote/disConnectAll", "断开远程 ADB 连接"), + ) + + fun initSchemaAndSeed() { + transaction { + SchemaUtils.createMissingTablesAndColumns( + OpenApiEndpointsTable, + OpenApiConsumersTable, + OpenApiUsageHourlyTable, + OpenApiIngestDedupTable, + ) + val now = timestampLiteral(Clock.System.now()) + seedEndpoints.forEach { seed -> + val exists = OpenApiEndpointsTable.selectAll().where { + (OpenApiEndpointsTable.serviceCode eq seed.serviceCode) and + (OpenApiEndpointsTable.httpMethod eq seed.method) and + (OpenApiEndpointsTable.pathTemplate eq seed.path) + }.any() + if (!exists) { + OpenApiEndpointsTable.insert { + it[name] = seed.name + it[serviceCode] = seed.serviceCode + it[category] = seed.category + it[httpMethod] = seed.method + it[pathTemplate] = seed.path + it[description] = seed.description + it[createdAt] = now + it[updatedAt] = now + } + } + } + } + } + + @Synchronized + fun ingest(rawBody: String): Int { + val root = runCatching { JsonParser.parseString(rawBody) }.getOrNull() ?: return 0 + val elements = if (root.isJsonArray) root.asJsonArray.toList() else listOf(root) + val events = elements.mapNotNull(::parseEvent) + if (events.isEmpty()) return 0 + + return transaction { + val now = Clock.System.now() + OpenApiIngestDedupTable.deleteWhere { + receivedAt less timestampLiteral(Instant.fromEpochMilliseconds(now.toEpochMilliseconds() - DEDUP_RETENTION_MILLIS)) + } + val endpoints = OpenApiEndpointsTable.selectAll() + .where { OpenApiEndpointsTable.monitorEnabled eq true } + .toList() + var accepted = 0 + + events.forEach { event -> + if (OpenApiIngestDedupTable.selectAll() + .where { OpenApiIngestDedupTable.requestId eq event.requestId } + .any() + ) { + return@forEach + } + val endpoint = matchEndpoint(event, endpoints) ?: return@forEach + OpenApiIngestDedupTable.insert { + it[requestId] = event.requestId + it[receivedAt] = timestampLiteral(now) + } + + if (event.consumerId.isNotBlank()) { + saveConsumer(event) + } + + val bucket = Instant.fromEpochMilliseconds( + event.startedAt.toEpochMilliseconds() / HOUR_MILLIS * HOUR_MILLIS + ) + val callerType = if (event.consumerId.isBlank()) "anonymous" else "consumer" + val callerKey = event.consumerId.ifBlank { event.clientIp.ifBlank { "unknown" } } + val endpointId = endpoint[OpenApiEndpointsTable.id].value + val existing = OpenApiUsageHourlyTable.selectAll().where { + (OpenApiUsageHourlyTable.bucketTime eq bucket) and + (OpenApiUsageHourlyTable.endpointId eq endpointId) and + (OpenApiUsageHourlyTable.callerType eq callerType) and + (OpenApiUsageHourlyTable.callerKey eq callerKey) + }.singleOrNull() + val success = event.statusCode in 200..399 + val unauthorized = event.statusCode == 401 || event.statusCode == 403 + + if (existing == null) { + OpenApiUsageHourlyTable.insert { + it[bucketTime] = timestampLiteral(bucket) + it[OpenApiUsageHourlyTable.endpointId] = endpointId + it[OpenApiUsageHourlyTable.callerType] = callerType + it[OpenApiUsageHourlyTable.callerKey] = callerKey + it[consumerId] = event.consumerId + it[consumerUsername] = event.consumerUsername + it[clientIp] = event.clientIp + it[callCount] = 1 + it[successCount] = if (success) 1 else 0 + it[failureCount] = if (success) 0 else 1 + it[unauthorizedCount] = if (unauthorized) 1 else 0 + it[firstCalledAt] = timestampLiteral(event.startedAt) + it[lastCalledAt] = timestampLiteral(event.startedAt) + it[updatedAt] = timestampLiteral(now) + } + } else { + OpenApiUsageHourlyTable.update({ OpenApiUsageHourlyTable.id eq existing[OpenApiUsageHourlyTable.id].value }) { + it[callCount] = existing[OpenApiUsageHourlyTable.callCount] + 1 + it[successCount] = existing[OpenApiUsageHourlyTable.successCount] + if (success) 1 else 0 + it[failureCount] = existing[OpenApiUsageHourlyTable.failureCount] + if (success) 0 else 1 + it[unauthorizedCount] = existing[OpenApiUsageHourlyTable.unauthorizedCount] + if (unauthorized) 1 else 0 + it[lastCalledAt] = timestampLiteral(maxOf(existing[OpenApiUsageHourlyTable.lastCalledAt], event.startedAt)) + it[updatedAt] = timestampLiteral(now) + } + } + accepted++ + } + accepted + } + } + + fun listEndpoints(): List = transaction { + val recentFrom = Instant.fromEpochMilliseconds(Clock.System.now().toEpochMilliseconds() - 24 * HOUR_MILLIS) + val recentRows = OpenApiUsageHourlyTable.selectAll() + .where { OpenApiUsageHourlyTable.bucketTime greaterEq timestampLiteral(recentFrom) } + .toList() + .groupBy { it[OpenApiUsageHourlyTable.endpointId].value } + OpenApiEndpointsTable.selectAll() + .orderBy(OpenApiEndpointsTable.serviceCode, SortOrder.ASC) + .map { row -> + val usage = recentRows[row[OpenApiEndpointsTable.id].value].orEmpty() + endpointResponse(row, usage) + } + } + + fun saveEndpoint(id: UUID?, request: SaveOpenApiEndpointRequest): OpenApiEndpointResponse = transaction { + val now = timestampLiteral(Clock.System.now()) + val endpointId = id ?: OpenApiEndpointsTable.insertAndGetId { + it[name] = request.name.trim() + it[serviceCode] = request.serviceCode.trim() + it[category] = request.category.trim() + it[httpMethod] = request.httpMethod.trim().uppercase().ifBlank { "*" } + it[pathTemplate] = normalizePath(request.pathTemplate) + it[kongServiceId] = request.kongServiceId.trim() + it[kongRouteId] = request.kongRouteId.trim() + it[description] = request.description.trim() + it[owner] = request.owner.trim() + it[monitorEnabled] = request.monitorEnabled + it[status] = request.status.trim().ifBlank { "enabled" } + it[createdAt] = now + it[updatedAt] = now + }.value + if (id != null) { + OpenApiEndpointsTable.update({ OpenApiEndpointsTable.id eq id }) { + it[name] = request.name.trim() + it[serviceCode] = request.serviceCode.trim() + it[category] = request.category.trim() + it[httpMethod] = request.httpMethod.trim().uppercase().ifBlank { "*" } + it[pathTemplate] = normalizePath(request.pathTemplate) + it[kongServiceId] = request.kongServiceId.trim() + it[kongRouteId] = request.kongRouteId.trim() + it[description] = request.description.trim() + it[owner] = request.owner.trim() + it[monitorEnabled] = request.monitorEnabled + it[status] = request.status.trim().ifBlank { "enabled" } + it[updatedAt] = now + } + } + val row = OpenApiEndpointsTable.selectAll() + .where { OpenApiEndpointsTable.id eq endpointId } + .single() + endpointResponse(row, emptyList()) + } + + fun overview( + from: Instant, + to: Instant, + serviceCode: String?, + category: String?, + ): OpenApiOverviewResponse = transaction { + val endpointRows = OpenApiEndpointsTable.selectAll().toList() + val endpointMap = endpointRows.associateBy { it[OpenApiEndpointsTable.id].value } + val allowedEndpointIds = endpointRows.asSequence() + .filter { serviceCode.isNullOrBlank() || it[OpenApiEndpointsTable.serviceCode] == serviceCode } + .filter { category.isNullOrBlank() || it[OpenApiEndpointsTable.category] == category } + .map { it[OpenApiEndpointsTable.id].value } + .toSet() + val rows = usageRows(from, to).filter { + it[OpenApiUsageHourlyTable.endpointId].value in allowedEndpointIds + } + val total = rows.sumOf { it[OpenApiUsageHourlyTable.callCount] } + val authenticated = rows.filter { it[OpenApiUsageHourlyTable.callerType] == "consumer" } + .sumOf { it[OpenApiUsageHourlyTable.callCount] } + val anonymous = total - authenticated + val unauthorized = rows.sumOf { it[OpenApiUsageHourlyTable.unauthorizedCount] } + val successes = rows.sumOf { it[OpenApiUsageHourlyTable.successCount] } + val useHourly = to.toEpochMilliseconds() - from.toEpochMilliseconds() <= 48 * HOUR_MILLIS + val trend = rows.groupBy { + val text = it[OpenApiUsageHourlyTable.bucketTime].toString() + if (useHourly) text.take(13) + ":00:00Z" else text.take(10) + }.toSortedMap().map { (bucket, items) -> + OpenApiTrendPointResponse( + bucket = bucket, + calls = items.sumOf { it[OpenApiUsageHourlyTable.callCount] }, + authenticatedCalls = items.filter { it[OpenApiUsageHourlyTable.callerType] == "consumer" } + .sumOf { it[OpenApiUsageHourlyTable.callCount] }, + anonymousCalls = items.filter { it[OpenApiUsageHourlyTable.callerType] == "anonymous" } + .sumOf { it[OpenApiUsageHourlyTable.callCount] }, + failures = items.sumOf { it[OpenApiUsageHourlyTable.failureCount] }, + ) + } + val topEndpoints = rows.groupBy { it[OpenApiUsageHourlyTable.endpointId].value } + .map { (id, items) -> + val endpoint = endpointMap[id] + OpenApiRankItemResponse( + key = id.toString(), + name = endpoint?.get(OpenApiEndpointsTable.name) ?: "未知接口", + value = items.sumOf { it[OpenApiUsageHourlyTable.callCount] }, + secondary = endpoint?.get(OpenApiEndpointsTable.serviceCode).orEmpty(), + ) + }.sortedByDescending { it.value }.take(8) + val topConsumers = rows.groupBy { + if (it[OpenApiUsageHourlyTable.callerType] == "consumer") { + it[OpenApiUsageHourlyTable.consumerId] + } else { + "ip:${it[OpenApiUsageHourlyTable.clientIp]}" + } + }.map { (key, items) -> + val first = items.first() + val name = if (first[OpenApiUsageHourlyTable.callerType] == "consumer") { + first[OpenApiUsageHourlyTable.consumerUsername].ifBlank { key } + } else { + "匿名 · ${first[OpenApiUsageHourlyTable.clientIp].ifBlank { "未知 IP" }}" + } + OpenApiRankItemResponse(key, name, items.sumOf { it[OpenApiUsageHourlyTable.callCount] }) + }.sortedByDescending { it.value }.take(8) + val categoryDistribution = rows.groupBy { + endpointMap[it[OpenApiUsageHourlyTable.endpointId].value]?.get(OpenApiEndpointsTable.category) ?: "其他" + }.map { (name, items) -> + OpenApiRankItemResponse(name, name, items.sumOf { it[OpenApiUsageHourlyTable.callCount] }) + }.sortedByDescending { it.value } + + OpenApiOverviewResponse( + totalCalls = total, + authenticatedCalls = authenticated, + anonymousCalls = anonymous, + unauthorizedCalls = unauthorized, + activeConsumers = rows.map { it[OpenApiUsageHourlyTable.consumerId] }.filter { it.isNotBlank() }.distinct().size, + activeIps = rows.map { it[OpenApiUsageHourlyTable.clientIp] }.filter { it.isNotBlank() }.distinct().size, + successRate = if (total == 0L) 100.0 else round(successes * 10_000.0 / total) / 100.0, + trend = trend, + topEndpoints = topEndpoints, + topConsumers = topConsumers, + categoryDistribution = categoryDistribution, + ) + } + + fun listConsumers(): List = transaction { + val usageByConsumer = OpenApiUsageHourlyTable.selectAll() + .where { OpenApiUsageHourlyTable.consumerId neq "" } + .toList() + .groupBy { it[OpenApiUsageHourlyTable.consumerId] } + OpenApiConsumersTable.selectAll() + .orderBy(OpenApiConsumersTable.lastSeenAt, SortOrder.DESC) + .map { row -> + val usage = usageByConsumer[row[OpenApiConsumersTable.kongConsumerId]].orEmpty() + OpenApiConsumerResponse( + id = row[OpenApiConsumersTable.id].value.toString(), + kongConsumerId = row[OpenApiConsumersTable.kongConsumerId], + username = row[OpenApiConsumersTable.username], + customId = row[OpenApiConsumersTable.customId], + displayName = row[OpenApiConsumersTable.displayName], + description = row[OpenApiConsumersTable.description], + firstSeenAt = row[OpenApiConsumersTable.firstSeenAt].toString(), + lastSeenAt = row[OpenApiConsumersTable.lastSeenAt].toString(), + status = row[OpenApiConsumersTable.status], + callCount = usage.sumOf { it[OpenApiUsageHourlyTable.callCount] }, + endpointCount = usage.map { it[OpenApiUsageHourlyTable.endpointId].value }.distinct().size, + recentIp = usage.maxByOrNull { it[OpenApiUsageHourlyTable.lastCalledAt] } + ?.get(OpenApiUsageHourlyTable.clientIp).orEmpty(), + ) + } + } + + fun updateConsumer(id: UUID, request: UpdateOpenApiConsumerRequest): OpenApiConsumerResponse? { + transaction { + OpenApiConsumersTable.update({ OpenApiConsumersTable.id eq id }) { + it[displayName] = request.displayName.trim() + it[description] = request.description.trim() + it[status] = request.status.trim().ifBlank { "active" } + } + } + return listConsumers().firstOrNull { it.id == id.toString() } + } + + fun listUsage( + from: Instant, + to: Instant, + serviceCode: String?, + category: String?, + endpointId: UUID?, + callerType: String?, + ): List = transaction { + val endpointRows = OpenApiEndpointsTable.selectAll().toList() + val endpointMap = endpointRows.associateBy { it[OpenApiEndpointsTable.id].value } + usageRows(from, to).asSequence() + .filter { endpointId == null || it[OpenApiUsageHourlyTable.endpointId].value == endpointId } + .filter { callerType.isNullOrBlank() || it[OpenApiUsageHourlyTable.callerType] == callerType } + .filter { + val endpoint = endpointMap[it[OpenApiUsageHourlyTable.endpointId].value] + (serviceCode.isNullOrBlank() || endpoint?.get(OpenApiEndpointsTable.serviceCode) == serviceCode) && + (category.isNullOrBlank() || endpoint?.get(OpenApiEndpointsTable.category) == category) + } + .sortedByDescending { it[OpenApiUsageHourlyTable.bucketTime] } + .take(1_000) + .map { row -> + val endpoint = endpointMap[row[OpenApiUsageHourlyTable.endpointId].value] + val callerName = if (row[OpenApiUsageHourlyTable.callerType] == "consumer") { + row[OpenApiUsageHourlyTable.consumerUsername] + .ifBlank { row[OpenApiUsageHourlyTable.consumerId] } + } else { + "匿名调用方" + } + OpenApiUsageResponse( + bucket = row[OpenApiUsageHourlyTable.bucketTime].toString(), + endpointId = row[OpenApiUsageHourlyTable.endpointId].value.toString(), + endpointName = endpoint?.get(OpenApiEndpointsTable.name) ?: "未知接口", + serviceCode = endpoint?.get(OpenApiEndpointsTable.serviceCode).orEmpty(), + category = endpoint?.get(OpenApiEndpointsTable.category).orEmpty(), + callerType = row[OpenApiUsageHourlyTable.callerType], + callerName = callerName, + consumerId = row[OpenApiUsageHourlyTable.consumerId], + clientIp = row[OpenApiUsageHourlyTable.clientIp], + callCount = row[OpenApiUsageHourlyTable.callCount], + successCount = row[OpenApiUsageHourlyTable.successCount], + failureCount = row[OpenApiUsageHourlyTable.failureCount], + unauthorizedCount = row[OpenApiUsageHourlyTable.unauthorizedCount], + lastCalledAt = row[OpenApiUsageHourlyTable.lastCalledAt].toString(), + ) + }.toList() + } + + private fun usageRows(from: Instant, to: Instant): List { + return OpenApiUsageHourlyTable.selectAll().where { + (OpenApiUsageHourlyTable.bucketTime greaterEq timestampLiteral(from)) and + (OpenApiUsageHourlyTable.bucketTime lessEq timestampLiteral(to)) + }.toList() + } + + private fun endpointResponse(row: ResultRow, usage: List): OpenApiEndpointResponse { + return OpenApiEndpointResponse( + id = row[OpenApiEndpointsTable.id].value.toString(), + name = row[OpenApiEndpointsTable.name], + serviceCode = row[OpenApiEndpointsTable.serviceCode], + category = row[OpenApiEndpointsTable.category], + httpMethod = row[OpenApiEndpointsTable.httpMethod], + pathTemplate = row[OpenApiEndpointsTable.pathTemplate], + kongServiceId = row[OpenApiEndpointsTable.kongServiceId], + kongRouteId = row[OpenApiEndpointsTable.kongRouteId], + description = row[OpenApiEndpointsTable.description], + owner = row[OpenApiEndpointsTable.owner], + monitorEnabled = row[OpenApiEndpointsTable.monitorEnabled], + status = row[OpenApiEndpointsTable.status], + recentCalls = usage.sumOf { it[OpenApiUsageHourlyTable.callCount] }, + recentConsumers = usage.map { it[OpenApiUsageHourlyTable.consumerId] }.filter { it.isNotBlank() }.distinct().size, + lastCalledAt = usage.maxByOrNull { it[OpenApiUsageHourlyTable.lastCalledAt] } + ?.get(OpenApiUsageHourlyTable.lastCalledAt)?.toString().orEmpty(), + updatedAt = row[OpenApiEndpointsTable.updatedAt].toString(), + ) + } + + private fun saveConsumer(event: IngestEvent) { + val row = OpenApiConsumersTable.selectAll() + .where { OpenApiConsumersTable.kongConsumerId eq event.consumerId } + .singleOrNull() + if (row == null) { + OpenApiConsumersTable.insert { + it[kongConsumerId] = event.consumerId + it[username] = event.consumerUsername + it[customId] = event.consumerCustomId + it[displayName] = event.consumerUsername + it[firstSeenAt] = timestampLiteral(event.startedAt) + it[lastSeenAt] = timestampLiteral(event.startedAt) + } + } else { + OpenApiConsumersTable.update({ OpenApiConsumersTable.id eq row[OpenApiConsumersTable.id].value }) { + it[username] = event.consumerUsername + it[customId] = event.consumerCustomId + it[lastSeenAt] = timestampLiteral(maxOf(row[OpenApiConsumersTable.lastSeenAt], event.startedAt)) + } + } + } + + private fun matchEndpoint(event: IngestEvent, endpoints: List): ResultRow? { + val path = normalizePath(event.uri.substringBefore('?')) + return endpoints.asSequence() + .filter { + val method = it[OpenApiEndpointsTable.httpMethod] + method == "*" || method.equals(event.method, ignoreCase = true) + } + .filter { pathMatches(it[OpenApiEndpointsTable.pathTemplate], path) } + .maxByOrNull { + var score = it[OpenApiEndpointsTable.pathTemplate].split('/').count { segment -> + segment.isNotBlank() && !(segment.startsWith("{") && segment.endsWith("}")) + } * 10 + if (event.routeId.isNotBlank() && it[OpenApiEndpointsTable.kongRouteId] == event.routeId) score += 1_000 + if (event.serviceId.isNotBlank() && it[OpenApiEndpointsTable.kongServiceId] == event.serviceId) score += 500 + if (event.serviceName.contains(it[OpenApiEndpointsTable.serviceCode], ignoreCase = true)) score += 100 + score + } + } + + private fun pathMatches(template: String, actual: String): Boolean { + val templateParts = normalizePath(template).trim('/').split('/').filter { it.isNotBlank() } + val actualParts = normalizePath(actual).trim('/').split('/').filter { it.isNotBlank() } + if (templateParts.size != actualParts.size) return false + return templateParts.zip(actualParts).all { (expected, value) -> + (expected.startsWith("{") && expected.endsWith("}")) || expected == value + } + } + + private fun normalizePath(path: String): String { + val normalized = "/" + path.trim().trim('/').replace(Regex("/+"), "/") + return if (normalized == "/") normalized else normalized.trimEnd('/') + } + + private fun parseEvent(element: JsonElement): IngestEvent? { + if (!element.isJsonObject) return null + val root = element.asJsonObject + val request = root.objectOrNull("request") ?: return null + val response = root.objectOrNull("response") + val service = root.objectOrNull("service") + val route = root.objectOrNull("route") + val consumer = root.objectOrNull("consumer") + val requestId = request.string("id") + val uri = request.string("uri") + if (requestId.isBlank() || uri.isBlank()) return null + val startedAtMillis = root.long("started_at").takeIf { it > 0 } ?: Clock.System.now().toEpochMilliseconds() + return IngestEvent( + requestId = requestId, + method = request.string("method").uppercase().ifBlank { "GET" }, + uri = uri, + startedAt = Instant.fromEpochMilliseconds(startedAtMillis), + serviceId = service?.string("id").orEmpty(), + serviceName = service?.string("name").orEmpty(), + routeId = route?.string("id").orEmpty(), + consumerId = consumer?.string("id").orEmpty(), + consumerUsername = consumer?.string("username").orEmpty(), + consumerCustomId = consumer?.string("custom_id").orEmpty(), + clientIp = root.string("client_ip"), + statusCode = response?.int("status") ?: 0, + ) + } + + private fun JsonObject.objectOrNull(name: String): JsonObject? { + val value = get(name) ?: return null + return if (value.isJsonObject) value.asJsonObject else null + } + + private fun JsonObject.string(name: String): String { + val value = get(name) ?: return "" + return runCatching { if (value.isJsonNull) "" else value.asString }.getOrDefault("") + } + + private fun JsonObject.long(name: String): Long { + val value = get(name) ?: return 0 + return runCatching { value.asLong }.getOrDefault(0) + } + + private fun JsonObject.int(name: String): Int { + val value = get(name) ?: return 0 + return runCatching { value.asInt }.getOrDefault(0) + } +} diff --git a/ktor/src/main/resources/application.yaml b/ktor/src/main/resources/application.yaml index 60dc279..6d9b8b9 100644 --- a/ktor/src/main/resources/application.yaml +++ b/ktor/src/main/resources/application.yaml @@ -44,3 +44,8 @@ ktor: # public-preview-base-url: "http://127.0.0.1:8081" # 开发测试用 public-preview-base-url: "https://trace.bbitcn.net" # 生产环境用 + open-api-statistics: + enabled: true + # 生产环境请通过 OPEN_API_LOG_TOKEN 环境变量覆盖 + ingest-token: "change-me-before-deploy" + diff --git a/vue2/apps/web-antd/src/api/llm/report-cube.ts b/vue2/apps/web-antd/src/api/llm/report-cube.ts index 9751ee7..90d7273 100644 --- a/vue2/apps/web-antd/src/api/llm/report-cube.ts +++ b/vue2/apps/web-antd/src/api/llm/report-cube.ts @@ -108,6 +108,10 @@ export interface CubeReportExportResult { url: string; } +export interface CubeReportExportRequest { + columnKeys: string[]; +} + export type CubeReportStreamEvent = | { content: string; type: 'message_delta' | 'message_replace' } | { @@ -233,7 +237,7 @@ export async function getCubeReportSession(id: string) { export async function getCubeReportData(id: string, params: { page: number }) { return pyRequestClient.get( `/llm/cube-report/sessions/${id}/data`, - { params }, + { params, showErrorMessage: false }, ); } @@ -243,19 +247,27 @@ export async function getCubeSavedReportData( ) { return pyRequestClient.get( `/llm/cube-report/reports/${id}/data`, - { params }, + { params, showErrorMessage: false }, ); } -export async function exportCubeReportSession(id: string) { +export async function exportCubeReportSession( + id: string, + data: CubeReportExportRequest, +) { return pyRequestClient.post( `/llm/cube-report/sessions/${id}/export`, + data, ); } -export async function exportCubeSavedReport(id: string) { +export async function exportCubeSavedReport( + id: string, + data: CubeReportExportRequest, +) { return pyRequestClient.post( `/llm/cube-report/reports/${id}/export`, + data, ); } diff --git a/vue2/apps/web-antd/src/api/open-api/index.ts b/vue2/apps/web-antd/src/api/open-api/index.ts new file mode 100644 index 0000000..2e63b58 --- /dev/null +++ b/vue2/apps/web-antd/src/api/open-api/index.ts @@ -0,0 +1,154 @@ +import { requestClient } from '#/api/request'; + +export namespace OpenApiCenterApi { + export interface Endpoint { + id: string; + name: string; + serviceCode: string; + category: string; + httpMethod: string; + pathTemplate: string; + kongServiceId: string; + kongRouteId: string; + description: string; + owner: string; + monitorEnabled: boolean; + status: string; + recentCalls: number; + recentConsumers: number; + lastCalledAt: string; + updatedAt: string; + } + + export interface TrendPoint { + bucket: string; + calls: number; + authenticatedCalls: number; + anonymousCalls: number; + failures: number; + } + + export interface RankItem { + key: string; + name: string; + value: number; + secondary: string; + } + + export interface Overview { + totalCalls: number; + authenticatedCalls: number; + anonymousCalls: number; + unauthorizedCalls: number; + activeConsumers: number; + activeIps: number; + successRate: number; + trend: TrendPoint[]; + topEndpoints: RankItem[]; + topConsumers: RankItem[]; + categoryDistribution: RankItem[]; + } + + export interface Consumer { + id: string; + kongConsumerId: string; + username: string; + customId: string; + displayName: string; + description: string; + firstSeenAt: string; + lastSeenAt: string; + status: string; + callCount: number; + endpointCount: number; + recentIp: string; + } + + export interface Usage { + bucket: string; + endpointId: string; + endpointName: string; + serviceCode: string; + category: string; + callerType: string; + callerName: string; + consumerId: string; + clientIp: string; + callCount: number; + successCount: number; + failureCount: number; + unauthorizedCount: number; + lastCalledAt: string; + } + + export interface Query { + from?: string; + to?: string; + serviceCode?: string; + category?: string; + endpointId?: string; + callerType?: string; + } + + export type SaveEndpoint = Pick< + Endpoint, + | 'category' + | 'description' + | 'httpMethod' + | 'kongRouteId' + | 'kongServiceId' + | 'monitorEnabled' + | 'name' + | 'owner' + | 'pathTemplate' + | 'serviceCode' + | 'status' + >; +} + +export function getOpenApiOverview(params: OpenApiCenterApi.Query) { + return requestClient.get('/open-api/overview', { + params, + }); +} + +export function getOpenApiEndpoints() { + return requestClient.get('/open-api/endpoints'); +} + +export function createOpenApiEndpoint(data: OpenApiCenterApi.SaveEndpoint) { + return requestClient.post( + '/open-api/endpoints', + data, + ); +} + +export function updateOpenApiEndpoint( + id: string, + data: OpenApiCenterApi.SaveEndpoint, +) { + return requestClient.put( + `/open-api/endpoints/${id}`, + data, + ); +} + +export function getOpenApiConsumers() { + return requestClient.get('/open-api/consumers'); +} + +export function updateOpenApiConsumer( + id: string, + data: Pick, +) { + return requestClient.put( + `/open-api/consumers/${id}`, + data, + ); +} + +export function getOpenApiUsage(params: OpenApiCenterApi.Query) { + return requestClient.get('/open-api/usage', { + params, + }); +} diff --git a/vue2/apps/web-antd/src/api/request.ts b/vue2/apps/web-antd/src/api/request.ts index 73639e1..b75c1d1 100644 --- a/vue2/apps/web-antd/src/api/request.ts +++ b/vue2/apps/web-antd/src/api/request.ts @@ -101,10 +101,14 @@ function createRequestClient(baseURL: string, options?: RequestClientOptions) { // 通用的错误处理,如果没有进入上面的错误处理逻辑，就会进入这里 client.addResponseInterceptor( errorMessageResponseInterceptor((msg: string, error) => { + if (error?.config?.showErrorMessage === false) return; // 这里可以根据业务进行定制,你可以拿到 error 内的信息进行定制化处理，根据不同的 code 做不同的提示，而不是直接使用 message.error 提示 msg // 当前mock接口返回的错误字段是 error 或者 message const responseData = error?.response?.data ?? {}; - const errorMessage = responseData?.error ?? responseData?.message ?? ''; + const detailMessage = + typeof responseData?.detail === 'string' ? responseData.detail : ''; + const errorMessage = + detailMessage || responseData?.error || responseData?.message || ''; // 如果没有错误信息，则会根据状态码进行提示 message.error(errorMessage || msg); }), diff --git a/vue2/apps/web-antd/src/router/routes/modules/open-api.ts b/vue2/apps/web-antd/src/router/routes/modules/open-api.ts new file mode 100644 index 0000000..e2f955f --- /dev/null +++ b/vue2/apps/web-antd/src/router/routes/modules/open-api.ts @@ -0,0 +1,18 @@ +import type { RouteRecordRaw } from 'vue-router'; + +const routes: RouteRecordRaw[] = [ + { + component: () => import('#/views/open-api/index.vue'), + meta: { + authority: ['manager'], + icon: 'lucide:waypoints', + keepAlive: true, + order: 3, + title: '开放接口中心', + }, + name: 'OpenApiCenter', + path: '/open-api/center', + }, +]; + +export default routes; diff --git a/vue2/apps/web-antd/src/views/llm/report/report-cube-chat/cube-data-grid.vue b/vue2/apps/web-antd/src/views/llm/report/report-cube-chat/cube-data-grid.vue new file mode 100644 index 0000000..f7184c8 --- /dev/null +++ b/vue2/apps/web-antd/src/views/llm/report/report-cube-chat/cube-data-grid.vue @@ -0,0 +1,523 @@ + + + + + diff --git a/vue2/apps/web-antd/src/views/llm/report/report-cube-chat/index.vue b/vue2/apps/web-antd/src/views/llm/report/report-cube-chat/index.vue index 2dda306..b5f7423 100644 --- a/vue2/apps/web-antd/src/views/llm/report/report-cube-chat/index.vue +++ b/vue2/apps/web-antd/src/views/llm/report/report-cube-chat/index.vue @@ -2,7 +2,6 @@ import type { DefaultOptionType } from 'ant-design-vue/es/select'; import type { - CubeReportColumn, CubeReportFile, CubeReportMessage, CubeReportMetaStatus, @@ -32,7 +31,6 @@ import { Segmented, Select, Spin, - Table, Tag, Textarea, Tooltip, @@ -41,7 +39,15 @@ import dayjs from 'dayjs'; import * as api from '#/api'; +import CubeDataGrid from './cube-data-grid.vue'; + type WorkspaceView = 'chat' | 'data' | 'split'; +interface DataLoadError { + detail: string; + hint?: string; + message: string; + title: string; +} const reports = ref([]); const reportPage = ref(1); @@ -78,6 +84,7 @@ const loadingSessions = ref(false); const loadingMoreSessions = ref(false); const loadingConversation = ref(false); const loadingData = ref(false); +const dataLoadError = ref(null); const exporting = ref(false); const favoriting = ref(false); const refreshingCubeMetadata = ref(false); @@ -92,11 +99,9 @@ const renameValue = ref(''); const renaming = ref(false); const chatContainer = ref(null); const panelContainer = ref(null); -const tableRegion = ref(null); -const tableScrollY = ref(240); +const dataGridRef = ref | null>(null); const fileInput = ref(null); const cubeMetadataStatus = ref(null); -let tableResizeObserver: null | ResizeObserver = null; let searchTimer: ReturnType | undefined; const selectedCompany = computed(() => @@ -117,14 +122,8 @@ const hasData = computed( Boolean(currentSession.value?.hasData) || dataPage.value.columns.length > 0, ); -const tableColumns = computed(() => - dataPage.value.columns.map((column: CubeReportColumn) => ({ - key: column.key, - dataIndex: column.key, - title: column.title, - ellipsis: true, - minWidth: 140, - })), +const dataGridStorageKey = computed( + () => currentReportId.value || currentSessionId.value || 'draft', ); const dataPanelStyle = computed(() => activeView.value === 'split' @@ -169,6 +168,7 @@ async function scrollToBottom() { } function resetData() { + dataLoadError.value = null; dataPage.value = { columns: [], page: 1, @@ -178,6 +178,59 @@ function resetData() { }; } +function getErrorDetail(error: unknown) { + if (typeof error === 'string') return error; + if (error && typeof error === 'object') { + const payload = error as Record; + if (typeof payload.detail === 'string') return payload.detail; + if (typeof payload.message === 'string') return payload.message; + } + return '数据查询失败，请稍后重试'; +} + +function extractCubeError(detail: string) { + const prefix = 'Cube SQL 生成失败：'; + if (!detail.startsWith(prefix)) return detail; + const rawPayload = detail.slice(prefix.length).trim(); + try { + const payload = JSON.parse(rawPayload) as Record; + if (typeof payload.error === 'string') return payload.error; + } catch { + const match = rawPayload.match(/"error"\s*:\s*"((?:\\.|[^"\\])*)"/); + if (match?.[1]) { + try { + return JSON.parse(`"${match[1]}"`) as string; + } catch { + return match[1]; + } + } + } + return detail; +} + +function createDataLoadError(error: unknown): DataLoadError { + const detail = getErrorDetail(error); + const cubeError = extractCubeError(detail); + const missingField = cubeError.match( + /'([^']+)' not found for path '([^']+)'/, + ); + if (detail.startsWith('Cube SQL 生成失败：')) { + return { + detail, + hint: missingField + ? `Cube 模型中找不到字段 ${missingField[2]}（成员 ${missingField[1]}）。字段可能已被删除、重命名或元数据尚未同步。` + : 'Cube 无法根据当前报表配置生成 SQL，请检查报表字段或刷新 Cube 元数据后重试。', + message: cubeError, + title: missingField ? 'Cube 字段不可用' : 'Cube SQL 生成失败', + }; + } + return { + detail, + message: detail, + title: '数据加载失败', + }; +} + async function loadScope() { const scope = await api.getCubeReportScope(); companies.value = scope.companies; @@ -309,23 +362,44 @@ async function loadReport(id: string) { const report = await api.getCubeReport(id); currentReport.value = report; currentSessionId.value = report.conversationId; - try { - const detail = await api.getCubeReportSession(report.conversationId); - currentSession.value = detail.session; - messages.value = detail.messages; - } catch { - currentSession.value = { - createdAt: report.createdAt, - hasData: true, - id: report.conversationId, - reportId: report.id, - tenantId: report.tenantId, - tenantName: report.tenantName, - title: report.title, - updatedAt: report.updatedAt, - }; - messages.value = []; - } + currentSession.value = { + createdAt: report.createdAt, + hasData: true, + id: report.conversationId, + reportId: report.id, + tenantId: report.tenantId, + tenantName: report.tenantName, + title: report.title, + updatedAt: report.updatedAt, + }; + messages.value = [ + ...(report.requirement + ? [ + { + content: report.requirement, + createdAt: report.createdAt, + difyMessageId: null, + files: [], + id: `saved-report-${report.id}-user`, + role: 'user' as const, + status: 'completed' as const, + }, + ] + : []), + ...(report.message + ? [ + { + content: report.message, + createdAt: report.updatedAt, + difyMessageId: null, + files: [], + id: `saved-report-${report.id}-assistant`, + role: 'assistant' as const, + status: 'completed' as const, + }, + ] + : []), + ]; pendingFiles.value = []; resetData(); activeView.value = 'split'; @@ -496,11 +570,14 @@ async function stopResponse() { async function loadData(page = dataPage.value.page) { if (!currentSessionId.value) return; + dataLoadError.value = null; loadingData.value = true; try { dataPage.value = currentReportId.value ? await api.getCubeSavedReportData(currentReportId.value, { page }) : await api.getCubeReportData(currentSessionId.value, { page }); + } catch (error) { + dataLoadError.value = createDataLoadError(error); } finally { loadingData.value = false; } @@ -520,13 +597,32 @@ async function copyCurrentSql() { } } +async function copyDataError() { + if (!dataLoadError.value) return; + try { + await navigator.clipboard.writeText(dataLoadError.value.detail); + message.success('错误详情已复制'); + } catch { + message.error('复制失败，请手动选择错误详情'); + } +} + async function exportCurrentData() { if (!currentSessionId.value || exporting.value) return; + const exportColumns = + dataGridRef.value?.getExportColumns() ?? dataPage.value.columns; + const columnKeys = exportColumns.map((column) => column.key); + if (columnKeys.length === 0) { + message.warning('请至少保留一列后再导出'); + return; + } exporting.value = true; try { const result = currentReportId.value - ? await api.exportCubeSavedReport(currentReportId.value) - : await api.exportCubeReportSession(currentSessionId.value); + ? await api.exportCubeSavedReport(currentReportId.value, { columnKeys }) + : await api.exportCubeReportSession(currentSessionId.value, { + columnKeys, + }); const anchor = document.createElement('a'); anchor.href = result.url; anchor.download = result.filename; @@ -534,7 +630,9 @@ async function exportCurrentData() { document.body.append(anchor); anchor.click(); anchor.remove(); - message.success(`已生成 ${result.rowCount} 条数据`); + message.success( + `已生成 ${result.rowCount} 条数据，共 ${columnKeys.length} 列`, + ); } catch (error) { message.error(error instanceof Error ? error.message : '数据导出失败'); } finally { @@ -735,20 +833,6 @@ function stopResize() { resizing.value = false; } -function observeTableRegion() { - tableResizeObserver?.disconnect(); - if (!tableRegion.value) return; - tableResizeObserver = new ResizeObserver(([entry]) => { - if (!entry) return; - tableScrollY.value = Math.max( - 120, - Math.floor(entry.contentRect.height - 48), - ); - }); - tableResizeObserver.observe(tableRegion.value); -} - -watch(tableRegion, () => nextTick(observeTableRegion)); watch(sessionKeyword, () => { if (searchTimer) clearTimeout(searchTimer); searchTimer = setTimeout(() => void reloadNavigation(), 320); @@ -763,13 +847,11 @@ onMounted(async () => { loadAppParameters(), loadCubeMetadataStatus(), ]); - await nextTick(observeTableRegion); }); onBeforeUnmount(() => { window.removeEventListener('pointermove', handlePointerMove); window.removeEventListener('pointerup', stopResize); - tableResizeObserver?.disconnect(); if (searchTimer) clearTimeout(searchTimer); }); @@ -1021,18 +1103,20 @@ onBeforeUnmount(() => { 刷新表头 - + + + - - 
{{ dataPage.sql }}
- -
- - -
-
-
- +
+
+ +
+
+ 数据查询未完成 +

{{ dataLoadError.title }}

+

{{ dataLoadError.message }}

+

+ + {{ dataLoadError.hint }} +

+
+ + +
+
+ 查看技术详情 + 
{{ dataLoadError.detail }}
+
+
+
@@ -1628,6 +1740,116 @@ onBeforeUnmount(() => { overflow: hidden; flex: 1; } +.data-error-state { + display: flex; + min-height: 0; + overflow: auto; + flex: 1; + align-items: flex-start; + justify-content: center; + gap: 18px; + padding: clamp(34px, 7vh, 72px) clamp(20px, 5vw, 64px); + background: + radial-gradient(circle at 50% 0%, rgb(254 242 242 / 72%), transparent 45%), + linear-gradient(180deg, #fff, #fcfdff); +} +.data-error-icon { + display: grid; + width: 48px; + height: 48px; + flex: 0 0 auto; + place-items: center; + color: #dc2626; + font-size: 23px; + background: #fff1f2; + border: 1px solid #fecdd3; + border-radius: 14px; + box-shadow: 0 8px 22px rgb(190 18 60 / 9%); +} +.data-error-content { + width: min(680px, 100%); + min-width: 0; +} +.data-error-eyebrow { + display: block; + margin: 1px 0 5px; + color: #e11d48; + font-size: 11px; + font-weight: 650; + letter-spacing: 0.08em; +} +.data-error-content h2 { + margin: 0; + color: #172033; + font-size: 20px; + font-weight: 680; + line-height: 1.4; +} +.data-error-message { + margin: 10px 0 0; + color: #475569; + font-family: 'JetBrains Mono', 'Cascadia Code', Consolas, monospace; + font-size: 12px; + line-height: 1.75; + overflow-wrap: anywhere; +} +.data-error-hint { + display: flex; + align-items: flex-start; + gap: 8px; + margin: 16px 0 0; + padding: 11px 13px; + color: #854d0e; + font-size: 12px; + line-height: 1.65; + background: #fffbeb; + border: 1px solid #fde68a; + border-radius: 9px; +} +.data-error-hint svg { + flex: 0 0 auto; + margin-top: 2px; + color: #d97706; + font-size: 15px; +} +.data-error-actions { + display: flex; + flex-wrap: wrap; + gap: 9px; + margin-top: 18px; +} +.data-error-actions :deep(.ant-btn) { + border-radius: 8px; +} +.data-error-details { + margin-top: 17px; + overflow: hidden; + background: #f8fafc; + border: 1px solid #e2e8f0; + border-radius: 9px; +} +.data-error-details summary { + padding: 9px 12px; + color: #64748b; + font-size: 11px; + font-weight: 600; + cursor: pointer; + user-select: none; +} +.data-error-details pre { + max-height: 220px; + margin: 0; + overflow: auto; + padding: 12px; + color: #cbd5e1; + font-family: 'JetBrains Mono', 'Cascadia Code', Consolas, monospace; + font-size: 11px; + line-height: 1.65; + white-space: pre-wrap; + overflow-wrap: anywhere; + background: #0f172a; + border-top: 1px solid #1e293b; +} .sql-disclosure { flex-shrink: 0; overflow: hidden; @@ -1675,13 +1897,6 @@ onBeforeUnmount(() => { background: #0f172a; border-top: 1px solid #1e293b; } -.cell-value { - display: inline-block; - max-width: 360px; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; -} .pagination-row { display: flex; position: relative; @@ -2030,6 +2245,12 @@ onBeforeUnmount(() => { } } @media (max-width: 820px) { + .data-error-state { + align-items: stretch; + flex-direction: column; + justify-content: flex-start; + padding: 28px 20px; + } .session-sidebar { width: 220px; } diff --git a/vue2/apps/web-antd/src/views/open-api/index.vue b/vue2/apps/web-antd/src/views/open-api/index.vue new file mode 100644 index 0000000..ff589b9 --- /dev/null +++ b/vue2/apps/web-antd/src/views/open-api/index.vue @@ -0,0 +1,1476 @@ + + + + + diff --git a/vue2/packages/effects/request/src/request-client/types.ts b/vue2/packages/effects/request/src/request-client/types.ts index d40ee8a..5fc8dd2 100644 --- a/vue2/packages/effects/request/src/request-client/types.ts +++ b/vue2/packages/effects/request/src/request-client/types.ts @@ -26,6 +26,11 @@ type ExtendOptions = { * - data: 解构响应的BODY数据，只返回其中的data节点数据（会检查status和code是否为成功状态）。 */ responseReturn?: 'body' | 'data' | 'raw'; + /** + * 是否由全局响应拦截器显示错误消息。 + * 页面需要使用内联错误态时可设为 false。 + */ + showErrorMessage?: boolean; }; type RequestClientConfig = AxiosRequestConfig & ExtendOptions; diff --git a/vue2/scripts/push_docker.ps1 b/vue2/scripts/push_docker.ps1 index db3520a..1dcbefb 100644 --- a/vue2/scripts/push_docker.ps1 +++ b/vue2/scripts/push_docker.ps1 @@ -1,7 +1,7 @@ # push_docker.ps1 # Set version -$env:VERSION = "1.5.7" +$env:VERSION = "1.5.8" # Docker registry/repository $registry = "docker.bbitcn.net/bbit_ai/ce_vue"