API统计页面

This commit is contained in:
BBIT-Kai
2026-07-27 10:16:24 +08:00
parent 89f6944e65
commit 38efd8ef1a
21 changed files with 3554 additions and 123 deletions
@@ -5,6 +5,7 @@ import ink.snowflake.server.controller.User
import ink.snowflake.server.controller.chat
import ink.snowflake.server.utils.plugins.configureSockets
import ink.snowflake.server.controller.ImageAnalytics
import ink.snowflake.server.controller.OpenApiStatistics
import ink.snowflake.server.controller.Public
import ink.snowflake.server.controller.RemoteDebug
import ink.snowflake.server.controller.Traceability
@@ -78,6 +79,8 @@ fun Application.module() {
// 业务-图片分析
ImageAnalytics()
Traceability(appConfig)
// 开放接口资产与 Kong 调用统计
OpenApiStatistics(appConfig)
// 业务-公开接口
Public()
}
@@ -0,0 +1,148 @@
package ink.snowflake.server.controller
import ink.snowflake.server.model.request.SaveOpenApiEndpointRequest
import ink.snowflake.server.model.request.UpdateOpenApiConsumerRequest
import ink.snowflake.server.model.response.BaseResponse
import ink.snowflake.server.utils.AppConfig
import ink.snowflake.server.utils.dao.OpenApiStatisticsDao
import io.ktor.http.HttpHeaders
import io.ktor.http.HttpStatusCode
import io.ktor.server.application.Application
import io.ktor.server.auth.authenticate
import io.ktor.server.request.receive
import io.ktor.server.request.receiveText
import io.ktor.server.response.respond
import io.ktor.server.routing.get
import io.ktor.server.routing.post
import io.ktor.server.routing.put
import io.ktor.server.routing.route
import io.ktor.server.routing.routing
import kotlinx.datetime.Clock
import kotlinx.datetime.Instant
import java.security.MessageDigest
import java.util.UUID
fun Application.OpenApiStatistics(config: AppConfig) {
OpenApiStatisticsDao.initSchemaAndSeed()
routing {
post("/internal/open-api/statistics/ingest") {
if (!config.openApiStatisticsEnabled) {
call.respond(HttpStatusCode.ServiceUnavailable)
return@post
}
val expected = "Bearer ${config.openApiStatisticsIngestToken}"
val actual = call.request.headers[HttpHeaders.Authorization].orEmpty()
if (
config.openApiStatisticsIngestToken.isBlank() ||
!MessageDigest.isEqual(expected.toByteArray(Charsets.UTF_8), actual.toByteArray(Charsets.UTF_8))
) {
call.respond(HttpStatusCode.Unauthorized)
return@post
}
OpenApiStatisticsDao.ingest(call.receiveText())
call.respond(HttpStatusCode.NoContent)
}
authenticate {
route("/open-api") {
get("/overview") {
val (from, to) = call.statisticsRange()
call.respond(
BaseResponse(
data = OpenApiStatisticsDao.overview(
from = from,
to = to,
serviceCode = call.request.queryParameters["serviceCode"],
category = call.request.queryParameters["category"],
)
)
)
}
get("/endpoints") {
call.respond(BaseResponse(data = OpenApiStatisticsDao.listEndpoints()))
}
post("/endpoints") {
val request = call.receive<SaveOpenApiEndpointRequest>()
if (request.name.isBlank() || request.serviceCode.isBlank() || request.pathTemplate.isBlank()) {
call.respond(
HttpStatusCode.BadRequest,
BaseResponse<Nothing>(status = false, message = "接口名称、所属服务和路径不能为空")
)
return@post
}
call.respond(BaseResponse(data = OpenApiStatisticsDao.saveEndpoint(null, request)))
}
put("/endpoints/{id}") {
val id = call.parameters["id"]?.let { runCatching { UUID.fromString(it) }.getOrNull() }
if (id == null) {
call.respond(
HttpStatusCode.BadRequest,
BaseResponse<Nothing>(status = false, message = "接口 ID 无效")
)
return@put
}
val request = call.receive<SaveOpenApiEndpointRequest>()
call.respond(BaseResponse(data = OpenApiStatisticsDao.saveEndpoint(id, request)))
}
get("/usage") {
val (from, to) = call.statisticsRange()
val endpointId = call.request.queryParameters["endpointId"]
?.let { runCatching { UUID.fromString(it) }.getOrNull() }
call.respond(
BaseResponse(
data = OpenApiStatisticsDao.listUsage(
from = from,
to = to,
serviceCode = call.request.queryParameters["serviceCode"],
category = call.request.queryParameters["category"],
endpointId = endpointId,
callerType = call.request.queryParameters["callerType"],
)
)
)
}
get("/consumers") {
call.respond(BaseResponse(data = OpenApiStatisticsDao.listConsumers()))
}
put("/consumers/{id}") {
val id = call.parameters["id"]?.let { runCatching { UUID.fromString(it) }.getOrNull() }
if (id == null) {
call.respond(
HttpStatusCode.BadRequest,
BaseResponse<Nothing>(status = false, message = "调用方 ID 无效")
)
return@put
}
val result = OpenApiStatisticsDao.updateConsumer(id, call.receive<UpdateOpenApiConsumerRequest>())
if (result == null) {
call.respond(
HttpStatusCode.NotFound,
BaseResponse<Nothing>(status = false, message = "调用方不存在")
)
return@put
}
call.respond(BaseResponse(data = result))
}
}
}
}
}
private fun io.ktor.server.application.ApplicationCall.statisticsRange(): Pair<Instant, Instant> {
val now = Clock.System.now()
val defaultFrom = Instant.fromEpochMilliseconds(now.toEpochMilliseconds() - 7L * 24 * 3_600_000)
val from = request.queryParameters["from"]
?.let { runCatching { Instant.parse(it) }.getOrNull() }
?: defaultFrom
val to = request.queryParameters["to"]
?.let { runCatching { Instant.parse(it) }.getOrNull() }
?: now
return if (from <= to) from to to else to to from
}
@@ -0,0 +1,64 @@
package ink.snowflake.server.model.database
import org.jetbrains.exposed.v1.core.Table
import org.jetbrains.exposed.v1.core.dao.id.UUIDTable
import org.jetbrains.exposed.v1.datetime.timestamp
object OpenApiEndpointsTable : UUIDTable("open_api_endpoints") {
val name = varchar("name", 160)
val serviceCode = varchar("service_code", 64)
val category = varchar("category", 80)
val httpMethod = varchar("http_method", 16).default("*")
val pathTemplate = varchar("path_template", 500)
val kongServiceId = varchar("kong_service_id", 80).default("")
val kongRouteId = varchar("kong_route_id", 80).default("")
val description = text("description").default("")
val owner = varchar("owner", 120).default("")
val monitorEnabled = bool("monitor_enabled").default(true)
val status = varchar("status", 32).default("enabled")
val createdAt = timestamp("created_at")
val updatedAt = timestamp("updated_at")
init {
uniqueIndex(serviceCode, httpMethod, pathTemplate)
}
}
object OpenApiConsumersTable : UUIDTable("open_api_consumers") {
val kongConsumerId = varchar("kong_consumer_id", 80).uniqueIndex()
val username = varchar("username", 160).default("")
val customId = varchar("custom_id", 160).default("")
val displayName = varchar("display_name", 160).default("")
val description = text("description").default("")
val firstSeenAt = timestamp("first_seen_at")
val lastSeenAt = timestamp("last_seen_at")
val status = varchar("status", 32).default("active")
}
object OpenApiUsageHourlyTable : UUIDTable("open_api_usage_hourly") {
val bucketTime = timestamp("bucket_time")
val endpointId = reference("endpoint_id", OpenApiEndpointsTable)
val callerType = varchar("caller_type", 24)
val callerKey = varchar("caller_key", 160)
val consumerId = varchar("consumer_id", 80).default("")
val consumerUsername = varchar("consumer_username", 160).default("")
val clientIp = varchar("client_ip", 80).default("")
val callCount = long("call_count").default(0)
val successCount = long("success_count").default(0)
val failureCount = long("failure_count").default(0)
val unauthorizedCount = long("unauthorized_count").default(0)
val firstCalledAt = timestamp("first_called_at")
val lastCalledAt = timestamp("last_called_at")
val updatedAt = timestamp("updated_at")
init {
uniqueIndex(bucketTime, endpointId, callerType, callerKey)
}
}
object OpenApiIngestDedupTable : Table("open_api_ingest_dedup") {
val requestId = varchar("request_id", 100)
val receivedAt = timestamp("received_at")
override val primaryKey = PrimaryKey(requestId)
}
@@ -0,0 +1,25 @@
package ink.snowflake.server.model.request
import kotlinx.serialization.Serializable
@Serializable
data class SaveOpenApiEndpointRequest(
val name: String,
val serviceCode: String,
val category: String,
val httpMethod: String = "*",
val pathTemplate: String,
val kongServiceId: String = "",
val kongRouteId: String = "",
val description: String = "",
val owner: String = "",
val monitorEnabled: Boolean = true,
val status: String = "enabled",
)
@Serializable
data class UpdateOpenApiConsumerRequest(
val displayName: String = "",
val description: String = "",
val status: String = "active",
)
@@ -0,0 +1,89 @@
package ink.snowflake.server.model.response
import kotlinx.serialization.Serializable
@Serializable
data class OpenApiEndpointResponse(
val id: String,
val name: String,
val serviceCode: String,
val category: String,
val httpMethod: String,
val pathTemplate: String,
val kongServiceId: String,
val kongRouteId: String,
val description: String,
val owner: String,
val monitorEnabled: Boolean,
val status: String,
val recentCalls: Long,
val recentConsumers: Int,
val lastCalledAt: String,
val updatedAt: String,
)
@Serializable
data class OpenApiTrendPointResponse(
val bucket: String,
val calls: Long,
val authenticatedCalls: Long,
val anonymousCalls: Long,
val failures: Long,
)
@Serializable
data class OpenApiRankItemResponse(
val key: String,
val name: String,
val value: Long,
val secondary: String = "",
)
@Serializable
data class OpenApiOverviewResponse(
val totalCalls: Long,
val authenticatedCalls: Long,
val anonymousCalls: Long,
val unauthorizedCalls: Long,
val activeConsumers: Int,
val activeIps: Int,
val successRate: Double,
val trend: List<OpenApiTrendPointResponse>,
val topEndpoints: List<OpenApiRankItemResponse>,
val topConsumers: List<OpenApiRankItemResponse>,
val categoryDistribution: List<OpenApiRankItemResponse>,
)
@Serializable
data class OpenApiConsumerResponse(
val id: String,
val kongConsumerId: String,
val username: String,
val customId: String,
val displayName: String,
val description: String,
val firstSeenAt: String,
val lastSeenAt: String,
val status: String,
val callCount: Long,
val endpointCount: Int,
val recentIp: String,
)
@Serializable
data class OpenApiUsageResponse(
val bucket: String,
val endpointId: String,
val endpointName: String,
val serviceCode: String,
val category: String,
val callerType: String,
val callerName: String,
val consumerId: String,
val clientIp: String,
val callCount: Long,
val successCount: Long,
val failureCount: Long,
val unauthorizedCount: Long,
val lastCalledAt: String,
)
@@ -33,4 +33,11 @@ class AppConfig(config: ApplicationConfig) {
?.trimEnd('/')
?.takeIf { it.isNotBlank() }
?: "http://127.0.0.1:8081"
val openApiStatisticsEnabled: Boolean =
config.propertyOrNull("ktor.open-api-statistics.enabled")?.getString()?.toBooleanStrictOrNull() ?: true
val openApiStatisticsIngestToken: String =
System.getenv("OPEN_API_LOG_TOKEN")
?.trim()
?.takeIf { it.isNotBlank() }
?: config.propertyOrNull("ktor.open-api-statistics.ingest-token")?.getString()?.trim().orEmpty()
}
@@ -0,0 +1,553 @@
package ink.snowflake.server.utils.dao
import com.google.gson.JsonElement
import com.google.gson.JsonObject
import com.google.gson.JsonParser
import ink.snowflake.server.model.database.OpenApiConsumersTable
import ink.snowflake.server.model.database.OpenApiEndpointsTable
import ink.snowflake.server.model.database.OpenApiIngestDedupTable
import ink.snowflake.server.model.database.OpenApiUsageHourlyTable
import ink.snowflake.server.model.request.SaveOpenApiEndpointRequest
import ink.snowflake.server.model.request.UpdateOpenApiConsumerRequest
import ink.snowflake.server.model.response.OpenApiConsumerResponse
import ink.snowflake.server.model.response.OpenApiEndpointResponse
import ink.snowflake.server.model.response.OpenApiOverviewResponse
import ink.snowflake.server.model.response.OpenApiRankItemResponse
import ink.snowflake.server.model.response.OpenApiTrendPointResponse
import ink.snowflake.server.model.response.OpenApiUsageResponse
import kotlinx.datetime.Clock
import kotlinx.datetime.Instant
import org.jetbrains.exposed.v1.core.ResultRow
import org.jetbrains.exposed.v1.core.SortOrder
import org.jetbrains.exposed.v1.core.SqlExpressionBuilder.eq
import org.jetbrains.exposed.v1.core.SqlExpressionBuilder.greaterEq
import org.jetbrains.exposed.v1.core.SqlExpressionBuilder.less
import org.jetbrains.exposed.v1.core.SqlExpressionBuilder.lessEq
import org.jetbrains.exposed.v1.core.and
import org.jetbrains.exposed.v1.datetime.timestampLiteral
import org.jetbrains.exposed.v1.jdbc.SchemaUtils
import org.jetbrains.exposed.v1.jdbc.deleteWhere
import org.jetbrains.exposed.v1.jdbc.insert
import org.jetbrains.exposed.v1.jdbc.insertAndGetId
import org.jetbrains.exposed.v1.jdbc.selectAll
import org.jetbrains.exposed.v1.jdbc.transactions.transaction
import org.jetbrains.exposed.v1.jdbc.update
import java.util.UUID
import kotlin.math.round
object OpenApiStatisticsDao {
private const val HOUR_MILLIS = 3_600_000L
private const val DEDUP_RETENTION_MILLIS = 7L * 24 * HOUR_MILLIS
private data class SeedEndpoint(
val name: String,
val serviceCode: String,
val category: String,
val method: String,
val path: String,
val description: String,
)
private data class IngestEvent(
val requestId: String,
val method: String,
val uri: String,
val startedAt: Instant,
val serviceId: String,
val serviceName: String,
val routeId: String,
val consumerId: String,
val consumerUsername: String,
val consumerCustomId: String,
val clientIp: String,
val statusCode: Int,
)
private val seedEndpoints = listOf(
SeedEndpoint("蓝字发票开具", "invoice", "票通开票", "*", "/api/open/v1/blue-invoices", "创建蓝字发票"),
SeedEndpoint("蓝字发票详情", "invoice", "票通开票", "*", "/api/open/v1/blue-invoices/{invoiceCode}", "查询蓝字发票详情"),
SeedEndpoint("蓝字发票样例", "invoice", "票通开票", "*", "/api/open/v1/blue-invoices/sample/{invoiceCode}", "查询蓝字发票样例"),
SeedEndpoint("蓝字发票批次", "invoice", "票通开票", "*", "/api/open/v1/blue-invoices/batches", "创建或查询发票批次"),
SeedEndpoint("蓝字发票批次详情", "invoice", "票通开票", "*", "/api/open/v1/blue-invoices/batches/{batchCode}", "查询发票批次详情"),
SeedEndpoint("生产开票任务", "invoice", "票通开票", "*", "/api/open/v1/blue-invoice-tasks/production", "提交生产开票任务"),
SeedEndpoint("蚕茧票据指标识别", "ai-fastapi", "AI识别", "POST", "/api/public/recognize-cocoon-metrics", "识别蚕茧票据指标"),
SeedEndpoint("许可证识别", "ai-fastapi", "AI识别", "POST", "/api/public/recognize-license", "识别许可证图片"),
SeedEndpoint("蚕茧图片识别", "ai-fastapi", "AI识别", "POST", "/api/public/recognize-silkworm-cocoon", "识别蚕茧图片"),
SeedEndpoint("云哨记录分析", "ai-fastapi", "牧安云哨", "POST", "/api/public/sentinel-record-analytics", "提交牧安云哨分析任务"),
SeedEndpoint("开放批次创建", "ktor-core", "溯源开放", "POST", "/traceability/public/integration/batches", "通过开放接口创建溯源批次"),
SeedEndpoint("批次公开地址", "ktor-core", "溯源开放", "GET", "/traceability/public/integration/batches/{id}/public-url", "查询批次公开地址"),
SeedEndpoint("溯源详情查询", "ktor-core", "溯源开放", "GET", "/traceability/public/by-code/{code}", "通过批次编码查询公开信息"),
SeedEndpoint("溯源预览查询", "ktor-core", "溯源开放", "GET", "/traceability/public/preview/by-code/{code}", "查询未发布批次预览"),
SeedEndpoint("预演页查询", "ktor-core", "溯源开放", "GET", "/traceability/public/preview-page/by-code/{code}", "查询溯源预演页"),
SeedEndpoint("溯源意见反馈", "ktor-core", "溯源开放", "POST", "/traceability/public/feedback", "提交消费者反馈"),
SeedEndpoint("溯源公开页面", "ktor-core", "溯源开放", "GET", "/traceability/public/page/{code}", "访问公开溯源页面"),
SeedEndpoint("远程设备连接", "ktor-core", "运维开放", "GET", "/silk-remote/connectLocalDevice", "连接远程 ADB 设备"),
SeedEndpoint("远程设备断开", "ktor-core", "运维开放", "GET", "/silk-remote/disConnectAll", "断开远程 ADB 连接"),
)
fun initSchemaAndSeed() {
transaction {
SchemaUtils.createMissingTablesAndColumns(
OpenApiEndpointsTable,
OpenApiConsumersTable,
OpenApiUsageHourlyTable,
OpenApiIngestDedupTable,
)
val now = timestampLiteral(Clock.System.now())
seedEndpoints.forEach { seed ->
val exists = OpenApiEndpointsTable.selectAll().where {
(OpenApiEndpointsTable.serviceCode eq seed.serviceCode) and
(OpenApiEndpointsTable.httpMethod eq seed.method) and
(OpenApiEndpointsTable.pathTemplate eq seed.path)
}.any()
if (!exists) {
OpenApiEndpointsTable.insert {
it[name] = seed.name
it[serviceCode] = seed.serviceCode
it[category] = seed.category
it[httpMethod] = seed.method
it[pathTemplate] = seed.path
it[description] = seed.description
it[createdAt] = now
it[updatedAt] = now
}
}
}
}
}
@Synchronized
fun ingest(rawBody: String): Int {
val root = runCatching { JsonParser.parseString(rawBody) }.getOrNull() ?: return 0
val elements = if (root.isJsonArray) root.asJsonArray.toList() else listOf(root)
val events = elements.mapNotNull(::parseEvent)
if (events.isEmpty()) return 0
return transaction {
val now = Clock.System.now()
OpenApiIngestDedupTable.deleteWhere {
receivedAt less timestampLiteral(Instant.fromEpochMilliseconds(now.toEpochMilliseconds() - DEDUP_RETENTION_MILLIS))
}
val endpoints = OpenApiEndpointsTable.selectAll()
.where { OpenApiEndpointsTable.monitorEnabled eq true }
.toList()
var accepted = 0
events.forEach { event ->
if (OpenApiIngestDedupTable.selectAll()
.where { OpenApiIngestDedupTable.requestId eq event.requestId }
.any()
) {
return@forEach
}
val endpoint = matchEndpoint(event, endpoints) ?: return@forEach
OpenApiIngestDedupTable.insert {
it[requestId] = event.requestId
it[receivedAt] = timestampLiteral(now)
}
if (event.consumerId.isNotBlank()) {
saveConsumer(event)
}
val bucket = Instant.fromEpochMilliseconds(
event.startedAt.toEpochMilliseconds() / HOUR_MILLIS * HOUR_MILLIS
)
val callerType = if (event.consumerId.isBlank()) "anonymous" else "consumer"
val callerKey = event.consumerId.ifBlank { event.clientIp.ifBlank { "unknown" } }
val endpointId = endpoint[OpenApiEndpointsTable.id].value
val existing = OpenApiUsageHourlyTable.selectAll().where {
(OpenApiUsageHourlyTable.bucketTime eq bucket) and
(OpenApiUsageHourlyTable.endpointId eq endpointId) and
(OpenApiUsageHourlyTable.callerType eq callerType) and
(OpenApiUsageHourlyTable.callerKey eq callerKey)
}.singleOrNull()
val success = event.statusCode in 200..399
val unauthorized = event.statusCode == 401 || event.statusCode == 403
if (existing == null) {
OpenApiUsageHourlyTable.insert {
it[bucketTime] = timestampLiteral(bucket)
it[OpenApiUsageHourlyTable.endpointId] = endpointId
it[OpenApiUsageHourlyTable.callerType] = callerType
it[OpenApiUsageHourlyTable.callerKey] = callerKey
it[consumerId] = event.consumerId
it[consumerUsername] = event.consumerUsername
it[clientIp] = event.clientIp
it[callCount] = 1
it[successCount] = if (success) 1 else 0
it[failureCount] = if (success) 0 else 1
it[unauthorizedCount] = if (unauthorized) 1 else 0
it[firstCalledAt] = timestampLiteral(event.startedAt)
it[lastCalledAt] = timestampLiteral(event.startedAt)
it[updatedAt] = timestampLiteral(now)
}
} else {
OpenApiUsageHourlyTable.update({ OpenApiUsageHourlyTable.id eq existing[OpenApiUsageHourlyTable.id].value }) {
it[callCount] = existing[OpenApiUsageHourlyTable.callCount] + 1
it[successCount] = existing[OpenApiUsageHourlyTable.successCount] + if (success) 1 else 0
it[failureCount] = existing[OpenApiUsageHourlyTable.failureCount] + if (success) 0 else 1
it[unauthorizedCount] = existing[OpenApiUsageHourlyTable.unauthorizedCount] + if (unauthorized) 1 else 0
it[lastCalledAt] = timestampLiteral(maxOf(existing[OpenApiUsageHourlyTable.lastCalledAt], event.startedAt))
it[updatedAt] = timestampLiteral(now)
}
}
accepted++
}
accepted
}
}
fun listEndpoints(): List<OpenApiEndpointResponse> = transaction {
val recentFrom = Instant.fromEpochMilliseconds(Clock.System.now().toEpochMilliseconds() - 24 * HOUR_MILLIS)
val recentRows = OpenApiUsageHourlyTable.selectAll()
.where { OpenApiUsageHourlyTable.bucketTime greaterEq timestampLiteral(recentFrom) }
.toList()
.groupBy { it[OpenApiUsageHourlyTable.endpointId].value }
OpenApiEndpointsTable.selectAll()
.orderBy(OpenApiEndpointsTable.serviceCode, SortOrder.ASC)
.map { row ->
val usage = recentRows[row[OpenApiEndpointsTable.id].value].orEmpty()
endpointResponse(row, usage)
}
}
fun saveEndpoint(id: UUID?, request: SaveOpenApiEndpointRequest): OpenApiEndpointResponse = transaction {
val now = timestampLiteral(Clock.System.now())
val endpointId = id ?: OpenApiEndpointsTable.insertAndGetId {
it[name] = request.name.trim()
it[serviceCode] = request.serviceCode.trim()
it[category] = request.category.trim()
it[httpMethod] = request.httpMethod.trim().uppercase().ifBlank { "*" }
it[pathTemplate] = normalizePath(request.pathTemplate)
it[kongServiceId] = request.kongServiceId.trim()
it[kongRouteId] = request.kongRouteId.trim()
it[description] = request.description.trim()
it[owner] = request.owner.trim()
it[monitorEnabled] = request.monitorEnabled
it[status] = request.status.trim().ifBlank { "enabled" }
it[createdAt] = now
it[updatedAt] = now
}.value
if (id != null) {
OpenApiEndpointsTable.update({ OpenApiEndpointsTable.id eq id }) {
it[name] = request.name.trim()
it[serviceCode] = request.serviceCode.trim()
it[category] = request.category.trim()
it[httpMethod] = request.httpMethod.trim().uppercase().ifBlank { "*" }
it[pathTemplate] = normalizePath(request.pathTemplate)
it[kongServiceId] = request.kongServiceId.trim()
it[kongRouteId] = request.kongRouteId.trim()
it[description] = request.description.trim()
it[owner] = request.owner.trim()
it[monitorEnabled] = request.monitorEnabled
it[status] = request.status.trim().ifBlank { "enabled" }
it[updatedAt] = now
}
}
val row = OpenApiEndpointsTable.selectAll()
.where { OpenApiEndpointsTable.id eq endpointId }
.single()
endpointResponse(row, emptyList())
}
fun overview(
from: Instant,
to: Instant,
serviceCode: String?,
category: String?,
): OpenApiOverviewResponse = transaction {
val endpointRows = OpenApiEndpointsTable.selectAll().toList()
val endpointMap = endpointRows.associateBy { it[OpenApiEndpointsTable.id].value }
val allowedEndpointIds = endpointRows.asSequence()
.filter { serviceCode.isNullOrBlank() || it[OpenApiEndpointsTable.serviceCode] == serviceCode }
.filter { category.isNullOrBlank() || it[OpenApiEndpointsTable.category] == category }
.map { it[OpenApiEndpointsTable.id].value }
.toSet()
val rows = usageRows(from, to).filter {
it[OpenApiUsageHourlyTable.endpointId].value in allowedEndpointIds
}
val total = rows.sumOf { it[OpenApiUsageHourlyTable.callCount] }
val authenticated = rows.filter { it[OpenApiUsageHourlyTable.callerType] == "consumer" }
.sumOf { it[OpenApiUsageHourlyTable.callCount] }
val anonymous = total - authenticated
val unauthorized = rows.sumOf { it[OpenApiUsageHourlyTable.unauthorizedCount] }
val successes = rows.sumOf { it[OpenApiUsageHourlyTable.successCount] }
val useHourly = to.toEpochMilliseconds() - from.toEpochMilliseconds() <= 48 * HOUR_MILLIS
val trend = rows.groupBy {
val text = it[OpenApiUsageHourlyTable.bucketTime].toString()
if (useHourly) text.take(13) + ":00:00Z" else text.take(10)
}.toSortedMap().map { (bucket, items) ->
OpenApiTrendPointResponse(
bucket = bucket,
calls = items.sumOf { it[OpenApiUsageHourlyTable.callCount] },
authenticatedCalls = items.filter { it[OpenApiUsageHourlyTable.callerType] == "consumer" }
.sumOf { it[OpenApiUsageHourlyTable.callCount] },
anonymousCalls = items.filter { it[OpenApiUsageHourlyTable.callerType] == "anonymous" }
.sumOf { it[OpenApiUsageHourlyTable.callCount] },
failures = items.sumOf { it[OpenApiUsageHourlyTable.failureCount] },
)
}
val topEndpoints = rows.groupBy { it[OpenApiUsageHourlyTable.endpointId].value }
.map { (id, items) ->
val endpoint = endpointMap[id]
OpenApiRankItemResponse(
key = id.toString(),
name = endpoint?.get(OpenApiEndpointsTable.name) ?: "未知接口",
value = items.sumOf { it[OpenApiUsageHourlyTable.callCount] },
secondary = endpoint?.get(OpenApiEndpointsTable.serviceCode).orEmpty(),
)
}.sortedByDescending { it.value }.take(8)
val topConsumers = rows.groupBy {
if (it[OpenApiUsageHourlyTable.callerType] == "consumer") {
it[OpenApiUsageHourlyTable.consumerId]
} else {
"ip:${it[OpenApiUsageHourlyTable.clientIp]}"
}
}.map { (key, items) ->
val first = items.first()
val name = if (first[OpenApiUsageHourlyTable.callerType] == "consumer") {
first[OpenApiUsageHourlyTable.consumerUsername].ifBlank { key }
} else {
"匿名 · ${first[OpenApiUsageHourlyTable.clientIp].ifBlank { "未知 IP" }}"
}
OpenApiRankItemResponse(key, name, items.sumOf { it[OpenApiUsageHourlyTable.callCount] })
}.sortedByDescending { it.value }.take(8)
val categoryDistribution = rows.groupBy {
endpointMap[it[OpenApiUsageHourlyTable.endpointId].value]?.get(OpenApiEndpointsTable.category) ?: "其他"
}.map { (name, items) ->
OpenApiRankItemResponse(name, name, items.sumOf { it[OpenApiUsageHourlyTable.callCount] })
}.sortedByDescending { it.value }
OpenApiOverviewResponse(
totalCalls = total,
authenticatedCalls = authenticated,
anonymousCalls = anonymous,
unauthorizedCalls = unauthorized,
activeConsumers = rows.map { it[OpenApiUsageHourlyTable.consumerId] }.filter { it.isNotBlank() }.distinct().size,
activeIps = rows.map { it[OpenApiUsageHourlyTable.clientIp] }.filter { it.isNotBlank() }.distinct().size,
successRate = if (total == 0L) 100.0 else round(successes * 10_000.0 / total) / 100.0,
trend = trend,
topEndpoints = topEndpoints,
topConsumers = topConsumers,
categoryDistribution = categoryDistribution,
)
}
fun listConsumers(): List<OpenApiConsumerResponse> = transaction {
val usageByConsumer = OpenApiUsageHourlyTable.selectAll()
.where { OpenApiUsageHourlyTable.consumerId neq "" }
.toList()
.groupBy { it[OpenApiUsageHourlyTable.consumerId] }
OpenApiConsumersTable.selectAll()
.orderBy(OpenApiConsumersTable.lastSeenAt, SortOrder.DESC)
.map { row ->
val usage = usageByConsumer[row[OpenApiConsumersTable.kongConsumerId]].orEmpty()
OpenApiConsumerResponse(
id = row[OpenApiConsumersTable.id].value.toString(),
kongConsumerId = row[OpenApiConsumersTable.kongConsumerId],
username = row[OpenApiConsumersTable.username],
customId = row[OpenApiConsumersTable.customId],
displayName = row[OpenApiConsumersTable.displayName],
description = row[OpenApiConsumersTable.description],
firstSeenAt = row[OpenApiConsumersTable.firstSeenAt].toString(),
lastSeenAt = row[OpenApiConsumersTable.lastSeenAt].toString(),
status = row[OpenApiConsumersTable.status],
callCount = usage.sumOf { it[OpenApiUsageHourlyTable.callCount] },
endpointCount = usage.map { it[OpenApiUsageHourlyTable.endpointId].value }.distinct().size,
recentIp = usage.maxByOrNull { it[OpenApiUsageHourlyTable.lastCalledAt] }
?.get(OpenApiUsageHourlyTable.clientIp).orEmpty(),
)
}
}
fun updateConsumer(id: UUID, request: UpdateOpenApiConsumerRequest): OpenApiConsumerResponse? {
transaction {
OpenApiConsumersTable.update({ OpenApiConsumersTable.id eq id }) {
it[displayName] = request.displayName.trim()
it[description] = request.description.trim()
it[status] = request.status.trim().ifBlank { "active" }
}
}
return listConsumers().firstOrNull { it.id == id.toString() }
}
fun listUsage(
from: Instant,
to: Instant,
serviceCode: String?,
category: String?,
endpointId: UUID?,
callerType: String?,
): List<OpenApiUsageResponse> = transaction {
val endpointRows = OpenApiEndpointsTable.selectAll().toList()
val endpointMap = endpointRows.associateBy { it[OpenApiEndpointsTable.id].value }
usageRows(from, to).asSequence()
.filter { endpointId == null || it[OpenApiUsageHourlyTable.endpointId].value == endpointId }
.filter { callerType.isNullOrBlank() || it[OpenApiUsageHourlyTable.callerType] == callerType }
.filter {
val endpoint = endpointMap[it[OpenApiUsageHourlyTable.endpointId].value]
(serviceCode.isNullOrBlank() || endpoint?.get(OpenApiEndpointsTable.serviceCode) == serviceCode) &&
(category.isNullOrBlank() || endpoint?.get(OpenApiEndpointsTable.category) == category)
}
.sortedByDescending { it[OpenApiUsageHourlyTable.bucketTime] }
.take(1_000)
.map { row ->
val endpoint = endpointMap[row[OpenApiUsageHourlyTable.endpointId].value]
val callerName = if (row[OpenApiUsageHourlyTable.callerType] == "consumer") {
row[OpenApiUsageHourlyTable.consumerUsername]
.ifBlank { row[OpenApiUsageHourlyTable.consumerId] }
} else {
"匿名调用方"
}
OpenApiUsageResponse(
bucket = row[OpenApiUsageHourlyTable.bucketTime].toString(),
endpointId = row[OpenApiUsageHourlyTable.endpointId].value.toString(),
endpointName = endpoint?.get(OpenApiEndpointsTable.name) ?: "未知接口",
serviceCode = endpoint?.get(OpenApiEndpointsTable.serviceCode).orEmpty(),
category = endpoint?.get(OpenApiEndpointsTable.category).orEmpty(),
callerType = row[OpenApiUsageHourlyTable.callerType],
callerName = callerName,
consumerId = row[OpenApiUsageHourlyTable.consumerId],
clientIp = row[OpenApiUsageHourlyTable.clientIp],
callCount = row[OpenApiUsageHourlyTable.callCount],
successCount = row[OpenApiUsageHourlyTable.successCount],
failureCount = row[OpenApiUsageHourlyTable.failureCount],
unauthorizedCount = row[OpenApiUsageHourlyTable.unauthorizedCount],
lastCalledAt = row[OpenApiUsageHourlyTable.lastCalledAt].toString(),
)
}.toList()
}
private fun usageRows(from: Instant, to: Instant): List<ResultRow> {
return OpenApiUsageHourlyTable.selectAll().where {
(OpenApiUsageHourlyTable.bucketTime greaterEq timestampLiteral(from)) and
(OpenApiUsageHourlyTable.bucketTime lessEq timestampLiteral(to))
}.toList()
}
private fun endpointResponse(row: ResultRow, usage: List<ResultRow>): OpenApiEndpointResponse {
return OpenApiEndpointResponse(
id = row[OpenApiEndpointsTable.id].value.toString(),
name = row[OpenApiEndpointsTable.name],
serviceCode = row[OpenApiEndpointsTable.serviceCode],
category = row[OpenApiEndpointsTable.category],
httpMethod = row[OpenApiEndpointsTable.httpMethod],
pathTemplate = row[OpenApiEndpointsTable.pathTemplate],
kongServiceId = row[OpenApiEndpointsTable.kongServiceId],
kongRouteId = row[OpenApiEndpointsTable.kongRouteId],
description = row[OpenApiEndpointsTable.description],
owner = row[OpenApiEndpointsTable.owner],
monitorEnabled = row[OpenApiEndpointsTable.monitorEnabled],
status = row[OpenApiEndpointsTable.status],
recentCalls = usage.sumOf { it[OpenApiUsageHourlyTable.callCount] },
recentConsumers = usage.map { it[OpenApiUsageHourlyTable.consumerId] }.filter { it.isNotBlank() }.distinct().size,
lastCalledAt = usage.maxByOrNull { it[OpenApiUsageHourlyTable.lastCalledAt] }
?.get(OpenApiUsageHourlyTable.lastCalledAt)?.toString().orEmpty(),
updatedAt = row[OpenApiEndpointsTable.updatedAt].toString(),
)
}
private fun saveConsumer(event: IngestEvent) {
val row = OpenApiConsumersTable.selectAll()
.where { OpenApiConsumersTable.kongConsumerId eq event.consumerId }
.singleOrNull()
if (row == null) {
OpenApiConsumersTable.insert {
it[kongConsumerId] = event.consumerId
it[username] = event.consumerUsername
it[customId] = event.consumerCustomId
it[displayName] = event.consumerUsername
it[firstSeenAt] = timestampLiteral(event.startedAt)
it[lastSeenAt] = timestampLiteral(event.startedAt)
}
} else {
OpenApiConsumersTable.update({ OpenApiConsumersTable.id eq row[OpenApiConsumersTable.id].value }) {
it[username] = event.consumerUsername
it[customId] = event.consumerCustomId
it[lastSeenAt] = timestampLiteral(maxOf(row[OpenApiConsumersTable.lastSeenAt], event.startedAt))
}
}
}
private fun matchEndpoint(event: IngestEvent, endpoints: List<ResultRow>): ResultRow? {
val path = normalizePath(event.uri.substringBefore('?'))
return endpoints.asSequence()
.filter {
val method = it[OpenApiEndpointsTable.httpMethod]
method == "*" || method.equals(event.method, ignoreCase = true)
}
.filter { pathMatches(it[OpenApiEndpointsTable.pathTemplate], path) }
.maxByOrNull {
var score = it[OpenApiEndpointsTable.pathTemplate].split('/').count { segment ->
segment.isNotBlank() && !(segment.startsWith("{") && segment.endsWith("}"))
} * 10
if (event.routeId.isNotBlank() && it[OpenApiEndpointsTable.kongRouteId] == event.routeId) score += 1_000
if (event.serviceId.isNotBlank() && it[OpenApiEndpointsTable.kongServiceId] == event.serviceId) score += 500
if (event.serviceName.contains(it[OpenApiEndpointsTable.serviceCode], ignoreCase = true)) score += 100
score
}
}
private fun pathMatches(template: String, actual: String): Boolean {
val templateParts = normalizePath(template).trim('/').split('/').filter { it.isNotBlank() }
val actualParts = normalizePath(actual).trim('/').split('/').filter { it.isNotBlank() }
if (templateParts.size != actualParts.size) return false
return templateParts.zip(actualParts).all { (expected, value) ->
(expected.startsWith("{") && expected.endsWith("}")) || expected == value
}
}
private fun normalizePath(path: String): String {
val normalized = "/" + path.trim().trim('/').replace(Regex("/+"), "/")
return if (normalized == "/") normalized else normalized.trimEnd('/')
}
private fun parseEvent(element: JsonElement): IngestEvent? {
if (!element.isJsonObject) return null
val root = element.asJsonObject
val request = root.objectOrNull("request") ?: return null
val response = root.objectOrNull("response")
val service = root.objectOrNull("service")
val route = root.objectOrNull("route")
val consumer = root.objectOrNull("consumer")
val requestId = request.string("id")
val uri = request.string("uri")
if (requestId.isBlank() || uri.isBlank()) return null
val startedAtMillis = root.long("started_at").takeIf { it > 0 } ?: Clock.System.now().toEpochMilliseconds()
return IngestEvent(
requestId = requestId,
method = request.string("method").uppercase().ifBlank { "GET" },
uri = uri,
startedAt = Instant.fromEpochMilliseconds(startedAtMillis),
serviceId = service?.string("id").orEmpty(),
serviceName = service?.string("name").orEmpty(),
routeId = route?.string("id").orEmpty(),
consumerId = consumer?.string("id").orEmpty(),
consumerUsername = consumer?.string("username").orEmpty(),
consumerCustomId = consumer?.string("custom_id").orEmpty(),
clientIp = root.string("client_ip"),
statusCode = response?.int("status") ?: 0,
)
}
private fun JsonObject.objectOrNull(name: String): JsonObject? {
val value = get(name) ?: return null
return if (value.isJsonObject) value.asJsonObject else null
}
private fun JsonObject.string(name: String): String {
val value = get(name) ?: return ""
return runCatching { if (value.isJsonNull) "" else value.asString }.getOrDefault("")
}
private fun JsonObject.long(name: String): Long {
val value = get(name) ?: return 0
return runCatching { value.asLong }.getOrDefault(0)
}
private fun JsonObject.int(name: String): Int {
val value = get(name) ?: return 0
return runCatching { value.asInt }.getOrDefault(0)
}
}