diff --git a/bbit_ai/app/app.py b/bbit_ai/app/app.py index 31f9723..f3bfd0b 100644 --- a/bbit_ai/app/app.py +++ b/bbit_ai/app/app.py @@ -22,6 +22,7 @@ from routers.Service import serviceRouter from routers.System import systemRouter from routers.Vision import visionRouter from routers.WS import iot_ws_router +from routers.dify_export_router import difyRouter from service.RabbitMQ import ( mq_client, ) @@ -62,6 +63,7 @@ async def ai_lab(): serviceRouter, botRouter, rqRouter, + difyRouter, ] for r in routers: app.include_router(r, prefix="/llm", tags=["llm"]) @@ -72,6 +74,7 @@ async def ai_lab(): app.include_router(sentinel_router, prefix="/iot/sentinel", tags=["iot_sentinel"]) app.include_router(iot_ws_router, prefix="/iot/ws", tags=["iot_ws"]) app.include_router(publicRouter, prefix="/api/public", tags=["api"]) + app.include_router(difyRouter, prefix="/api/dify", tags=["dify"]) # ----------- 全局异常捕获 --------- @app.exception_handler(Exception) diff --git a/bbit_ai/app/routers/dify_export_router.py b/bbit_ai/app/routers/dify_export_router.py new file mode 100644 index 0000000..d0b051e --- /dev/null +++ b/bbit_ai/app/routers/dify_export_router.py @@ -0,0 +1,366 @@ +from __future__ import annotations + +import json +import re +import uuid +from datetime import date, datetime +from decimal import Decimal +from io import BytesIO +from typing import Any, Dict, List, Union + +import unicodedata +from fastapi import APIRouter, HTTPException +from openpyxl import Workbook +from openpyxl.cell import WriteOnlyCell +from openpyxl.styles import Alignment, Border, Font, PatternFill, Side +from openpyxl.utils import get_column_letter +from pydantic import BaseModel, Field + +from config import minIO +from config.minIO import minio_client +from db.sqlserver import executeSQL + +EXCEL_MAX_ROWS = 1_048_576 +EXCEL_MAX_COLUMNS = 16_384 +EXCEL_DATA_ROWS_PER_SHEET = EXCEL_MAX_ROWS - 1 +WIDTH_SAMPLE_ROWS = 200 +MIN_COLUMN_WIDTH = 11 +MAX_COLUMN_WIDTH = 50 +STYLED_DATA_MAX_ROWS = 5000 +FONT_NAME = "Microsoft YaHei" +EXPORT_BUCKET_NAME = "dify-export" + +difyRouter = APIRouter() + + +class DifySQLExportRequest(BaseModel): + sql: str = Field(..., min_length=1) + filename: str = Field(default="dify_export", min_length=1, max_length=120) + headers: Union[Dict[str, Any], str, None] = None + headers_json: Union[Dict[str, Any], str, None] = None + + +class DifySQLExportResponse(BaseModel): + url: str + filename: str + bucket_name: str + object_name: str + row_count: int + sheet_count: int + + +def normalize_sql(raw_sql: str) -> str: + sql = raw_sql.strip() + while sql.endswith(";"): + sql = sql[:-1].strip() + if not sql: + raise HTTPException(status_code=400, detail="sql cannot be empty") + if ";" in sql: + raise HTTPException(status_code=400, detail="Only one SQL statement is allowed") + return sql + + +def validate_read_only_sql(sql: str) -> None: + if "--" in sql or "/*" in sql or "*/" in sql: + raise HTTPException(status_code=400, detail="SQL comments are not allowed") + + normalized = re.sub(r"\s+", " ", sql).strip().lower() + if not (normalized.startswith("select ") or normalized.startswith("with ")): + raise HTTPException(status_code=400, detail="Only SELECT queries are allowed") + + forbidden_patterns = [ + r"\binsert\b", + r"\bupdate\b", + r"\bdelete\b", + r"\bdrop\b", + r"\balter\b", + r"\bcreate\b", + r"\btruncate\b", + r"\bmerge\b", + r"\bexec\b", + r"\bexecute\b", + r"\binto\b", + r"\bxp_cmdshell\b", + r"\bsp_executesql\b", + ] + for pattern in forbidden_patterns: + if re.search(pattern, normalized): + raise HTTPException(status_code=400, detail="Only read-only SQL is allowed") + + +def build_xlsx( + rows: List[Dict[str, Any]], headers: Dict[str, str] +) -> tuple[bytes, int]: + workbook = Workbook(write_only=True) + sheet_count = 1 + + if not rows: + sheet = workbook.create_sheet(title="Sheet1") + if headers: + excel_headers = list(headers.values()) + apply_sheet_layout(sheet, excel_headers, 0) + sheet.append(build_header_cells(sheet, excel_headers)) + buffer = BytesIO() + workbook.save(buffer) + return buffer.getvalue(), sheet_count + + columns = list(rows[0].keys()) + if len(columns) > EXCEL_MAX_COLUMNS: + raise HTTPException( + status_code=413, + detail=f"Excel supports at most {EXCEL_MAX_COLUMNS} columns per sheet", + ) + + excel_headers = [headers.get(column, column) for column in columns] + column_widths = calculate_column_widths(columns, excel_headers, rows) + style_data = len(rows) <= STYLED_DATA_MAX_ROWS + sheet = create_export_sheet( + workbook, + "Sheet1", + excel_headers, + column_widths, + style_data, + ) + rows_in_current_sheet = 0 + + for row in rows: + if rows_in_current_sheet >= EXCEL_DATA_ROWS_PER_SHEET: + apply_auto_filter(sheet, len(columns), rows_in_current_sheet) + sheet_count += 1 + sheet = create_export_sheet( + workbook, + f"Sheet{sheet_count}", + excel_headers, + column_widths, + style_data, + ) + rows_in_current_sheet = 0 + + values = [serialize_cell(row.get(column)) for column in columns] + if style_data: + sheet.append(build_data_cells(sheet, values, rows_in_current_sheet)) + else: + sheet.append(values) + rows_in_current_sheet += 1 + + apply_auto_filter(sheet, len(columns), rows_in_current_sheet) + buffer = BytesIO() + workbook.save(buffer) + return buffer.getvalue(), sheet_count + + +def create_export_sheet( + workbook: Workbook, + title: str, + excel_headers: List[str], + column_widths: List[float], + style_data: bool, +): + sheet = workbook.create_sheet(title=title) + apply_sheet_layout(sheet, excel_headers, 0, column_widths, style_data) + sheet.append(build_header_cells(sheet, excel_headers)) + return sheet + + +def apply_sheet_layout( + sheet, + excel_headers: List[str], + data_row_count: int, + column_widths: Union[List[float], None] = None, + style_data: bool = False, +) -> None: + sheet.freeze_panes = "A2" + sheet.sheet_view.showGridLines = not style_data + sheet.row_dimensions[1].height = 24 + sheet.sheet_format.defaultRowHeight = 20 + + widths = column_widths or calculate_header_widths(excel_headers) + for index, width in enumerate(widths, start=1): + sheet.column_dimensions[get_column_letter(index)].width = width + + apply_auto_filter(sheet, len(excel_headers), data_row_count) + + +def apply_auto_filter(sheet, column_count: int, data_row_count: int) -> None: + if column_count <= 0: + return + last_column = get_column_letter(column_count) + last_row = max(data_row_count + 1, 1) + sheet.auto_filter.ref = f"A1:{last_column}{last_row}" + + +def build_header_cells(sheet, excel_headers: List[str]): + header_fill = PatternFill("solid", fgColor="F2F5F9") + header_font = Font(name=FONT_NAME, color="111827", bold=True, size=10.5) + header_alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) + border_side = Side(style="thin", color="CBD5E1") + header_border = Border( + left=border_side, + right=border_side, + top=border_side, + bottom=border_side, + ) + + cells = [] + for header in excel_headers: + cell = WriteOnlyCell(sheet, value=header) + cell.fill = header_fill + cell.font = header_font + cell.alignment = header_alignment + cell.border = header_border + cells.append(cell) + return cells + + +def build_data_cells(sheet, values: List[Any], row_index: int): + data_font = Font(name=FONT_NAME, color="1F2937", size=10) + data_alignment = Alignment(horizontal="left", vertical="center") + border_side = Side(style="thin", color="E2E8F0") + data_border = Border( + left=border_side, + right=border_side, + top=border_side, + bottom=border_side, + ) + fill = PatternFill("solid", fgColor="F8FAFC") if row_index % 2 else None + + cells = [] + for value in values: + cell = WriteOnlyCell(sheet, value=value) + cell.font = data_font + cell.alignment = data_alignment + cell.border = data_border + if fill: + cell.fill = fill + if should_write_as_text(value): + cell.number_format = "@" + cells.append(cell) + return cells + + +def should_write_as_text(value: Any) -> bool: + return isinstance(value, str) and value.isdigit() and len(value) >= 6 + + +def calculate_column_widths( + columns: List[str], + excel_headers: List[str], + rows: List[Dict[str, Any]], +) -> List[float]: + widths = calculate_header_widths(excel_headers) + + for row in rows[:WIDTH_SAMPLE_ROWS]: + for index, column in enumerate(columns): + value = serialize_cell(row.get(column)) + widths[index] = max(widths[index], fit_column_width(value)) + + return widths + + +def calculate_header_widths(excel_headers: List[str]) -> List[float]: + return [fit_column_width(header) for header in excel_headers] + + +def fit_column_width(value: Any) -> float: + width = display_width(value) + 3 + return max(MIN_COLUMN_WIDTH, min(width, MAX_COLUMN_WIDTH)) + + +def display_width(value: Any) -> int: + text = "" if value is None else str(value) + width = 0 + for char in text: + width += 2 if unicodedata.east_asian_width(char) in {"F", "W"} else 1 + if width >= MAX_COLUMN_WIDTH: + return width + return width + + +def normalize_headers( + headers_value: Union[Dict[str, Any], str, None], +) -> Dict[str, str]: + if headers_value is None: + return {} + + if isinstance(headers_value, str): + text = headers_value.strip() + if not text: + return {} + try: + headers_value = json.loads(text) + except json.JSONDecodeError as exc: + raise HTTPException( + status_code=400, detail="headers must be a JSON object" + ) from exc + + if not isinstance(headers_value, dict): + raise HTTPException(status_code=400, detail="headers must be a JSON object") + + return { + str(key): str(value) + for key, value in headers_value.items() + if key is not None and value is not None and str(value).strip() + } + + +def serialize_cell(value: Any) -> Any: + if value is None: + return "" + if isinstance(value, datetime): + return value.isoformat(sep=" ", timespec="seconds") + if isinstance(value, date): + return value.isoformat() + if isinstance(value, Decimal): + return str(value) + if isinstance(value, bytes): + return value.hex() + if isinstance(value, (dict, list, tuple)): + return json.dumps(value, ensure_ascii=False) + return value + + +def sanitize_filename(filename: str) -> str: + cleaned = re.sub(r'[\\/:*?"<>|\r\n\t]+', "_", filename).strip(" ._") + if not cleaned: + cleaned = "dify_export" + if not cleaned.lower().endswith(".xlsx"): + cleaned = f"{cleaned}.xlsx" + return cleaned[:140] + + +def upload_xlsx(contents: bytes, filename: str) -> tuple[str, str]: + object_name = f"dify/sql-export/{datetime.now().strftime('%Y/%m/%d')}/{uuid.uuid4().hex}/{filename}" + file_bytes = BytesIO(contents) + + if not minio_client.bucket_exists(EXPORT_BUCKET_NAME): + minio_client.make_bucket(EXPORT_BUCKET_NAME) + + minIO.push_file( + EXPORT_BUCKET_NAME, + object_name, + file_bytes, + contents, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + return object_name, minIO.get_temp_url(EXPORT_BUCKET_NAME, object_name) + + +@difyRouter.post("/sql/export", response_model=DifySQLExportResponse) +def export_sql_to_excel(payload: DifySQLExportRequest) -> DifySQLExportResponse: + sql = normalize_sql(payload.sql) + validate_read_only_sql(sql) + + filename = sanitize_filename(payload.filename) + rows = executeSQL(sql) + headers = normalize_headers(payload.headers or payload.headers_json) + contents, sheet_count = build_xlsx(rows, headers) + object_name, url = upload_xlsx(contents, filename) + + return DifySQLExportResponse( + url=url, + filename=filename, + bucket_name=EXPORT_BUCKET_NAME, + object_name=object_name, + row_count=len(rows), + sheet_count=sheet_count, + ) diff --git a/f10/Dockerfile b/f10/Dockerfile new file mode 100644 index 0000000..8d20dc2 --- /dev/null +++ b/f10/Dockerfile @@ -0,0 +1,9 @@ +FROM amazoncorretto:21-alpine + +WORKDIR /app + +COPY build/install/f10/ /app/ + +EXPOSE 8096 + +CMD ["./bin/f10"] \ No newline at end of file diff --git a/f10/build.ps1 b/f10/build.ps1 new file mode 100644 index 0000000..3074e64 --- /dev/null +++ b/f10/build.ps1 @@ -0,0 +1,50 @@ +$ErrorActionPreference = "Stop" + +$VERSION = "1.0" + +$IMAGE = "docker.bbitcn.net/bbit_f10/trace" + +Write-Host "Gradle installDist" +Write-Host "===============================" + +.\gradlew.bat clean installDist + +if ($LASTEXITCODE -ne 0) { +throw "installDist failed" +} + +Write-Host "Docker Build" +Write-Host "===============================" + +docker build ` -t "${IMAGE}:${VERSION}"` +-t "${IMAGE}:latest" ` +. + +if ($LASTEXITCODE -ne 0) { +throw "docker build failed" +} + +Write-Host "Push Version" +Write-Host "===============================" + +docker push "${IMAGE}:${VERSION}" + +if ($LASTEXITCODE -ne 0) { +throw "push version failed" +} + +Write-Host "Push Latest" +Write-Host "===============================" + +docker push "${IMAGE}:latest" + +if ($LASTEXITCODE -ne 0) { +throw "push latest failed" +} + +Write-Host "SUCCESS" +Write-Host "===============================" +Write-Host "" +Write-Host "Published:" +Write-Host " ${IMAGE}:${VERSION}" +Write-Host " ${IMAGE}:latest" diff --git a/f10/src/main/resources/application.yaml b/f10/src/main/resources/application.yaml index 163ed8c..049e95b 100644 --- a/f10/src/main/resources/application.yaml +++ b/f10/src/main/resources/application.yaml @@ -3,10 +3,9 @@ ktor: modules: - com.bbitcn.ApplicationKt.module deployment: - port: 8081 + port: 8096 traceability: # 访问主服务的地址 # core-base-url: "http://127.0.0.1:8089" # 开发 - core-base-url: "https://171.212.101.200:8090/api" # 生产 用IP -# core-base-url: "https://ai.ronsunny.cn:8090/api" # 生产 用域名 + core-base-url: "https://ai.bbitcn.net/api" # 生产 diff --git a/iot/Sentinel/.gitignore b/iot/Sentinel/.gitignore index 9f11b75..41f7f34 100644 --- a/iot/Sentinel/.gitignore +++ b/iot/Sentinel/.gitignore @@ -1 +1,2 @@ .idea/ +build/ diff --git a/ktor/src/main/kotlin/ink/snowflake/server/controller/Public.kt b/ktor/src/main/kotlin/ink/snowflake/server/controller/Public.kt index c91536f..fcec666 100644 --- a/ktor/src/main/kotlin/ink/snowflake/server/controller/Public.kt +++ b/ktor/src/main/kotlin/ink/snowflake/server/controller/Public.kt @@ -18,7 +18,7 @@ fun Application.Public() { runAdbCommand("disconnect") runAdbCommand("connect ${SERVER_PATH_FRP}:$port") val url = - "https://ai.ronsunny.cn:8090/remote#!action=stream&udid=s3.ronsunny.cn%3ATTT&player=mse&ws=wss%3A%2F%2Fai.ronsunny.cn%3A8090%2Fremote%3Faction%3Dproxy-adb%26remote%3Dtcp%253A8886%26udid%3Ds3.ronsunny.cn%253ATTT" + "https://remote.bbitcn.net/#!action=stream&udid=171.212.101.201%3ATTT&player=mse&ws=wss%3A%2F%2Fremote.bbitcn.net%2F%3Faction%3Dproxy-adb%26remote%3Dtcp%253A8886%26udid%3D171.212.101.201%253ATTT" .replace( "TTT", port diff --git a/ktor/src/main/resources/application.yaml b/ktor/src/main/resources/application.yaml index 7513f50..60dc279 100644 --- a/ktor/src/main/resources/application.yaml +++ b/ktor/src/main/resources/application.yaml @@ -42,5 +42,5 @@ ktor: traceability: # public-preview-base-url: "http://127.0.0.1:8081" # 开发测试用 - public-preview-base-url: "https://ats.f10.bbitcn.com" # 生产环境用 + public-preview-base-url: "https://trace.bbitcn.net" # 生产环境用 diff --git a/readme.md b/readme.md index d8230a6..482e94d 100644 --- a/readme.md +++ b/readme.md @@ -90,6 +90,7 @@ | | | | 19530 | 19530 | ce_milvus | Milvus | Minvus数据访问 | | | | /webui | 9091 | 9091 | ce_milvus | Milvus | Minvus**管理界面**，无需登录 | | | | | 3002 | 3000 | ce_attu | Attu | Minvus Attu**管理界面**，无需密码 | +| trace | / | / | 8096 | 8096 | f10-trace-server | Ktor | F10溯源模块 | - 暂时未部署项目 - CVAT