diff --git a/ai/video-sca/README b/ai/video-sca/README new file mode 100644 index 0000000..07ec6cf --- /dev/null +++ b/ai/video-sca/README @@ -0,0 +1,148 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +Prerequisites: +- DeepStreamSDK 8.0 +- NVIDIA Triton Inference Server (optional) +- Python 3.12 +- Gst-python + +To set up Triton Inference Server: (optional) +For x86_64 and Jetson Docker: + 1. Use the provided docker container and follow directions for + Triton Inference Server in the SDK README -- + be sure to prepare the detector models. + 2. Run the docker with this Python Bindings directory mapped + 3. Install required Python packages inside the container: + $ apt update + $ apt install python3-gi python3-dev python3-gst-1.0 -y + $ pip3 install pathlib + 4. Build and install pyds bindings: + Follow the instructions in bindings README in this repo to build and install + pyds wheel for Ubuntu 24.04 + 5. For Triton gRPC setup, please follow the instructions at below location: + /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app-triton-grpc/README + +For Jetson without Docker: + 1. Follow instructions in the DeepStream SDK README to set up + Triton Inference Server: + 2.1 Compile and install the nvdsinfer_customparser + 2.2 Prepare at least the Triton detector models + 2. Build and install pyds bindings: + Follow the instructions in bindings README in this repo to build and install + pyds wheel for Ubuntu 24.04 + 3. Clear the GStreamer cache if pipeline creation fails: + rm ~/.cache/gstreamer-1.0/* + 4. For Triton gRPC setup, please follow the instructions at below location: + /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app-triton-grpc/README + +To setup peoplenet model and configs (optional): +Download Peoplenet model: + $ mkdir -p /opt/nvidia/deepstream/deepstream/samples/models/peoplenet + $ cd /opt/nvidia/deepstream/deepstream/samples/models/peoplenet + $ wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=resnet34_peoplenet_int8.onnx' -O resnet34_peoplenet_int8.onnx + $ wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=resnet34_peoplenet_int8.txt' -O resnet34_peoplenet_int8.txt + $ wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=labels.txt' -O labels.txt + +Additionally, for Triton and Triton gRPC + $ cp config.pbtxt /opt/nvidia/deepstream/deepstream/samples/models/peoplenet + $ mkdir -p /opt/nvidia/deepstream/deepstream/samples/models/peoplenet/1 + $ /usr/src/tensorrt/bin/trtexec --onnx=/opt/nvidia/deepstream/deepstream/samples/models/peoplenet/resnet34_peoplenet_int8.onnx --fp16 \ + --saveEngine=/opt/nvidia/deepstream/deepstream/samples/models/peoplenet/1/resnet34_peoplenet_int8.onnx_b2_gpu0_fp16.engine \ + --minShapes="input_1:0":1x3x544x960 \ + --optShapes="input_1:0":2x3x544x960 \ + --maxShapes="input_1:0":2x3x544x960 + +To run: + $ python3 deepstream_test_3.py -i [uri2] ... [uriN] [--no-display] [--silent] +e.g. + $ python3 deepstream_test_3.py -i file:///home/ubuntu/video1.mp4 file:///home/ubuntu/video2.mp4 + $ python3 deepstream_test_3.py -i rtsp://127.0.0.1/video1 rtsp://127.0.0.1/video2 -s + +To run peoplenet, test3 now supports 3 modes: + + 1. nvinfer + peoplenet: this mode still uses TRT for inferencing. + + $ python3 deepstream_test_3.py -i [uri2] ... [uriN] --pgie nvinfer -c config_infer_primary_peoplenet.txt [--no-display] [--silent] + + 2. nvinferserver + peoplenet : this mode uses Triton for inferencing. + + $ python3 deepstream_test_3.py -i [uri2] ... [uriN] --pgie nvinferserver -c config_triton_infer_primary_peoplenet.txt [--no-display] [-s] + + 3. nvinferserver (gRPC) + peoplenet : this mode uses Triton gRPC for inferencing. + + $ mkdir -p /opt/nvidia/deepstream/deepstream/samples/models/peoplenet-grpc/peoplenet + $ cp /opt/nvidia/deepstream/deepstream/samples/models/peoplenet/config.pbtxt /opt/nvidia/deepstream/deepstream/samples/models/peoplenet-grpc/peoplenet/config.pbtxt + $ cp -a /opt/nvidia/deepstream/deepstream/samples/models/peoplenet/1 /opt/nvidia/deepstream/deepstream/samples/models/peoplenet-grpc/peoplenet/1 + $ tritonserver --model-repository=/opt/nvidia/deepstream/deepstream/samples/models/peoplenet-grpc + $ python3 deepstream_test_3.py -i [uri2] ... [uriN] --pgie nvinferserver-grpc -c config_triton_grpc_infer_primary_peoplenet.txt [--no-display] [--silent] + +e.g. + $ python3 deepstream_test_3.py -i file:///home/ubuntu/video1.mp4 file:///home/ubuntu/video2.mp4 --pgie nvinfer -c config_infer_primary_peoplenet.txt --no-display --silent + $ python3 deepstream_test_3.py -i rtsp://127.0.0.1/video1 rtsp://127.0.0.1/video2 --pgie nvinferserver -c config_triton_infer_primary_peoplenet.txt -s + $ python3 deepstream_test_3.py -i rtsp://127.0.0.1/video1 rtsp://127.0.0.1/video2 --pgie nvinferserver-grpc -c config_triton_grpc_infer_primary_peoplenet.txt --no-display --silent + +Note: +1) if --pgie is not specified, test3 uses nvinfer and default model, not peoplenet. +2) Both --pgie and -c need to be provided for custom models. +3) Configs other than peoplenet can also be provided using the above approach. +4) --no-display option disables on-screen video display. +5) -s/--silent option can be used to suppress verbose output. +6) --file-loop option can be used to loop input files after EOS. +7) --disable-probe option can be used to disable the probe function and to use nvdslogger for perf measurements. +8) To enable Pipeline Latency Measurement, set environment variable : NVDS_ENABLE_LATENCY_MEASUREMENT=1 +9) To enable Component Level Latency Measurement, set environment variable : NVDS_ENABLE_COMPONENT_LATENCY_MEASUREMENT=1 in addition to NVDS_ENABLE_LATENCY_MEASUREMENT=1 + +This document describes the sample deepstream-test3 application. + + + * Use multiple sources in the pipeline. + * Use a uridecodebin so that any type of input (e.g. RTSP/File), any GStreamer + supported container format, and any codec can be used as input. + * Configure the stream-muxer to generate a batch of frames and infer on the + batch for better resource utilization. + * Extract the stream metadata, which contains useful information about the + frames in the batched buffer. + * Showcases how to enable latency measurement using probe function + +Refer to the deepstream-test1 sample documentation for an example of simple +single-stream inference, bounding-box overlay, and rendering. + +This sample accepts one or more H.264/H.265 video streams as input. It creates +a source bin for each input and connects the bins to an instance of the +"nvstreammux" element, which forms the batch of frames. The batch of +frames is fed to "nvinfer" for batched inferencing. The batched buffer is +composited into a 2D tile array using "nvmultistreamtiler." The rest of the +pipeline is similar to the deepstream-test1 sample. + +The "width" and "height" properties must be set on the stream-muxer to set the +output resolution. If the input frame resolution is different from +stream-muxer's "width" and "height", the input frame will be scaled to muxer's +output resolution. + +The stream-muxer waits for a user-defined timeout before forming the batch. The +timeout is set using the "batched-push-timeout" property. If the complete batch +is formed before the timeout is reached, the batch is pushed to the downstream +element. If the timeout is reached before the complete batch can be formed +(which can happen in case of rtsp sources), the batch is formed from the +available input buffers and pushed. Ideally, the timeout of the stream-muxer +should be set based on the framerate of the fastest source. It can also be set +to -1 to make the stream-muxer wait infinitely. + +The "nvmultistreamtiler" composite streams based on their stream-ids in +row-major order (starting from stream 0, left to right across the top row, then +across the next row, etc.). diff --git a/ai/video-sca/app.py b/ai/video-sca/app.py new file mode 100644 index 0000000..7e4460f --- /dev/null +++ b/ai/video-sca/app.py @@ -0,0 +1,97 @@ +import pika +import uuid +import time +import os +import urllib.parse +from minio import Minio +import os +from sca import sca + + +# MinIO 配置 +MINIO_HOST = "ai.ronsunny.cn:9000" +MINIO_ACCESS_KEY = "minioadmin" +MINIO_SECRET_KEY = "minioadmin" +MINIO_SECURE = True +BUCKET_NAME = "video-sca" +TEMP_DIR = "tmp/" + +# RabbitMQ 配置 +RABBITMQ_HOST = "10.10.12.101" +RABBITMQ_PORT = 5672 +RABBITMQ_VHOST = "bbit_ai" +RABBITMQ_USER = "ai_lab_iva_sca" +RABBITMQ_PASS = "123456" +RABBITMQ_QUEUE = "/sca_queue" + +# 初始化 MinIO 客户端 +minio_client = Minio( + MINIO_HOST, + access_key=MINIO_ACCESS_KEY, + secret_key=MINIO_SECRET_KEY, + secure=MINIO_SECURE +) + +# ------------------- 消息处理 ------------------- +def callback(ch, method, properties, body): + import json + data = json.loads(body) + + for record in data.get("Records", []): + # 解析桶名和文件 Key + bucket = record["s3"]["bucket"]["name"] + key_encoded = record["s3"]["object"]["key"] + key = urllib.parse.unquote(key_encoded) + filename = os.path.basename(key) + temp_file_path = os.path.join(TEMP_DIR, filename) + + print(f"[1] 下载文件 {key} 到临时目录 {temp_file_path}") + minio_client.fget_object(bucket, key, temp_file_path) + + # AI 分析耗时 + print(f"[2] AI 分析...") + # 生成分析后文件 UUID 名 + new_uuid = str(uuid.uuid4()) + new_filename = f"{new_uuid}" + new_file_path = os.path.join(TEMP_DIR, new_filename) + + # 转为绝对路径 + temp_file_path = os.path.abspath(temp_file_path) + new_file_path = os.path.abspath(new_file_path) + + # 这里暂时复制同一个视频模拟分析结果 + sca(["file://" + temp_file_path], new_file_path) + print(f"[3] AI 分析完成，生成文件 {new_file_path}") + + # 上传分析后视频到 video-sca/ai/ + dest_key = f"ai/{new_filename}" + minio_client.fput_object(bucket, dest_key, new_file_path) + print(f"[4] 上传分析视频到 MinIO {dest_key}") + + # 删除临时文件 + os.remove(temp_file_path) + os.remove(new_file_path) + print("[5] 临时文件已删除\n") + print("[6] 本次任务完成\n") + + # 确认消息 + ch.basic_ack(delivery_tag=method.delivery_tag) +# 连接 RabbitMQ +# 设置凭证 +credentials = pika.PlainCredentials(RABBITMQ_USER, RABBITMQ_PASS) +connection = pika.BlockingConnection( + pika.ConnectionParameters( + host=RABBITMQ_HOST, + port=RABBITMQ_PORT, + virtual_host=RABBITMQ_VHOST, + credentials=credentials + ) +) +channel = connection.channel() +channel.queue_declare(queue=RABBITMQ_QUEUE, durable=True) + +channel.basic_qos(prefetch_count=1) # 一次只处理一个文件 +channel.basic_consume(queue=RABBITMQ_QUEUE, on_message_callback=callback) + +print("Waiting for messages...") +channel.start_consuming() diff --git a/ai/video-sca/common/FPS.py b/ai/video-sca/common/FPS.py new file mode 100644 index 0000000..3a31823 --- /dev/null +++ b/ai/video-sca/common/FPS.py @@ -0,0 +1,67 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +import time +from threading import Lock +start_time=time.time() + +fps_mutex = Lock() + +class GETFPS: + def __init__(self,stream_id): + global start_time + self.start_time=start_time + self.is_first=True + self.frame_count=0 + self.stream_id=stream_id + + def update_fps(self): + end_time = time.time() + if self.is_first: + self.start_time = end_time + self.is_first = False + else: + global fps_mutex + with fps_mutex: + self.frame_count = self.frame_count + 1 + + def get_fps(self): + end_time = time.time() + with fps_mutex: + stream_fps = float(self.frame_count/(end_time - self.start_time)) + self.frame_count = 0 + self.start_time = end_time + return round(stream_fps, 2) + + def print_data(self): + print('frame_count=',self.frame_count) + print('start_time=',self.start_time) + +class PERF_DATA: + def __init__(self, num_streams=1): + self.perf_dict = {} + self.all_stream_fps = {} + for i in range(num_streams): + self.all_stream_fps["stream{0}".format(i)]=GETFPS(i) + + def perf_print_callback(self): + self.perf_dict = {stream_index:stream.get_fps() for (stream_index, stream) in self.all_stream_fps.items()} + print ("\n**PERF: ", self.perf_dict, "\n") + return True + + def update_fps(self, stream_index): + self.all_stream_fps[stream_index].update_fps() diff --git a/ai/video-sca/common/__init__.py b/ai/video-sca/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai/video-sca/common/bus_call.py b/ai/video-sca/common/bus_call.py new file mode 100644 index 0000000..37073c8 --- /dev/null +++ b/ai/video-sca/common/bus_call.py @@ -0,0 +1,34 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +import gi +import sys +gi.require_version('Gst', '1.0') +from gi.repository import Gst +def bus_call(bus, message, loop): + t = message.type + if t == Gst.MessageType.EOS: + sys.stdout.write("End-of-stream\n") + loop.quit() + elif t==Gst.MessageType.WARNING: + err, debug = message.parse_warning() + sys.stderr.write("Warning: %s: %s\n" % (err, debug)) + elif t == Gst.MessageType.ERROR: + err, debug = message.parse_error() + sys.stderr.write("Error: %s: %s\n" % (err, debug)) + loop.quit() + return True diff --git a/ai/video-sca/common/platform_info.py b/ai/video-sca/common/platform_info.py new file mode 100644 index 0000000..8382d1f --- /dev/null +++ b/ai/video-sca/common/platform_info.py @@ -0,0 +1,94 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +import sys +import platform +from threading import Lock +from cuda.bindings import runtime +from cuda.bindings import driver + +guard_platform_info = Lock() + +class PlatformInfo: + def __init__(self): + self.is_wsl_system = False + self.wsl_verified = False + self.is_integrated_gpu_system = False + self.is_integrated_gpu_verified = False + self.is_aarch64_platform = False + self.is_aarch64_verified = False + + def is_wsl(self): + with guard_platform_info: + # Check if its already verified as WSL system or not. + if not self.wsl_verified: + try: + # Open /proc/version file + with open("/proc/version", "r") as version_file: + # Read the content + version_info = version_file.readline() + version_info = version_info.lower() + self.wsl_verified = True + + # Check if "microsoft" is present in the version information + if "microsoft" in version_info: + self.is_wsl_system = True + except Exception as e: + print(f"ERROR: Opening /proc/version failed: {e}") + + return self.is_wsl_system + + def is_integrated_gpu(self): + #Using cuda apis to identify whether integrated/discreet + #This is required to distinguish Tegra and ARM_SBSA devices + with guard_platform_info: + #Cuda initialize + if not self.is_integrated_gpu_verified: + cuda_init_result, = driver.cuInit(0) + if cuda_init_result == driver.CUresult.CUDA_SUCCESS: + #Get cuda devices count + device_count_result, num_devices = driver.cuDeviceGetCount() + if device_count_result == driver.CUresult.CUDA_SUCCESS: + #If atleast one device is found, we can use the property from + #the first device + if num_devices >= 1: + #Get properties from first device + property_result, properties = runtime.cudaGetDeviceProperties(0) + if property_result == runtime.cudaError_t.cudaSuccess: + print("Is it Integrated GPU? :", properties.integrated) + self.is_integrated_gpu_system = properties.integrated + self.is_integrated_gpu_verified = True + else: + print("ERROR: Getting cuda device property failed: {}".format(property_result)) + else: + print("ERROR: No cuda devices found to check whether iGPU/dGPU") + else: + print("ERROR: Getting cuda device count failed: {}".format(device_count_result)) + else: + print("ERROR: Cuda init failed: {}".format(cuda_init_result)) + + return self.is_integrated_gpu_system + + def is_platform_aarch64(self): + #Check if platform is aarch64 using uname + if not self.is_aarch64_verified: + if platform.uname()[4] == 'aarch64': + self.is_aarch64_platform = True + self.is_aarch64_verified = True + return self.is_aarch64_platform + +sys.path.append('/opt/nvidia/deepstream/deepstream/lib') diff --git a/ai/video-sca/common/utils.py b/ai/video-sca/common/utils.py new file mode 100644 index 0000000..f0e9e17 --- /dev/null +++ b/ai/video-sca/common/utils.py @@ -0,0 +1,24 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +import ctypes +import sys +sys.path.append('/opt/nvidia/deepstream/deepstream/lib') + +def long_to_uint64(l): + value = ctypes.c_uint64(l & 0xffffffffffffffff).value + return value diff --git a/ai/video-sca/config.pbtxt b/ai/video-sca/config.pbtxt new file mode 100644 index 0000000..66d6a01 --- /dev/null +++ b/ai/video-sca/config.pbtxt @@ -0,0 +1,43 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +name: "peoplenet" +platform: "tensorrt_plan" +max_batch_size: 2 +default_model_filename: "resnet34_peoplenet_int8.onnx_b2_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + dims: [ 3, 544, 960 ] + } +] +output [ + { + name: "output_bbox/BiasAdd:0" + data_type: TYPE_FP32 + dims: [ 12, 34, 60 ] + }, + { + name: "output_cov/Sigmoid:0" + data_type: TYPE_FP32 + dims: [ 3, 34, 60 ] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/ai/video-sca/config/config_tracker_NvDCF_perf.yml b/ai/video-sca/config/config_tracker_NvDCF_perf.yml new file mode 100644 index 0000000..71c845f --- /dev/null +++ b/ai/video-sca/config/config_tracker_NvDCF_perf.yml @@ -0,0 +1,58 @@ +%YAML:1.0 + +BaseConfig: + minDetectorConfidence: 0.6 # If the confidence of a detector bbox is lower than this, then it won't be considered for tracking + +TargetManagement: + enableBboxUnClipping: 1 # In case the bbox is likely to be clipped by image border, unclip bbox + maxTargetsPerStream: 150 # Max number of targets to track per stream. Recommended to set >10. Note: this value should account for the targets being tracked in shadow mode as well. Max value depends on the GPU memory capacity + + # [Creation & Termination Policy] + minIouDiff4NewTarget: 0.55 # If the IOU between the newly detected object and any of the existing targets is higher than this threshold, this newly detected object will be discarded. + minTrackerConfidence: 0.2 # If the confidence of an object tracker is lower than this on the fly, then it will be tracked in shadow mode. Valid Range: [0.0, 1.0] + probationAge: 5 # If the target's age exceeds this, the target will be considered to be valid. + maxShadowTrackingAge: 30 # Max length of shadow tracking. If the shadowTrackingAge exceeds this limit, the tracker will be terminated. + earlyTerminationAge: 1 # If the shadowTrackingAge reaches this threshold while in TENTATIVE period, the target will be terminated prematurely. + +TrajectoryManagement: + useUniqueID: 0 # Use 64-bit long Unique ID when assignining tracker ID. Default is [true] + +DataAssociator: + dataAssociatorType: 0 # the type of data associator among { DEFAULT= 0 } + associationMatcherType: 0 # the type of matching algorithm among { GREEDY=0, GLOBAL=1 } + checkClassMatch: 1 # If checked, only the same-class objects are associated with each other. Default: true + + # [Association Metric: Thresholds for valid candidates] + minMatchingScore4Overall: 0.0 # Min total score + minMatchingScore4SizeSimilarity: 0.6 # Min bbox size similarity score + minMatchingScore4Iou: 0.0 # Min IOU score + minMatchingScore4VisualSimilarity: 0.7 # Min visual similarity score + + # [Association Metric: Weights] + matchingScoreWeight4VisualSimilarity: 0.6 # Weight for the visual similarity (in terms of correlation response ratio) + matchingScoreWeight4SizeSimilarity: 0.0 # Weight for the Size-similarity score + matchingScoreWeight4Iou: 0.4 # Weight for the IOU score + +StateEstimator: + stateEstimatorType: 1 # the type of state estimator among { DUMMY=0, SIMPLE=1, REGULAR=2 } + + # [Dynamics Modeling] + processNoiseVar4Loc: 2.0 # Process noise variance for bbox center + processNoiseVar4Size: 1.0 # Process noise variance for bbox size + processNoiseVar4Vel: 0.1 # Process noise variance for velocity + measurementNoiseVar4Detector: 4.0 # Measurement noise variance for detector's detection + measurementNoiseVar4Tracker: 16.0 # Measurement noise variance for tracker's localization + +VisualTracker: + visualTrackerType: 1 # the type of visual tracker among { DUMMY=0, NvDCF=1 } + + # [NvDCF: Feature Extraction] + useColorNames: 1 # Use ColorNames feature + useHog: 0 # Use Histogram-of-Oriented-Gradient (HOG) feature + featureImgSizeLevel: 2 # Size of a feature image. Valid range: {1, 2, 3, 4, 5}, from the smallest to the largest + featureFocusOffsetFactor_y: -0.2 # The offset for the center of hanning window relative to the feature height. The center of hanning window would move by (featureFocusOffsetFactor_y*featureMatSize.height) in vertical direction + + # [NvDCF: Correlation Filter] + filterLr: 0.075 # learning rate for DCF filter in exponential moving average. Valid Range: [0.0, 1.0] + filterChannelWeightsLr: 0.1 # learning rate for the channel weights among feature channels. Valid Range: [0.0, 1.0] + gaussianSigma: 0.75 # Standard deviation for Gaussian for desired response when creating DCF filter [pixels] diff --git a/ai/video-sca/config/pgie_config.txt b/ai/video-sca/config/pgie_config.txt new file mode 100644 index 0000000..e5b0664 --- /dev/null +++ b/ai/video-sca/config/pgie_config.txt @@ -0,0 +1,81 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.00392156862745098 +onnx-file=../../models/best.pt.onnx +model-engine-file=../../models/best.pt.onnx_b1_gpu0_fp16.engine +labelfile-path=../../models/labels.txt +batch-size=1 +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=5 +interval=0 +gie-unique-id=1 +## 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) +cluster-mode=2 +custom-lib-path=../lib/libnvdsinfer_custom_impl_Yolo.so +parse-bbox-func-name=NvDsInferParseYolo +engine-create-func-name=NvDsInferYoloCudaEngineGet +network-type=0 +maintain-aspect-ratio=1 +symmetric-padding=1 + +[class-attrs-all] +topk=20 +nms-iou-threshold=0.45 +pre-cluster-threshold=0.2 diff --git a/ai/video-sca/config_infer_primary_peoplenet.txt b/ai/video-sca/config_infer_primary_peoplenet.txt new file mode 100644 index 0000000..0d9d51f --- /dev/null +++ b/ai/video-sca/config_infer_primary_peoplenet.txt @@ -0,0 +1,60 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +onnx-file=/opt/nvidia/deepstream/deepstream/samples/models/peoplenet/resnet34_peoplenet_int8.onnx +labelfile-path=/opt/nvidia/deepstream/deepstream/samples/models/peoplenet/labels.txt +model-engine-file=/opt/nvidia/deepstream/deepstream/samples/models/peoplenet/resnet34_peoplenet_int8.onnx_b1_gpu0_fp16.engine +int8-calib-file=/opt/nvidia/deepstream/deepstream/samples/models/peoplenet/resnet34_peoplenet_int8.txt +infer-dims=3;544;960 +uff-input-blob-name=input_1 +batch-size=1 +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=3 +cluster-mode=2 +interval=0 +gie-unique-id=1 +output-blob-names=output_bbox/BiasAdd:0;output_cov/Sigmoid:0 + +#Use the config params below for dbscan clustering mode +#[class-attrs-all] +#detected-min-w=4 +#detected-min-h=4 +#minBoxes=3 +#eps=0.7 + +#Use the config params below for NMS clustering mode +[class-attrs-all] +topk=20 +nms-iou-threshold=0.5 +pre-cluster-threshold=0.2 + +## Per class configurations +[class-attrs-0] +topk=20 +nms-iou-threshold=0.5 +pre-cluster-threshold=0.4 + +#[class-attrs-1] +#pre-cluster-threshold=0.05 +#eps=0.7 +#dbscan-min-score=0.5 diff --git a/ai/video-sca/config_triton_grpc_infer_primary_peoplenet.txt b/ai/video-sca/config_triton_grpc_infer_primary_peoplenet.txt new file mode 100644 index 0000000..8218acd --- /dev/null +++ b/ai/video-sca/config_triton_grpc_infer_primary_peoplenet.txt @@ -0,0 +1,79 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 1 + backend { + inputs: [ { + name: "input_1:0" + }] + outputs: [ + {name: "output_bbox/BiasAdd:0"}, + {name: "output_cov/Sigmoid:0"} + ] + triton { + model_name: "peoplenet" + version: -1 + grpc { + url: "0.0.0.0:8001" + enable_cuda_buffer_sharing: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + channel_offsets: [0, 0, 0] + } + } + + postprocess { + labelfile_path: "/opt/nvidia/deepstream/deepstream/samples/models/peoplenet/labels.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} + diff --git a/ai/video-sca/config_triton_infer_primary_peoplenet.txt b/ai/video-sca/config_triton_infer_primary_peoplenet.txt new file mode 100644 index 0000000..0bc87c8 --- /dev/null +++ b/ai/video-sca/config_triton_infer_primary_peoplenet.txt @@ -0,0 +1,79 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 1 + backend { + inputs: [ { + name: "input_1:0" + }] + outputs: [ + {name: "output_bbox/BiasAdd:0"}, + {name: "output_cov/Sigmoid:0"} + ] + triton { + model_name: "peoplenet" + version: -1 + model_repo { + root: "/opt/nvidia/deepstream/deepstream/samples/models/" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + channel_offsets: [0, 0, 0] + } + } + + postprocess { + labelfile_path: "/opt/nvidia/deepstream/deepstream/samples/models/peoplenet/labels.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} + diff --git a/ai/video-sca/lib/libnvds_nvmultiobjecttracker.so b/ai/video-sca/lib/libnvds_nvmultiobjecttracker.so new file mode 100644 index 0000000..e559958 Binary files /dev/null and b/ai/video-sca/lib/libnvds_nvmultiobjecttracker.so differ diff --git a/ai/video-sca/lib/libnvdsinfer_custom_impl_Yolo.so b/ai/video-sca/lib/libnvdsinfer_custom_impl_Yolo.so new file mode 100644 index 0000000..578afa1 Binary files /dev/null and b/ai/video-sca/lib/libnvdsinfer_custom_impl_Yolo.so differ diff --git a/ai/video-sca/main copy.py b/ai/video-sca/main copy.py new file mode 100644 index 0000000..c917e43 --- /dev/null +++ b/ai/video-sca/main copy.py @@ -0,0 +1,635 @@ +#!/usr/bin/env python3 + +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +import sys +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent # apps/mine +DEEPSTREAM_PY_DIR = SCRIPT_DIR.parent # deepstream_python_apps + +sys.path.insert(0, str(DEEPSTREAM_PY_DIR)) # common/ 在这里 +sys.path.insert(0, str(DEEPSTREAM_PY_DIR / "pyds")) # pyds 模块 + +from pathlib import Path +from os import environ +import gi +import configparser +import argparse +gi.require_version('Gst', '1.0') +from gi.repository import GLib, Gst +from ctypes import * +import time +import sys +import math +import platform +from common.platform_info import PlatformInfo +from common.bus_call import bus_call +from common.FPS import PERF_DATA + +import pyds +import sys +from pathlib import Path +from sources.dev.app.utils.my_utils import * + +display_type = 0 +silent = False +file_loop = False +perf_data = None +measure_latency = False + +MAX_DISPLAY_LEN=64 +PGIE_CLASS_ID_NORMAL = 0 +PGIE_CLASS_ID_DOUBLE_PUPA = 1 +PGIE_CLASS_ID_SPOT = 2 +PGIE_CLASS_ID_HAIRY = 3 +PGIE_CLASS_ID_MAGGOT_SHELL = 4 +MUXER_OUTPUT_WIDTH=1920 +MUXER_OUTPUT_HEIGHT=1080 +MUXER_BATCH_TIMEOUT_USEC = 33000 +TILED_OUTPUT_WIDTH=1280 +TILED_OUTPUT_HEIGHT=720 +YOLO_SIZE=768 +GST_CAPS_FEATURES_NVMM="memory:NVMM" +OSD_PROCESS_MODE= 0 +OSD_DISPLAY_TEXT= 1 +pgie_classes_str= ["Vehicle", "TwoWheeler", "Person","RoadSign"] + +count_sc = 0 +od_labels = [] + +with open("../models/labels.txt", "r") as f: + od_labels = [line.strip() for line in f.readlines()] + +def add_fps_display_meta(lineCount, batch_meta, frame_meta, display_text): + display_meta=pyds.nvds_acquire_display_meta_from_pool(batch_meta) + display_meta.num_labels = 1 + py_nvosd_text_params = display_meta.text_params[0] + py_nvosd_text_params.display_text = display_text + py_nvosd_text_params.x_offset = 12 + py_nvosd_text_params.y_offset = 12 + lineCount * 24 + py_nvosd_text_params.font_params.font_name = "Serif" + py_nvosd_text_params.font_params.font_size = 10 + py_nvosd_text_params.font_params.font_color.set(1.0, 1.0, 1.0, 1.0) + py_nvosd_text_params.set_bg_clr = 1 + py_nvosd_text_params.text_bg_clr.set(0.0, 0.0, 0.0, 1.0) + # print(pyds.get_string(py_nvosd_text_params.display_text)) + pyds.nvds_add_display_meta_to_frame(frame_meta, display_meta) + +# pgie_src_pad_buffer_probe will extract metadata received on tiler sink pad +# and update params for drawing rectangle, object information etc. +def pgie_src_pad_buffer_probe(pad,info,u_data): + frame_number=0 + num_rects=0 + got_fps = False + gst_buffer = info.get_buffer() + if not gst_buffer: + print("Unable to get GstBuffer ") + return + # Retrieve batch metadata from the gst_buffer + # Note that pyds.gst_buffer_get_nvds_batch_meta() expects the + # C address of gst_buffer as input, which is obtained with hash(gst_buffer) + + # Enable latency measurement via probe if environment variable NVDS_ENABLE_LATENCY_MEASUREMENT=1 is set. + # To enable component level latency measurement, please set environment variable + # NVDS_ENABLE_COMPONENT_LATENCY_MEASUREMENT=1 in addition to the above. + global measure_latency + if measure_latency: + num_sources_in_batch = pyds.nvds_measure_buffer_latency(hash(gst_buffer)) + if num_sources_in_batch == 0: + print("Unable to get number of sources in GstBuffer for latency measurement") + + batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer)) + l_frame = batch_meta.frame_meta_list + global count_sc + while l_frame is not None: + try: + # Note that l_frame.data needs a cast to pyds.NvDsFrameMeta + # The casting is done by pyds.NvDsFrameMeta.cast() + # The casting also keeps ownership of the underlying memory + # in the C code, so the Python garbage collector will leave + # it alone. + frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data) + except StopIteration: + break + + frame_number=frame_meta.frame_num + l_obj=frame_meta.obj_meta_list + num_rects = frame_meta.num_obj_meta + obj_counter = { + PGIE_CLASS_ID_NORMAL:0, + PGIE_CLASS_ID_SPOT:0, + PGIE_CLASS_ID_DOUBLE_PUPA:0, + PGIE_CLASS_ID_HAIRY:0, + PGIE_CLASS_ID_MAGGOT_SHELL:0 + } + while l_obj is not None: + try: + # Casting l_obj.data to pyds.NvDsObjectMeta + obj_meta=pyds.NvDsObjectMeta.cast(l_obj.data) + except StopIteration: + break + obj_counter[obj_meta.class_id] += 1 + try: + l_obj=l_obj.next + except StopIteration: + break + # if not silent: + # print("帧ID=", frame_number, "总蚕茧数=",num_rects,"正茧数=",obj_counter[PGIE_CLASS_ID_NORMAL],"黄斑茧数=",obj_counter[PGIE_CLASS_ID_SPOT]) + + # Update frame rate through this probe + stream_index = "stream{0}".format(frame_meta.pad_index) + global perf_data + perf_data.update_fps(stream_index) + + text = f"Object Count: {num_rects}\n" + for i in range(len(od_labels)): + if obj_counter[i] > 0: + text += f"\t{od_labels[i]}: {obj_counter[i]}\n" + add_fps_display_meta(0,batch_meta,frame_meta, text) + try: + l_frame=l_frame.next + except StopIteration: + break + + # Multi Object Tracker---------------------------------------------------- + l_obj=batch_meta.batch_user_meta_list + while l_obj is not None: + try: + user_meta=pyds.NvDsUserMeta.cast(l_obj.data) + except StopIteration: + break + # 遍历user_meta.base_meta.meta_type + if user_meta and user_meta.base_meta.meta_type==pyds.NvDsMetaType.NVDS_TRACKER_PAST_FRAME_META : + try: + pPastDataBatch = pyds.NvDsTargetMiscDataBatch.cast(user_meta.user_meta_data) + except StopIteration: + break + for miscDataStream in pyds.NvDsTargetMiscDataBatch.list(pPastDataBatch): + for miscDataObj in pyds.NvDsTargetMiscDataStream.list(miscDataStream): + unique_id = miscDataObj.uniqueId + if unique_id > count_sc: + count_sc = unique_id + try: + l_obj=l_obj.next + except StopIteration: + break + + return Gst.PadProbeReturn.OK + +def cb_newpad(decodebin, decoder_src_pad,data): + print("In cb_newpad\n") + caps=decoder_src_pad.get_current_caps() + if not caps: + caps = decoder_src_pad.query_caps() + gststruct=caps.get_structure(0) + gstname=gststruct.get_name() + source_bin=data + features=caps.get_features(0) + + # Need to check if the pad created by the decodebin is for video and not + # audio. + print("gstname=",gstname) + if(gstname.find("video")!=-1): + # Link the decodebin pad only if decodebin has picked nvidia + # decoder plugin nvdec_*. We do this by checking if the pad caps contain + # NVMM memory features. + print("features=",features) + if features.contains("memory:NVMM"): + # Get the source bin ghost pad + bin_ghost_pad=source_bin.get_static_pad("src") + if not bin_ghost_pad.set_target(decoder_src_pad): + sys.stderr.write("Failed to link decoder src pad to source bin ghost pad\n") + else: + sys.stderr.write(" Error: Decodebin did not pick nvidia decoder plugin.\n") + +def decodebin_child_added(child_proxy,Object,name,user_data): + print("Decodebin child added:", name, "\n") + if(name.find("decodebin") != -1): + Object.connect("child-added",decodebin_child_added,user_data) + + if "source" in name: + source_element = child_proxy.get_by_name("source") + if source_element.find_property('drop-on-latency') != None: + Object.set_property("drop-on-latency", True) + +def create_source_bin(index,uri): + print("Creating source bin") + + # Create a source GstBin to abstract this bin's content from the rest of the + # pipeline + bin_name="source-bin-%02d" %index + print(bin_name) + nbin=Gst.Bin.new(bin_name) + if not nbin: + sys.stderr.write(" Unable to create source bin \n") + + # Source element for reading from the uri. + # We will use decodebin and let it figure out the container format of the + # stream and the codec and plug the appropriate demux and decode plugins. + if file_loop: + # use nvurisrcbin to enable file-loop + uri_decode_bin=Gst.ElementFactory.make("nvurisrcbin", "uri-decode-bin") + uri_decode_bin.set_property("file-loop", 1) + uri_decode_bin.set_property("cudadec-memtype", 0) + else: + uri_decode_bin=Gst.ElementFactory.make("uridecodebin", "uri-decode-bin") + if not uri_decode_bin: + sys.stderr.write(" Unable to create uri decode bin \n") + # We set the input uri to the source element + uri_decode_bin.set_property("uri",uri) + # Connect to the "pad-added" signal of the decodebin which generates a + # callback once a new pad for raw data has beed created by the decodebin + uri_decode_bin.connect("pad-added",cb_newpad,nbin) + uri_decode_bin.connect("child-added",decodebin_child_added,nbin) + + # We need to create a ghost pad for the source bin which will act as a proxy + # for the video decoder src pad. The ghost pad will not have a target right + # now. Once the decode bin creates the video decoder and generates the + # cb_newpad callback, we will set the ghost pad target to the video decoder + # src pad. + Gst.Bin.add(nbin,uri_decode_bin) + bin_pad=nbin.add_pad(Gst.GhostPad.new_no_target("src",Gst.PadDirection.SRC)) + if not bin_pad: + sys.stderr.write(" Failed to add ghost pad in source bin \n") + return None + return nbin + +def main(args, requested_pgie=None, config=None, disable_probe=False): + global perf_data,TILED_OUTPUT_WIDTH,TILED_OUTPUT_HEIGHT,YOLO_SIZE,count_sc + perf_data = PERF_DATA(len(args)) + + number_sources=len(args) + if number_sources == 1: + print("重置输出分辨率至:{} x {}".format(TILED_OUTPUT_WIDTH, TILED_OUTPUT_HEIGHT)) + TILED_OUTPUT_WIDTH, TILED_OUTPUT_HEIGHT = getVideoResolution(stream_paths[0]) + + platform_info = PlatformInfo() + # Standard GStreamer initialization + Gst.init(None) + + # Create gstreamer elements */ + # Create Pipeline element that will form a connection of other elements + print("Creating Pipeline \n ") + pipeline = Gst.Pipeline() + is_live = False + + if not pipeline: + sys.stderr.write(" Unable to create Pipeline \n") + print("Creating streamux \n ") + + # Create nvstreammux instance to form batches from one or more sources. + nvstreammux = Gst.ElementFactory.make("nvstreammux", "Stream-muxer") + if not nvstreammux: + sys.stderr.write(" Unable to create NvStreamMux \n") + + pipeline.add(nvstreammux) + for i in range(number_sources): + print("Creating source_bin ",i," \n ") + uri_name=args[i] + if uri_name.find("rtsp://") == 0 : + is_live = True + source_bin=create_source_bin(i, uri_name) + if not source_bin: + sys.stderr.write("Unable to create source bin \n") + pipeline.add(source_bin) + padname="sink_%u" %i + sinkpad= nvstreammux.request_pad_simple(padname) + if not sinkpad: + sys.stderr.write("Unable to create sink pad bin \n") + srcpad=source_bin.get_static_pad("src") + if not srcpad: + sys.stderr.write("Unable to create src pad bin \n") + srcpad.link(sinkpad) + queue1=Gst.ElementFactory.make("queue","queue1") + queue2=Gst.ElementFactory.make("queue","queue2") + queue3=Gst.ElementFactory.make("queue","queue3") + queue4=Gst.ElementFactory.make("queue","queue4") + queue5=Gst.ElementFactory.make("queue","queue5") + pipeline.add(queue1) + pipeline.add(queue2) + pipeline.add(queue3) + pipeline.add(queue4) + pipeline.add(queue5) + + nvdslogger = None + + print("Creating Pgie \n ") + if requested_pgie != None and (requested_pgie == 'nvinferserver' or requested_pgie == 'nvinferserver-grpc') : + pgie = Gst.ElementFactory.make("nvinferserver", "primary-inference") + elif requested_pgie != None and requested_pgie == 'nvinfer': + # local tensorrt infer + pgie = Gst.ElementFactory.make("nvinfer", "primary-inference") + else: + pgie = Gst.ElementFactory.make("nvinfer", "primary-inference") + + if not pgie: + sys.stderr.write(" Unable to create pgie : %s\n" % requested_pgie) + + if disable_probe: + # Use nvdslogger for perf measurement instead of probe function + print ("Creating nvdslogger \n") + nvdslogger = Gst.ElementFactory.make("nvdslogger", "nvdslogger") + + print("Creating tiler \n ") + nvmultistreamtiler=Gst.ElementFactory.make("nvmultistreamtiler", "nvtiler") + if not nvmultistreamtiler: + sys.stderr.write(" Unable to create tiler \n") + print("Creating nvvidconv \n ") + nvvidconv = Gst.ElementFactory.make("nvvideoconvert", "convertor") + if not nvvidconv: + sys.stderr.write(" Unable to create nvvidconv \n") + print("Creating nvosd \n ") + nvosd = Gst.ElementFactory.make("nvdsosd", "onscreendisplay") + if not nvosd: + sys.stderr.write(" Unable to create nvosd \n") + nvosd.set_property('process-mode',OSD_PROCESS_MODE) + nvosd.set_property('display-text',OSD_DISPLAY_TEXT) + + if file_loop: + if platform_info.is_integrated_gpu(): + # Set nvbuf-memory-type=4 for integrated gpu for file-loop (nvurisrcbin case) + nvstreammux.set_property('nvbuf-memory-type', 4) + else: + # Set nvbuf-memory-type=2 for x86 for file-loop (nvurisrcbin case) + nvstreammux.set_property('nvbuf-memory-type', 2) + + if display_type == 0: + sink = Gst.ElementFactory.make("filesink", "file-sink") + sink.set_property("location", "./out.mp4") + sink.set_property("sync", 0) + elif display_type == 1: + print("Creating Fakesink \n") + sink = Gst.ElementFactory.make("fakesink", "fakesink") + sink.set_property('enable-last-sample', 0) + sink.set_property('sync', 0) + else: + if platform_info.is_integrated_gpu(): + print("Creating nv3dsink \n") + sink = Gst.ElementFactory.make("nv3dsink", "nv3d-sink") + if not sink: + sys.stderr.write(" Unable to create nv3dsink \n") + else: + if platform_info.is_platform_aarch64(): + print("Creating nv3dsink \n") + sink = Gst.ElementFactory.make("nv3dsink", "nv3d-sink") + else: + print("Creating EGLSink \n") + sink = Gst.ElementFactory.make("nveglglessink", "nvvideo-renderer") + if not sink: + sys.stderr.write(" Unable to create egl sink \n") + + if not sink: + sys.stderr.write(" Unable to create sink element \n") + + if is_live: + print("At least one of the sources is live") + nvstreammux.set_property('live-source', 1) + + + nvvidconv2 = Gst.ElementFactory.make("nvvideoconvert", "convertor2") + if not nvvidconv2: + sys.stderr.write(" Unable to create nvvidconv2 \n") + + encoder = Gst.ElementFactory.make("nvv4l2h264enc", "encoder") # for h264 + # encoder = Gst.ElementFactory.make("avenc_mpeg4", "encoder") # for mpeg4 + if not encoder: + sys.stderr.write(" Unable to create encoder \n") + encoder.set_property("bitrate", 2000000) + + h264parse = Gst.ElementFactory.make("h264parse", "parser") + # codeparser = Gst.ElementFactory.make("mpeg4videoparse", "mpeg4-parser") + if not h264parse: + sys.stderr.write(" Unable to create code parser \n") + + qtmux = Gst.ElementFactory.make("qtmux", "qtmux") + if not qtmux: + sys.stderr.write(" Unable to create code parser \n") + + nvstreammux.set_property('width', YOLO_SIZE) + nvstreammux.set_property('height', YOLO_SIZE) + nvstreammux.set_property('batch-size', number_sources) + nvstreammux.set_property('batched-push-timeout', MUXER_BATCH_TIMEOUT_USEC) + if requested_pgie == "nvinferserver" and config != None: + pgie.set_property('config-file-path', config) + elif requested_pgie == "nvinferserver-grpc" and config != None: + pgie.set_property('config-file-path', config) + elif requested_pgie == "nvinfer" and config != None: + pgie.set_property('config-file-path', config) + else: + pgie.set_property('config-file-path', "config/pgie_config.txt") + pgie_batch_size=pgie.get_property("batch-size") + if(pgie_batch_size != number_sources): + print("WARNING: Overriding infer-config batch-size",pgie_batch_size," with number of sources ", number_sources," \n") + pgie.set_property("batch-size",number_sources) + tiler_rows=int(math.sqrt(number_sources)) + tiler_columns=int(math.ceil((1.0*number_sources)/tiler_rows)) + nvmultistreamtiler.set_property("rows",tiler_rows) + nvmultistreamtiler.set_property("columns",tiler_columns) + nvmultistreamtiler.set_property("width", TILED_OUTPUT_WIDTH) + nvmultistreamtiler.set_property("height", TILED_OUTPUT_HEIGHT) + if platform_info.is_integrated_gpu(): + nvmultistreamtiler.set_property("compute-hw", 2) + else: + nvmultistreamtiler.set_property("compute-hw", 1) + sink.set_property("qos",0) + + tracker = Gst.ElementFactory.make("nvtracker", "tracker") + if not tracker: + sys.stderr.write(" Unable to create tracker \n") + tracker.set_property('tracker-width', YOLO_SIZE) + tracker.set_property('tracker-height', YOLO_SIZE) + # To fix 'gstnvtracker: Unable to acquire a user meta buffer. Try increasing user-meta-pool-size' + tracker.set_property('user-meta-pool-size', 128) + tracker.set_property('gpu_id', 0) + tracker.set_property('ll-lib-file', 'lib/libnvds_nvmultiobjecttracker.so') + tracker.set_property('ll-config-file', 'config/config_tracker_NvDCF_perf.yml') + + print("Adding elements to Pipeline \n") + pipeline.add(pgie) + if nvdslogger: + pipeline.add(nvdslogger) + pipeline.add(nvmultistreamtiler) + pipeline.add(tracker) + pipeline.add(nvvidconv) + pipeline.add(nvosd) + pipeline.add(sink) + + print("Linking elements in the Pipeline \n") + nvstreammux.link(queue1) + queue1.link(pgie) + pgie.link(tracker) + tracker.link(queue2) + if nvdslogger: + queue2.link(nvdslogger) + nvdslogger.link(nvmultistreamtiler) + else: + queue2.link(nvmultistreamtiler) + nvmultistreamtiler.link(queue3) + queue3.link(nvvidconv) + nvvidconv.link(queue4) + queue4.link(nvosd) + nvosd.link(queue5) + + if display_type == 0: + pipeline.add(nvvidconv2) + pipeline.add(encoder) + pipeline.add(h264parse) + pipeline.add(qtmux) + + queue5.link(nvvidconv2) + nvvidconv2.link(encoder) + encoder.link(h264parse) + h264parse.link(qtmux) + qtmux.link(sink) + else: + queue5.link(sink) + + # create an event loop and feed gstreamer bus mesages to it + loop = GLib.MainLoop() + bus = pipeline.get_bus() + bus.add_signal_watch() + bus.connect ("message", bus_call, loop) + pgie_src_pad=nvosd.get_static_pad("sink") + if not pgie_src_pad: + sys.stderr.write(" Unable to get src pad \n") + else: + if not disable_probe: + pgie_src_pad.add_probe(Gst.PadProbeType.BUFFER, pgie_src_pad_buffer_probe, 0) + # perf callback function to print fps every 5 sec + GLib.timeout_add(5000, perf_data.perf_print_callback) + + # Enable latency measurement via probe if environment variable NVDS_ENABLE_LATENCY_MEASUREMENT=1 is set. + # To enable component level latency measurement, please set environment variable + # NVDS_ENABLE_COMPONENT_LATENCY_MEASUREMENT=1 in addition to the above. + if environ.get('NVDS_ENABLE_LATENCY_MEASUREMENT') == '1': + print ("Pipeline Latency Measurement enabled!\nPlease set env var NVDS_ENABLE_COMPONENT_LATENCY_MEASUREMENT=1 for Component Latency Measurement") + global measure_latency + measure_latency = True + + # List the sources + print("Now playing...") + for i, source in enumerate(args): + print(i, ": ", source) + + print("Starting pipeline \n") + # start play back and listed to events + pipeline.set_state(Gst.State.PLAYING) + try: + loop.run() + except: + pass + + print("总蚕茧数：",count_sc-1) + # cleanup + print("Exiting app\n") + pipeline.set_state(Gst.State.NULL) + +def parse_args(): + + parser = argparse.ArgumentParser(prog="deepstream_test_3", + description="deepstream-test3 multi stream, multi model inference reference app") + parser.add_argument( + "-i", + "--input", + help="Path to input streams", + nargs="+", + metavar="URIs", + default=["a"], + required=True, + ) + parser.add_argument( + "-c", + "--configfile", + metavar="config_location.txt", + default=None, + help="Choose the config-file to be used with specified pgie", + ) + parser.add_argument( + "-g", + "--pgie", + default=None, + help="Choose Primary GPU Inference Engine", + choices=["nvinfer", "nvinferserver", "nvinferserver-grpc"], + ) + parser.add_argument( + "--display_type", + type=int, + default=0, + help="Display type: 0=file, 1=fake" + ) + + parser.add_argument( + "--file-loop", + action="store_true", + default=False, + dest='file_loop', + help="Loop the input file sources after EOS", + ) + parser.add_argument( + "--disable-probe", + action="store_true", + default=False, + dest='disable_probe', + help="Disable the probe function and use nvdslogger for FPS", + ) + parser.add_argument( + "-s", + "--silent", + action="store_true", + default=False, + dest='silent', + help="Disable verbose output", + ) + # Check input arguments + if len(sys.argv) == 1: + parser.print_help(sys.stderr) + sys.exit(1) + args = parser.parse_args() + + stream_paths = args.input + pgie = args.pgie + config = args.configfile + disable_probe = args.disable_probe + global display_type + global silent + global file_loop + display_type = args.display_type + silent = args.silent + file_loop = args.file_loop + + if config and not pgie or pgie and not config: + sys.stderr.write ("\nEither pgie or configfile is missing. Please specify both! Exiting...\n\n\n\n") + parser.print_help() + sys.exit(1) + if config: + config_path = Path(config) + if not config_path.is_file(): + sys.stderr.write ("Specified config-file: %s doesn't exist. Exiting...\n\n" % config) + sys.exit(1) + + print(vars(args)) + return stream_paths, pgie, config, disable_probe + +if __name__ == '__main__': + # 此程序支持多个视频流输入，但只为单视频流输入开发 + + # 外部输入参数 + stream_paths, pgie, config, disable_probe = parse_args() + + sys.exit(main(stream_paths, pgie, config, disable_probe)) \ No newline at end of file diff --git a/ai/video-sca/sca.py b/ai/video-sca/sca.py new file mode 100644 index 0000000..34aa6aa --- /dev/null +++ b/ai/video-sca/sca.py @@ -0,0 +1,515 @@ +#!/usr/bin/env python3 + +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +import sys +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent # apps/mine +DEEPSTREAM_PY_DIR = SCRIPT_DIR.parent # deepstream_python_apps + +sys.path.insert(0, str(DEEPSTREAM_PY_DIR)) # common/ 在这里 +sys.path.insert(0, str(DEEPSTREAM_PY_DIR / "pyds")) # pyds 模块 + +from pathlib import Path +from os import environ +import gi +gi.require_version('Gst', '1.0') +from gi.repository import GLib, Gst +from ctypes import * +import time +import sys +import math +from common.platform_info import PlatformInfo +from common.bus_call import bus_call +from common.FPS import PERF_DATA +import pyds +import sys +from pathlib import Path +from utils.my_utils import * + +display_type = 0 +silent = False +file_loop = False +perf_data = None +measure_latency = False + +MAX_DISPLAY_LEN=64 +PGIE_CLASS_ID_NORMAL = 0 +PGIE_CLASS_ID_DOUBLE_PUPA = 1 +PGIE_CLASS_ID_SPOT = 2 +PGIE_CLASS_ID_HAIRY = 3 +PGIE_CLASS_ID_MAGGOT_SHELL = 4 + +MUXER_OUTPUT_WIDTH=1920 +MUXER_OUTPUT_HEIGHT=1080 +MUXER_BATCH_TIMEOUT_USEC = 33000 +TILED_OUTPUT_WIDTH=1280 +TILED_OUTPUT_HEIGHT=720 +YOLO_SIZE=768 +GST_CAPS_FEATURES_NVMM="memory:NVMM" +OSD_PROCESS_MODE= 0 +OSD_DISPLAY_TEXT= 1 +tracker_dict = {} + +obj_counter = { + PGIE_CLASS_ID_NORMAL:0, + PGIE_CLASS_ID_SPOT:0, + PGIE_CLASS_ID_DOUBLE_PUPA:0, + PGIE_CLASS_ID_HAIRY:0, + PGIE_CLASS_ID_MAGGOT_SHELL:0 +} + +od_labels = [] +statics={} +timestamp_in_video = 0 + +with open("../models/labels.txt", "r") as f: + od_labels = [line.strip() for line in f.readlines()] + +def stats_callback(): + # 这里统计你需要的数据，例如： + global timestamp_in_video,statics + timestamp_in_video += 1 # 假设每秒钟调用一次 + temp_json = {} + + for class_id, count in obj_counter.items(): + class_name = get_type_name_by_id(class_id) + temp_json[class_name] = count + statics[timestamp_in_video] = temp_json + return True # 返回 True 才会继续循环调用 + + +def add_fps_display_meta(lineCount, batch_meta, frame_meta, display_text): + display_meta=pyds.nvds_acquire_display_meta_from_pool(batch_meta) + display_meta.num_labels = 1 + py_nvosd_text_params = display_meta.text_params[0] + py_nvosd_text_params.display_text = display_text + py_nvosd_text_params.x_offset = 12 + py_nvosd_text_params.y_offset = 12 + lineCount * 24 + py_nvosd_text_params.font_params.font_name = "Serif" + py_nvosd_text_params.font_params.font_size = 10 + py_nvosd_text_params.font_params.font_color.set(1.0, 1.0, 1.0, 1.0) + py_nvosd_text_params.set_bg_clr = 1 + py_nvosd_text_params.text_bg_clr.set(0.0, 0.0, 0.0, 1.0) + # print(pyds.get_string(py_nvosd_text_params.display_text)) + pyds.nvds_add_display_meta_to_frame(frame_meta, display_meta) + +# pgie_src_pad_buffer_probe will extract metadata received on tiler sink pad +# and update params for drawing rectangle, object information etc. +def pgie_src_pad_buffer_probe(pad,info,u_data): + global measure_latency, obj_counter + frame_number=0 + num_rects=0 + got_fps = False + gst_buffer = info.get_buffer() + if not gst_buffer: + print("Unable to get GstBuffer ") + return + # Retrieve batch metadata from the gst_buffer + # Note that pyds.gst_buffer_get_nvds_batch_meta() expects the + # C address of gst_buffer as input, which is obtained with hash(gst_buffer) + + # Enable latency measurement via probe if environment variable NVDS_ENABLE_LATENCY_MEASUREMENT=1 is set. + # To enable component level latency measurement, please set environment variable + # NVDS_ENABLE_COMPONENT_LATENCY_MEASUREMENT=1 in addition to the above. + if measure_latency: + num_sources_in_batch = pyds.nvds_measure_buffer_latency(hash(gst_buffer)) + if num_sources_in_batch == 0: + print("Unable to get number of sources in GstBuffer for latency measurement") + + batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer)) + l_frame = batch_meta.frame_meta_list + while l_frame is not None: + try: + # Note that l_frame.data needs a cast to pyds.NvDsFrameMeta + # The casting is done by pyds.NvDsFrameMeta.cast() + # The casting also keeps ownership of the underlying memory + # in the C code, so the Python garbage collector will leave + # it alone. + frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data) + except StopIteration: + break + + frame_number=frame_meta.frame_num + l_obj=frame_meta.obj_meta_list + num_rects = frame_meta.num_obj_meta + + # 初始化 + obj_counter = { + PGIE_CLASS_ID_NORMAL:0, + PGIE_CLASS_ID_SPOT:0, + PGIE_CLASS_ID_DOUBLE_PUPA:0, + PGIE_CLASS_ID_HAIRY:0, + PGIE_CLASS_ID_MAGGOT_SHELL:0 + } + while l_obj is not None: + try: + # Casting l_obj.data to pyds.NvDsObjectMeta + obj_meta=pyds.NvDsObjectMeta.cast(l_obj.data) + except StopIteration: + break + obj_counter[obj_meta.class_id] += 1 + try: + l_obj=l_obj.next + except StopIteration: + break + # if not silent: + # print("帧ID=", frame_number, "总蚕茧数=",num_rects,"正茧数=",obj_counter[PGIE_CLASS_ID_NORMAL],"黄斑茧数=",obj_counter[PGIE_CLASS_ID_SPOT]) + + # Update frame rate through this probe + stream_index = "stream{0}".format(frame_meta.pad_index) + global perf_data + perf_data.update_fps(stream_index) + + text = f"Object Count: {num_rects}\n" + for i in range(len(od_labels)): + if obj_counter[i] > 0: + text += f"\t{od_labels[i]}: {obj_counter[i]}\n" + add_fps_display_meta(0,batch_meta,frame_meta, text) + try: + l_frame=l_frame.next + except StopIteration: + break + + # Multi Object Tracker---------------------------------------------------- + l_obj=batch_meta.batch_user_meta_list + while l_obj is not None: + try: + user_meta=pyds.NvDsUserMeta.cast(l_obj.data) + except StopIteration: + break + # 遍历user_meta.base_meta.meta_type + if user_meta and user_meta.base_meta.meta_type==pyds.NvDsMetaType.NVDS_TRACKER_PAST_FRAME_META : + try: + pPastDataBatch = pyds.NvDsTargetMiscDataBatch.cast(user_meta.user_meta_data) + except StopIteration: + break + for miscDataStream in pyds.NvDsTargetMiscDataBatch.list(pPastDataBatch): + for miscDataObj in pyds.NvDsTargetMiscDataStream.list(miscDataStream): + unique_id = miscDataObj.uniqueId + tracker_dict[unique_id] = miscDataObj.classId + try: + l_obj=l_obj.next + except StopIteration: + break + + return Gst.PadProbeReturn.OK + +def cb_newpad(decodebin, decoder_src_pad,data): + print("In cb_newpad\n") + caps=decoder_src_pad.get_current_caps() + if not caps: + caps = decoder_src_pad.query_caps() + gststruct=caps.get_structure(0) + gstname=gststruct.get_name() + source_bin=data + features=caps.get_features(0) + + # Need to check if the pad created by the decodebin is for video and not + # audio. + print("gstname=",gstname) + if(gstname.find("video")!=-1): + # Link the decodebin pad only if decodebin has picked nvidia + # decoder plugin nvdec_*. We do this by checking if the pad caps contain + # NVMM memory features. + print("features=",features) + if features.contains("memory:NVMM"): + # Get the source bin ghost pad + bin_ghost_pad=source_bin.get_static_pad("src") + if not bin_ghost_pad.set_target(decoder_src_pad): + sys.stderr.write("Failed to link decoder src pad to source bin ghost pad\n") + else: + sys.stderr.write(" Error: Decodebin did not pick nvidia decoder plugin.\n") + +def decodebin_child_added(child_proxy,Object,name,user_data): + print("Decodebin child added:", name, "\n") + if(name.find("decodebin") != -1): + Object.connect("child-added",decodebin_child_added,user_data) + + if "source" in name: + source_element = child_proxy.get_by_name("source") + if source_element.find_property('drop-on-latency') != None: + Object.set_property("drop-on-latency", True) + +def create_source_bin(index,uri): + print("Creating source bin") + + # Create a source GstBin to abstract this bin's content from the rest of the + # pipeline + bin_name="source-bin-%02d" %index + print(bin_name) + nbin=Gst.Bin.new(bin_name) + if not nbin: + sys.stderr.write(" Unable to create source bin \n") + + # Source element for reading from the uri. + # We will use decodebin and let it figure out the container format of the + # stream and the codec and plug the appropriate demux and decode plugins. + if file_loop: + # use nvurisrcbin to enable file-loop + uri_decode_bin=Gst.ElementFactory.make("nvurisrcbin", "uri-decode-bin") + uri_decode_bin.set_property("file-loop", 1) + uri_decode_bin.set_property("cudadec-memtype", 0) + else: + uri_decode_bin=Gst.ElementFactory.make("uridecodebin", "uri-decode-bin") + if not uri_decode_bin: + sys.stderr.write(" Unable to create uri decode bin \n") + # We set the input uri to the source element + uri_decode_bin.set_property("uri",uri) + # Connect to the "pad-added" signal of the decodebin which generates a + # callback once a new pad for raw data has beed created by the decodebin + uri_decode_bin.connect("pad-added",cb_newpad,nbin) + uri_decode_bin.connect("child-added",decodebin_child_added,nbin) + + # We need to create a ghost pad for the source bin which will act as a proxy + # for the video decoder src pad. The ghost pad will not have a target right + # now. Once the decode bin creates the video decoder and generates the + # cb_newpad callback, we will set the ghost pad target to the video decoder + # src pad. + Gst.Bin.add(nbin,uri_decode_bin) + bin_pad=nbin.add_pad(Gst.GhostPad.new_no_target("src",Gst.PadDirection.SRC)) + if not bin_pad: + sys.stderr.write(" Failed to add ghost pad in source bin \n") + return None + return nbin + +def sca(stream_paths, out_put_file_name:str): + disable_probe=False + global perf_data,TILED_OUTPUT_WIDTH,TILED_OUTPUT_HEIGHT,YOLO_SIZE + perf_data = PERF_DATA(len(stream_paths)) + + number_sources=len(stream_paths) + if number_sources == 1: + print("重置输出分辨率至:{} x {}".format(TILED_OUTPUT_WIDTH, TILED_OUTPUT_HEIGHT)) + TILED_OUTPUT_WIDTH, TILED_OUTPUT_HEIGHT = getVideoResolution(stream_paths[0]) + + platform_info = PlatformInfo() + Gst.init(None) + + pipeline = Gst.Pipeline() + is_live = False + + if not pipeline: + sys.stderr.write(" Unable to create Pipeline \n") + + nvstreammux = Gst.ElementFactory.make("nvstreammux", "Stream-muxer") + if not nvstreammux: + sys.stderr.write(" Unable to create NvStreamMux \n") + + pipeline.add(nvstreammux) + for i in range(number_sources): + uri_name=stream_paths[i] + source_bin=create_source_bin(i, uri_name) + if not source_bin: + sys.stderr.write("Unable to create source bin \n") + pipeline.add(source_bin) + padname="sink_%u" %i + sinkpad= nvstreammux.request_pad_simple(padname) + if not sinkpad: + sys.stderr.write("Unable to create sink pad bin \n") + srcpad=source_bin.get_static_pad("src") + if not srcpad: + sys.stderr.write("Unable to create src pad bin \n") + srcpad.link(sinkpad) + queue1=Gst.ElementFactory.make("queue","queue1") + queue2=Gst.ElementFactory.make("queue","queue2") + queue3=Gst.ElementFactory.make("queue","queue3") + queue4=Gst.ElementFactory.make("queue","queue4") + queue5=Gst.ElementFactory.make("queue","queue5") + pipeline.add(queue1) + pipeline.add(queue2) + pipeline.add(queue3) + pipeline.add(queue4) + pipeline.add(queue5) + + nvdslogger = None + + pgie = Gst.ElementFactory.make("nvinfer", "primary-inference") + + if not pgie: + sys.stderr.write(" Unable to create pgie") + pgie.set_property('config-file-path', "config/pgie_config.txt") + + if disable_probe: + # Use nvdslogger for perf measurement instead of probe function + nvdslogger = Gst.ElementFactory.make("nvdslogger", "nvdslogger") + + nvmultistreamtiler=Gst.ElementFactory.make("nvmultistreamtiler", "nvtiler") + if not nvmultistreamtiler: + sys.stderr.write(" Unable to create tiler \n") + nvvidconv = Gst.ElementFactory.make("nvvideoconvert", "convertor") + if not nvvidconv: + sys.stderr.write(" Unable to create nvvidconv \n") + nvosd = Gst.ElementFactory.make("nvdsosd", "onscreendisplay") + if not nvosd: + sys.stderr.write(" Unable to create nvosd \n") + nvosd.set_property('process-mode',OSD_PROCESS_MODE) + nvosd.set_property('display-text',OSD_DISPLAY_TEXT) + if file_loop: + if platform_info.is_integrated_gpu(): + # Set nvbuf-memory-type=4 for integrated gpu for file-loop (nvurisrcbin case) + nvstreammux.set_property('nvbuf-memory-type', 4) + else: + # Set nvbuf-memory-type=2 for x86 for file-loop (nvurisrcbin case) + nvstreammux.set_property('nvbuf-memory-type', 2) + + sink = Gst.ElementFactory.make("filesink", "file-sink") + sink.set_property("location", out_put_file_name) + sink.set_property("sync", 0) + + if not sink: + sys.stderr.write(" Unable to create sink element \n") + if is_live: + nvstreammux.set_property('live-source', 1) + nvvidconv2 = Gst.ElementFactory.make("nvvideoconvert", "convertor2") + if not nvvidconv2: + sys.stderr.write(" Unable to create nvvidconv2 \n") + encoder = Gst.ElementFactory.make("nvv4l2h264enc", "encoder") # for h264 + # encoder = Gst.ElementFactory.make("avenc_mpeg4", "encoder") # for mpeg4 + if not encoder: + sys.stderr.write(" Unable to create encoder \n") + encoder.set_property("bitrate", 2000000) + + h264parse = Gst.ElementFactory.make("h264parse", "parser") + # codeparser = Gst.ElementFactory.make("mpeg4videoparse", "mpeg4-parser") + if not h264parse: + sys.stderr.write(" Unable to create code parser \n") + + qtmux = Gst.ElementFactory.make("qtmux", "qtmux") + if not qtmux: + sys.stderr.write(" Unable to create code parser \n") + + nvstreammux.set_property('width', YOLO_SIZE) + nvstreammux.set_property('height', YOLO_SIZE) + nvstreammux.set_property('batch-size', number_sources) + nvstreammux.set_property('batched-push-timeout', MUXER_BATCH_TIMEOUT_USEC) + + pgie_batch_size=pgie.get_property("batch-size") + if(pgie_batch_size != number_sources): + pgie.set_property("batch-size",number_sources) + tiler_rows=int(math.sqrt(number_sources)) + tiler_columns=int(math.ceil((1.0*number_sources)/tiler_rows)) + nvmultistreamtiler.set_property("rows",tiler_rows) + nvmultistreamtiler.set_property("columns",tiler_columns) + nvmultistreamtiler.set_property("width", TILED_OUTPUT_WIDTH) + nvmultistreamtiler.set_property("height", TILED_OUTPUT_HEIGHT) + if platform_info.is_integrated_gpu(): + nvmultistreamtiler.set_property("compute-hw", 2) + else: + nvmultistreamtiler.set_property("compute-hw", 1) + sink.set_property("qos",0) + + tracker = Gst.ElementFactory.make("nvtracker", "tracker") + if not tracker: + sys.stderr.write(" Unable to create tracker \n") + tracker.set_property('tracker-width', YOLO_SIZE) + tracker.set_property('tracker-height', YOLO_SIZE) + # To fix 'gstnvtracker: Unable to acquire a user meta buffer. Try increasing user-meta-pool-size' + tracker.set_property('user-meta-pool-size', 128) + tracker.set_property('gpu_id', 0) + tracker.set_property('ll-lib-file', 'lib/libnvds_nvmultiobjecttracker.so') + tracker.set_property('ll-config-file', 'config/config_tracker_NvDCF_perf.yml') + + pipeline.add(pgie) + if nvdslogger: + pipeline.add(nvdslogger) + pipeline.add(nvmultistreamtiler) + pipeline.add(tracker) + pipeline.add(nvvidconv) + pipeline.add(nvosd) + pipeline.add(sink) + + nvstreammux.link(queue1) + queue1.link(pgie) + pgie.link(tracker) + tracker.link(queue2) + if nvdslogger: + queue2.link(nvdslogger) + nvdslogger.link(nvmultistreamtiler) + else: + queue2.link(nvmultistreamtiler) + nvmultistreamtiler.link(queue3) + queue3.link(nvvidconv) + nvvidconv.link(queue4) + queue4.link(nvosd) + nvosd.link(queue5) + + if display_type == 0: + pipeline.add(nvvidconv2) + pipeline.add(encoder) + pipeline.add(h264parse) + pipeline.add(qtmux) + + queue5.link(nvvidconv2) + nvvidconv2.link(encoder) + encoder.link(h264parse) + h264parse.link(qtmux) + qtmux.link(sink) + else: + queue5.link(sink) + + # create an event loop and feed gstreamer bus mesages to it + loop = GLib.MainLoop() + bus = pipeline.get_bus() + bus.add_signal_watch() + # 关闭日志 + bus.connect ("message", bus_call, loop) + pgie_src_pad=nvosd.get_static_pad("sink") + if not pgie_src_pad: + sys.stderr.write(" Unable to get src pad \n") + else: + if not disable_probe: + pgie_src_pad.add_probe(Gst.PadProbeType.BUFFER, pgie_src_pad_buffer_probe, 0) + # perf callback function to print fps every 5 sec + GLib.timeout_add(5000, perf_data.perf_print_callback) + GLib.timeout_add(500, stats_callback) + + + # Enable latency measurement via probe if environment variable NVDS_ENABLE_LATENCY_MEASUREMENT=1 is set. + # To enable component level latency measurement, please set environment variable + # NVDS_ENABLE_COMPONENT_LATENCY_MEASUREMENT=1 in addition to the above. + if environ.get('NVDS_ENABLE_LATENCY_MEASUREMENT') == '1': + print ("Pipeline Latency Measurement enabled!\nPlease set env var NVDS_ENABLE_COMPONENT_LATENCY_MEASUREMENT=1 for Component Latency Measurement") + global measure_latency + measure_latency = True + + # List the sources + for i, source in enumerate(stream_paths): + print(i, ": ", source) + + print("Starting pipeline \n") + # start play back and listed to events + pipeline.set_state(Gst.State.PLAYING) + # 开始时间 + start_time = time.time() + try: + loop.run() + except: + pass + + pipeline.set_state(Gst.State.NULL) + print("总蚕茧数：",len(tracker_dict)) + print("Exiting app\n") + # 结束时间 + end_time = time.time() + analysis_time = end_time - start_time + print(f"处理时间: {analysis_time} 秒") + + global statics + save_sca_results(stream_paths[0], out_put_file_name,tracker_dict,analysis_time,statics) diff --git a/ai/video-sca/utils/GlobalVariable.py b/ai/video-sca/utils/GlobalVariable.py new file mode 100644 index 0000000..610c90d --- /dev/null +++ b/ai/video-sca/utils/GlobalVariable.py @@ -0,0 +1 @@ +LOCAL_IP = "10.10.12.101" diff --git a/ai/video-sca/utils/__init__.py b/ai/video-sca/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai/video-sca/utils/db.py b/ai/video-sca/utils/db.py new file mode 100644 index 0000000..cb9ec0d --- /dev/null +++ b/ai/video-sca/utils/db.py @@ -0,0 +1,78 @@ +from utils.pgDb import pg_pool +import json + +def insert_sca_video( + name, + raw_object_name, + ai_object_name, + duration, + size, + video_codec, + audio_codec, + overall_bit_rate, + resolution, + sc_analysis_time, + sc_analysis_total_count, + sc_analysis_max_count, + sc_analysis_primary_type, + sc_analysis_secondary_type, + other_info, +): + with pg_pool.getConn() as conn: + with conn.cursor() as cursor: + other_info = json.dumps(other_info, ensure_ascii=False) + cursor.execute( + """ + INSERT INTO sca_videos ( + name, raw_object_name, ai_object_name, duration, size, video_codec, audio_codec, + overall_bit_rate, resolution, sc_analysis_time, sc_analysis_total_count, sc_analysis_max_count, + sc_analysis_primary_type, sc_analysis_secondary_type, other_info, created_at + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW()) + RETURNING id + """, + ( + name, + raw_object_name, + ai_object_name, + duration, + size, + video_codec, + audio_codec, + overall_bit_rate, + resolution, + sc_analysis_time, + sc_analysis_total_count, + sc_analysis_max_count, + sc_analysis_primary_type, + sc_analysis_secondary_type, + other_info, + ), + ) + + new_id = cursor.fetchone()[0] + conn.commit() + return new_id + +def insert_sca_video_details( + v_id, + time_stamp, + other_info, +): + with pg_pool.getConn() as conn: + with conn.cursor() as cursor: + other_info = json.dumps(other_info, ensure_ascii=False) + cursor.execute( + """ + INSERT INTO sca_video_details ( + v_id, time_stamp, other_info + ) + VALUES (%s, %s, %s) + """, + ( + v_id, + time_stamp, + other_info, + ), + ) + conn.commit() diff --git a/ai/video-sca/utils/my_utils.py b/ai/video-sca/utils/my_utils.py new file mode 100644 index 0000000..5a88796 --- /dev/null +++ b/ai/video-sca/utils/my_utils.py @@ -0,0 +1,97 @@ + +from pymediainfo import MediaInfo +from datetime import datetime +import json +from utils.db import insert_sca_video,insert_sca_video_details +from collections import Counter + +def getVideoResolution(file_path): + media_info = MediaInfo.parse(file_path) # 解析视频文件 + + for track in media_info.tracks: + if track.track_type == 'Video': + return track.width,track.height + +def get_type_name_by_id(id): + type_dict = { + 0: "正茧", + 1: "双宫茧", + 2: "黄斑茧", + 3: "毛茧", + 4: "蛆壳茧", + } + return type_dict.get(id, "未知类型") + +# 将视频信息保存至数据库 +def save_sca_results(file_path, out_put_file_name,tracker_dict, + analysis_time,statics =[]): + media_info = MediaInfo.parse(file_path) # 解析视频文件 + video_data = {} # 用于存储视频信息 + + for track in media_info.tracks: + if track.track_type == 'General': + video_data.update({ + "v_file_name": track.file_name_extension, + "v_duration": round(track.duration / 1000), + "v_size": round(track.file_size/1024,2), + "v_video_codec": track.video_format_list, + "v_audio_codec": track.audio_codecs, + "v_overall_bit_rate": track.overall_bit_rate, + }) + if track.track_type == 'Video': + video_data.update({ + "v_resolution": f"{track.width}x{track.height}" + }) + + # 分析 得出数量最多的种类 以及 出现次多的种类 + counter = Counter(tracker_dict.values()) # 统计所有 value 的出现次数 + most_common = counter.most_common(2) # 取出现次数前两名（返回列表） + primary_type = get_type_name_by_id(most_common[0][0]) if most_common else "" + secondary_type = get_type_name_by_id(most_common[1][0]) if len(most_common) > 1 else "" + + # 1️⃣ 统计 { value: count } + count_map = {} + for k, v in tracker_dict.items(): + v_int = int(v) + count_map[v_int] = count_map.get(v_int, 0) + 1 + + # 2️⃣ 替换成类型名称 + result = {} + for type_value, count in count_map.items(): + type_name = get_type_name_by_id(type_value) + result[type_name] = count + + other_info = result + + # 分析结果进行统计 得出 单帧最大蚕茧数量 + max_count_in_same_frame = 0 + for timestamp_in_video,json in statics.items(): + count_in_frame = sum(json.values()) + if count_in_frame > max_count_in_same_frame: + max_count_in_same_frame = count_in_frame + + # 保存至数据库 + video_id = insert_sca_video( + # 取后八位 + name=out_put_file_name.split('/')[-1].split('.')[0].replace('-','')[-8:], + raw_object_name=video_data.get("v_file_name", "").split('/')[-1], + ai_object_name=out_put_file_name.split('/')[-1], + duration=video_data.get("v_duration", 0), + size=video_data.get("v_size", 0), + video_codec=video_data.get("v_video_codec", ""), + audio_codec=video_data.get("v_audio_codec", ""), + overall_bit_rate=video_data.get("v_overall_bit_rate", 0), + resolution=video_data.get("v_resolution", ""), + sc_analysis_time= analysis_time, + sc_analysis_total_count=len(tracker_dict), + sc_analysis_max_count=max_count_in_same_frame, + sc_analysis_primary_type= primary_type, + sc_analysis_secondary_type= secondary_type, + other_info= other_info, + ) + + # 将细则存入数据库 + counter = Counter() + + for timestamp_in_video, json in statics.items(): + insert_sca_video_details(video_id,timestamp_in_video, json) \ No newline at end of file diff --git a/ai/video-sca/utils/pgDb.py b/ai/video-sca/utils/pgDb.py new file mode 100644 index 0000000..bac137f --- /dev/null +++ b/ai/video-sca/utils/pgDb.py @@ -0,0 +1,72 @@ +import logging +import time +from contextlib import contextmanager + +import psycopg +from psycopg_pool import ConnectionPool +from utils.GlobalVariable import LOCAL_IP + +logger = logging.getLogger("PGPool") +logger.setLevel(logging.INFO) + +class PGPool: + """ + PostgreSQL 连接池封装 + """ + + def __init__( + self, + uri: str, + min_size: int = 1, + max_size: int = 20, + max_idle: int = 30, + max_lifetime: int = 300, + timeout: int = 10, + check: bool = False, + ): + """ + :param uri: PostgreSQL 连接 URI + """ + self.uri = uri + self.pool = ConnectionPool( + self.uri, + min_size=min_size, + max_size=max_size, + max_idle=max_idle, + max_lifetime=max_lifetime, + timeout=timeout, + check=check, + ) + + @contextmanager + def getConn(self, retries: int = 2, delay: float = 1.0): + """ + 获取数据库连接，带重试机制，自动健康检查。 + 使用方式： + with pg_pool.get_conn() as conn: + with conn.cursor() as cur: + cur.execute(...) + """ + attempt = 0 + while attempt <= retries: + try: + with self.pool.connection() as conn: + conn.autocommit = True + yield conn + return + except psycopg.OperationalError as e: + logger.warning(f"数据库连接异常: {e}. 尝试重试 ({attempt+1}/{retries})") + self.pool.check() # 丢掉坏连接，重新建 + attempt += 1 + time.sleep(delay) + except Exception as e: + logger.error(f"SQL执行异常: {e}") + raise + raise psycopg.OperationalError("无法获取数据库连接，多次重试失败") + + +pg_pool = PGPool( + uri="postgresql://postgres:123456@" + LOCAL_IP + "/ktor2", + min_size=1, + max_size=20, +) diff --git a/ai/video-sca/utils/rabbitMQ.py b/ai/video-sca/utils/rabbitMQ.py new file mode 100644 index 0000000..51c911d --- /dev/null +++ b/ai/video-sca/utils/rabbitMQ.py @@ -0,0 +1,7 @@ +from utils.GlobalVariable import LOCAL_IP + +RABBIT_HOST = LOCAL_IP +RABBIT_VHOST = "bbit_ai" +RABBIT_USER = "ai_lab" +RABBIT_PASSWORD = "123456" +QUEUE_NAME = "analysis_queue"