{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 21, "status": "ok", "timestamp": 1787619221155, "user": { "displayName": "ming zhao", "userId": "11679614087766135097" }, "user_tz": -480 }, "id": "yO5a2QDwuwx3", "outputId": "a8282fc7-1c7d-4bec-e2fd-4603b5156cc8" }, "outputs": [ { "data": { "text/plain": [ "115" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# #获取需要的库 \n", "# %%writefile /content/requirements.txt\n", "# # transformers==5.7.0\n", "# accelerate\n", "# safetensors\n", "# pillow\n", "# numpy<2\n", "# torchvision\n", "# opencv-python-headless<4.12\n", "# codec-video-prep\n", "from pathlib import Path\n", "\n", "Path(\"/content/requirements.txt\").write_text(\n", "\"\"\"transformers==5.7.0\n", "accelerate\n", "safetensors\n", "pillow\n", "numpy<2\n", "torchvision\n", "opencv-python-headless<4.12\n", "codec-video-prep\n", "\"\"\",\n", "encoding=\"utf-8\"\n", ")" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Python 3.12.13\n" ] } ], "source": [ "!python --version" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 1820, "status": "ok", "timestamp": 1787619235573, "user": { "displayName": "ming zhao", "userId": "11679614087766135097" }, "user_tz": -480 }, "id": "_vF76oN3uk_e", "outputId": "7079eabd-43d2-403f-900e-7172736febe0" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cloning into '/content/mage-vl-demo'...\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "remote: Enumerating objects: 31, done.\u001b[K\n", "remote: Counting objects: 100% (27/27), done.\u001b[K\n", "remote: Compressing objects: 100% (25/25), done.\u001b[K\n", "remote: Total 31 (delta 8), reused 0 (delta 0), pack-reused 4 (from 1)\u001b[K\n", "Receiving objects: 100% (31/31), 22.13 KiB | 11.06 MiB/s, done.\n", "Resolving deltas: 100% (8/8), done.\n", "mamba_ssm 路径加载成功！\n" ] } ], "source": [ "# 1. 自动克隆官方 Demo 仓库（包含纯 PyTorch 版本的 mamba_ssm 文件夹）\n", "!git clone https://huggingface.co/spaces/microsoft/mage-vl-demo /content/mage-vl-demo\n", "\n", "# 2. 将包含 mamba_ssm 的目录加入 Python 模块搜索路径，确保 import mamba_ssm 永不报错\n", "import sys\n", "\n", "if \"/content/mage-vl-demo\" not in sys.path:\n", " sys.path.append(\"/content/mage-vl-demo\")\n", "\n", "print(\"mamba_ssm 路径加载成功！\")\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "executionInfo": { "elapsed": 233, "status": "ok", "timestamp": 1787619239776, "user": { "displayName": "ming zhao", "userId": "11679614087766135097" }, "user_tz": -480 }, "id": "LSKj4-71SO8H" }, "outputs": [], "source": [ "#移动目录\n", "!mv /content/mage-vl-demo/mamba_ssm /content/\n", "!mv /content/mage-vl-demo/examples/soccer-broadcast.mp4 /content/" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 581, "status": "ok", "timestamp": 1787619244608, "user": { "displayName": "ming zhao", "userId": "11679614087766135097" }, "user_tz": -480 }, "id": "IePDcWu59e1G", "outputId": "61107950-f0fa-4be8-a14f-e2bd8447046d" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2026-09-11 01:39:26-- http://172.245.243.238:8000/test02.mp4\n", "Connecting to 172.245.243.238:8000... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 30270635 (29M) [video/mp4]\n", "Saving to: ‘test02.mp4’\n", "\n", "test02.mp4 0%[ ] 0 --.-KB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "test02.mp4 100%[===================>] 28.87M 33.2MB/s in 0.9s \n", "\n", "2026-09-11 01:39:27 (33.2 MB/s) - ‘test02.mp4’ saved [30270635/30270635]\n", "\n", "--2026-09-11 01:39:27-- http://172.245.243.238:8000/test03.mp4\n", "Connecting to 172.245.243.238:8000... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 13549203 (13M) [video/mp4]\n", "Saving to: ‘test03.mp4’\n", "\n", "test03.mp4 100%[===================>] 12.92M 25.4MB/s in 0.5s \n", "\n", "2026-09-11 01:39:27 (25.4 MB/s) - ‘test03.mp4’ saved [13549203/13549203]\n", "\n", "--2026-09-11 01:39:27-- http://172.245.243.238:8000/test04.mp4\n", "Connecting to 172.245.243.238:8000... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 27143071 (26M) [video/mp4]\n", "Saving to: ‘test04.mp4’\n", "\n", "test04.mp4 100%[===================>] 25.88M 43.0MB/s in 0.6s \n", "\n", "2026-09-11 01:39:28 (43.0 MB/s) - ‘test04.mp4’ saved [27143071/27143071]\n", "\n", "--2026-09-11 01:39:28-- http://172.245.243.238:8000/test05.mp4\n", "Connecting to 172.245.243.238:8000... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 36972551 (35M) [video/mp4]\n", "Saving to: ‘test05.mp4’\n", "\n", "test05.mp4 100%[===================>] 35.26M 48.5MB/s in 0.7s \n", "\n", "2026-09-11 01:39:29 (48.5 MB/s) - ‘test05.mp4’ saved [36972551/36972551]\n", "\n" ] } ], "source": [ "# !wget http://172.245.243.238:8000/test.mp4\n", "!wget http://172.245.243.238:8000/test02.mp4\n", "!wget http://172.245.243.238:8000/test03.mp4\n", "!wget http://172.245.243.238:8000/test04.mp4\n", "!wget http://172.245.243.238:8000/test05.mp4" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 17882, "status": "ok", "timestamp": 1787619266366, "user": { "displayName": "ming zhao", "userId": "11679614087766135097" }, "user_tz": -480 }, "id": "BBNwlkbfiu89", "outputId": "cf3a98a7-400f-40fe-e130-b7223e411286" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Get:1 https://cli.github.com/packages stable InRelease [4,685 B]\n", "Get:2 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease [3,632 B]\n", "Get:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 InRelease [1,578 B]\n", "Get:4 https://cli.github.com/packages stable/main amd64 Packages [356 B] \n", "Get:5 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] \n", "Hit:6 http://archive.ubuntu.com/ubuntu jammy InRelease \n", "Get:7 https://r2u.stat.illinois.edu/ubuntu jammy InRelease [6,555 B] \n", "Get:8 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] \n", "Get:9 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ Packages [113 kB]\n", "Get:10 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 Packages [2,993 kB]\n", "Get:11 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease [18.1 kB]\n", "Get:12 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] \n", "Hit:13 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n", "Get:14 https://r2u.stat.illinois.edu/ubuntu jammy/main amd64 Packages [3,198 kB]\n", "Hit:15 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n", "Get:16 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1,317 kB]\n", "Get:17 https://r2u.stat.illinois.edu/ubuntu jammy/main all Packages [10.9 MB] \n", "Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [93.2 kB]\n", "Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [4,623 kB]\n", "Get:20 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy/main amd64 Packages [44.3 kB]\n", "Get:21 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [4,289 kB]\n", "Get:22 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [84.6 kB]\n", "Get:23 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [7,717 kB]\n", "Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,620 kB]\n", "Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [8,007 kB]\n", "Get:26 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [35.6 kB]\n", "Get:27 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [82.8 kB]\n", "Fetched 45.5 MB in 4s (10.5 MB/s)\n", "Reading package lists... Done\n", "W: Skipping acquire of configured file 'main/source/Sources' as repository 'https://r2u.stat.illinois.edu/ubuntu jammy InRelease' does not seem to provide it (sources.list entry misspelt?)\n", "Reading package lists... Done\n", "Building dependency tree... Done\n", "Reading state information... Done\n", "ffmpeg is already the newest version (7:4.4.2-0ubuntu0.22.04.1).\n", "0 upgraded, 0 newly installed, 0 to remove and 212 not upgraded.\n" ] } ], "source": [ "!apt-get update && apt-get install -y ffmpeg" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000 }, "executionInfo": { "elapsed": 379491, "status": "ok", "timestamp": 1787619653638, "user": { "displayName": "ming zhao", "userId": "11679614087766135097" }, "user_tz": -480 }, "id": "I3NLQ1Rvq5Ii", "outputId": "084d334b-66ef-4ee7-979c-5d3df5351c75" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting transformers==5.7.0 (from -r /content/requirements.txt (line 1))\n", " Downloading transformers-5.7.0-py3-none-any.whl.metadata (33 kB)\n", "Requirement already satisfied: accelerate in /usr/local/lib/python3.12/dist-packages (from -r /content/requirements.txt (line 2)) (1.14.0)\n", "Requirement already satisfied: safetensors in /usr/local/lib/python3.12/dist-packages (from -r /content/requirements.txt (line 3)) (0.8.0)\n", "Requirement already satisfied: pillow in /usr/local/lib/python3.12/dist-packages (from -r /content/requirements.txt (line 4)) (11.3.0)\n", "Collecting numpy<2 (from -r /content/requirements.txt (line 5))\n", " Downloading numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (61 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.0/61.0 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: torchvision in /usr/local/lib/python3.12/dist-packages (from -r /content/requirements.txt (line 6)) (0.26.0+cu128)\n", "Collecting opencv-python-headless<4.12 (from -r /content/requirements.txt (line 7))\n", " Downloading opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (20 kB)\n", "Collecting codec-video-prep (from -r /content/requirements.txt (line 8))\n", " Downloading codec_video_prep-0.2.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (11 kB)\n", "Requirement already satisfied: huggingface-hub<2.0,>=1.5.0 in /usr/local/lib/python3.12/dist-packages (from transformers==5.7.0->-r /content/requirements.txt (line 1)) (1.23.0)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.12/dist-packages (from transformers==5.7.0->-r /content/requirements.txt (line 1)) (26.2)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.12/dist-packages (from transformers==5.7.0->-r /content/requirements.txt (line 1)) (6.0.3)\n", "Requirement already satisfied: regex>=2025.10.22 in /usr/local/lib/python3.12/dist-packages (from transformers==5.7.0->-r /content/requirements.txt (line 1)) (2025.11.3)\n", "Requirement already satisfied: tokenizers<=0.23.0,>=0.22.0 in /usr/local/lib/python3.12/dist-packages (from transformers==5.7.0->-r /content/requirements.txt (line 1)) (0.22.2)\n", "Requirement already satisfied: typer in /usr/local/lib/python3.12/dist-packages (from transformers==5.7.0->-r /content/requirements.txt (line 1)) (0.26.8)\n", "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.12/dist-packages (from transformers==5.7.0->-r /content/requirements.txt (line 1)) (4.67.3)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.12/dist-packages (from accelerate->-r /content/requirements.txt (line 2)) (5.9.5)\n", "Requirement already satisfied: torch>=2.0.0 in /usr/local/lib/python3.12/dist-packages (from accelerate->-r /content/requirements.txt (line 2)) (2.11.0+cu128)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.12/dist-packages (from torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (3.29.7)\n", "Requirement already satisfied: typing-extensions>=4.10.0 in /usr/local/lib/python3.12/dist-packages (from torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (4.16.0)\n", "Requirement already satisfied: setuptools<82 in /usr/local/lib/python3.12/dist-packages (from torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (75.2.0)\n", "Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (1.14.0)\n", "Requirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.12/dist-packages (from torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (3.6.1)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (3.1.6)\n", "Requirement already satisfied: fsspec>=0.8.5 in /usr/local/lib/python3.12/dist-packages (from torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (2025.3.0)\n", "Requirement already satisfied: cuda-toolkit==12.8.1 in /usr/local/lib/python3.12/dist-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==12.8.1; platform_system == \"Linux\"->torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (12.8.1)\n", "Requirement already satisfied: cuda-bindings<13,>=12.9.4 in /usr/local/lib/python3.12/dist-packages (from torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (12.9.7)\n", "Requirement already satisfied: nvidia-cudnn-cu12==9.19.0.56 in /usr/local/lib/python3.12/dist-packages (from torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (9.19.0.56)\n", "Requirement already satisfied: nvidia-cusparselt-cu12==0.7.1 in /usr/local/lib/python3.12/dist-packages (from torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (0.7.1)\n", "Requirement already satisfied: nvidia-nccl-cu12==2.28.9 in /usr/local/lib/python3.12/dist-packages (from torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (2.28.9)\n", "Requirement already satisfied: nvidia-nvshmem-cu12==3.4.5 in /usr/local/lib/python3.12/dist-packages (from torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (3.4.5)\n", "Requirement already satisfied: triton==3.6.0 in /usr/local/lib/python3.12/dist-packages (from torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (3.6.0)\n", "Requirement already satisfied: nvidia-cublas-cu12==12.8.4.1.* in /usr/local/lib/python3.12/dist-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==12.8.1; platform_system == \"Linux\"->torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (12.8.4.1)\n", "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.8.90.* in /usr/local/lib/python3.12/dist-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==12.8.1; platform_system == \"Linux\"->torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (12.8.90)\n", "Requirement already satisfied: nvidia-cufft-cu12==11.3.3.83.* in /usr/local/lib/python3.12/dist-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==12.8.1; platform_system == \"Linux\"->torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (11.3.3.83)\n", "Requirement already satisfied: nvidia-cufile-cu12==1.13.1.3.* in /usr/local/lib/python3.12/dist-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==12.8.1; platform_system == \"Linux\"->torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (1.13.1.3)\n", "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.8.90.* in /usr/local/lib/python3.12/dist-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==12.8.1; platform_system == \"Linux\"->torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (12.8.90)\n", "Requirement already satisfied: nvidia-curand-cu12==10.3.9.90.* in /usr/local/lib/python3.12/dist-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==12.8.1; platform_system == \"Linux\"->torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (10.3.9.90)\n", "Requirement already satisfied: nvidia-cusolver-cu12==11.7.3.90.* in /usr/local/lib/python3.12/dist-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==12.8.1; platform_system == \"Linux\"->torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (11.7.3.90)\n", "Requirement already satisfied: nvidia-cusparse-cu12==12.5.8.93.* in /usr/local/lib/python3.12/dist-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==12.8.1; platform_system == \"Linux\"->torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (12.5.8.93)\n", "Requirement already satisfied: nvidia-nvjitlink-cu12==12.8.93.* in /usr/local/lib/python3.12/dist-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==12.8.1; platform_system == \"Linux\"->torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (12.8.93)\n", "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.8.93.* in /usr/local/lib/python3.12/dist-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==12.8.1; platform_system == \"Linux\"->torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (12.8.93)\n", "Requirement already satisfied: nvidia-nvtx-cu12==12.8.90.* in /usr/local/lib/python3.12/dist-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==12.8.1; platform_system == \"Linux\"->torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (12.8.90)\n", "Requirement already satisfied: click<9.0.0,>=8.4.2 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub<2.0,>=1.5.0->transformers==5.7.0->-r /content/requirements.txt (line 1)) (8.4.2)\n", "Requirement already satisfied: hf-xet<2.0.0,>=1.5.1 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub<2.0,>=1.5.0->transformers==5.7.0->-r /content/requirements.txt (line 1)) (1.5.1)\n", "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub<2.0,>=1.5.0->transformers==5.7.0->-r /content/requirements.txt (line 1)) (0.28.1)\n", "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.12/dist-packages (from typer->transformers==5.7.0->-r /content/requirements.txt (line 1)) (1.5.4)\n", "Requirement already satisfied: rich>=13.8.0 in /usr/local/lib/python3.12/dist-packages (from typer->transformers==5.7.0->-r /content/requirements.txt (line 1)) (13.9.4)\n", "Requirement already satisfied: annotated-doc>=0.0.2 in /usr/local/lib/python3.12/dist-packages (from typer->transformers==5.7.0->-r /content/requirements.txt (line 1)) (0.0.4)\n", "Requirement already satisfied: cuda-pathfinder~=1.1 in /usr/local/lib/python3.12/dist-packages (from cuda-bindings<13,>=12.9.4->torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (1.5.6)\n", "Requirement already satisfied: anyio in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers==5.7.0->-r /content/requirements.txt (line 1)) (4.14.2)\n", "Requirement already satisfied: certifi in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers==5.7.0->-r /content/requirements.txt (line 1)) (2026.6.17)\n", "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers==5.7.0->-r /content/requirements.txt (line 1)) (1.0.9)\n", "Requirement already satisfied: idna in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers==5.7.0->-r /content/requirements.txt (line 1)) (3.18)\n", "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers==5.7.0->-r /content/requirements.txt (line 1)) (0.16.0)\n", "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich>=13.8.0->typer->transformers==5.7.0->-r /content/requirements.txt (line 1)) (4.2.0)\n", "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich>=13.8.0->typer->transformers==5.7.0->-r /content/requirements.txt (line 1)) (2.20.0)\n", "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (1.3.0)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->torch>=2.0.0->accelerate->-r /content/requirements.txt (line 2)) (3.0.3)\n", "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich>=13.8.0->typer->transformers==5.7.0->-r /content/requirements.txt (line 1)) (0.1.2)\n", "Downloading transformers-5.7.0-py3-none-any.whl (10.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.5/10.5 MB\u001b[0m \u001b[31m104.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m:01\u001b[0m\n", "\u001b[?25hDownloading numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (18.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m18.0/18.0 MB\u001b[0m \u001b[31m109.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (50.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.0/50.0 MB\u001b[0m \u001b[31m21.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading codec_video_prep-0.2.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (29.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m29.8/29.8 MB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hInstalling collected packages: numpy, opencv-python-headless, codec-video-prep, transformers\n", " Attempting uninstall: numpy\n", " Found existing installation: numpy 2.0.2\n", " Uninstalling numpy-2.0.2:\n", " Successfully uninstalled numpy-2.0.2\n", " Attempting uninstall: opencv-python-headless\n", " Found existing installation: opencv-python-headless 5.0.0.93\n", " Uninstalling opencv-python-headless-5.0.0.93:\n", " Successfully uninstalled opencv-python-headless-5.0.0.93\n", " Attempting uninstall: transformers\n", " Found existing installation: transformers 5.13.1\n", " Uninstalling transformers-5.13.1:\n", " Successfully uninstalled transformers-5.13.1\n", "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", "cupy-cuda12x 14.0.1 requires numpy<2.6,>=2.0, but you have numpy 1.26.4 which is incompatible.\n", "jax 0.7.2 requires numpy>=2.0, but you have numpy 1.26.4 which is incompatible.\n", "jaxlib 0.7.2 requires numpy>=2.0, but you have numpy 1.26.4 which is incompatible.\n", "opencv-contrib-python 4.13.0.92 requires numpy>=2; python_version >= \"3.9\", but you have numpy 1.26.4 which is incompatible.\n", "opencv-python 5.0.0.93 requires numpy>=2; python_version >= \"3.9\", but you have numpy 1.26.4 which is incompatible.\n", "pytensor 2.38.3 requires numpy>=2.0, but you have numpy 1.26.4 which is incompatible.\n", "rasterio 1.5.0 requires numpy>=2, but you have numpy 1.26.4 which is incompatible.\n", "shap 0.52.0 requires numpy>=2, but you have numpy 1.26.4 which is incompatible.\n", "tifffile 2026.4.11 requires numpy>=2.0, but you have numpy 1.26.4 which is incompatible.\n", "tobler 0.14.0 requires numpy>=2.0, but you have numpy 1.26.4 which is incompatible.\n", "xarray-einstats 0.10.0 requires numpy>=2.0, but you have numpy 1.26.4 which is incompatible.\u001b[0m\u001b[31m\n", "\u001b[0mSuccessfully installed codec-video-prep-0.2.5 numpy-1.26.4 opencv-python-headless-4.11.0.86 transformers-5.7.0\n" ] }, { "data": { "application/vnd.colab-display-data+json": { "id": "4d53f1dc8a6940d2aa2803bbecf37f7e", "pip_warning": { "packages": [ "numpy" ] } } }, "metadata": {}, "output_type": "display_data" } ], "source": [ "!pip install -r /content/requirements.txt" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "from huggingface_hub import login\n", "\n", "login(token=\"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\") " ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "============================================================\n", "正在加载模型（只加载一次）...\n", "============================================================\n", "[GPU] Tesla T4\n", "[CUDA] 12.8\n", "[BF16] True\n", "[MODEL DTYPE] torch.bfloat16\n", "\n", "[init] 加载 Processor...\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "ea0d8e9332a043c893d7d2317ba419cc", "version_major": 2, "version_minor": 0 }, "text/plain": [ "preprocessor_config.json: 0%| | 0.00/1.81k [00:00 4:\n", " print(f\"[warn] 数量偏差较大，强制纠正: 模型={current_count} → 期望={expected_count}\")\n", " current_count = expected_count\n", "\n", " colors = normalize_colors(data.get(\"colors\", previous_colors))\n", " if not colors:\n", " colors = previous_colors\n", "\n", " try:\n", " occupancy = int(round(float(data.get(\"occupancy\", previous_occupancy))))\n", " except:\n", " occupancy = previous_occupancy\n", " occupancy = max(0, min(100, occupancy))\n", "\n", " try:\n", " confidence = float(data.get(\"confidence\", 0.0))\n", " except:\n", " confidence = 0.0\n", " confidence = max(0.0, min(1.0, confidence))\n", "\n", " print(f\"[debug] 原始 delta={data.get('delta')} | 原始 current={data.get('current_count')} → 最终 delta={delta}, count={current_count}\")\n", "\n", " return {\n", " \"delta\": delta,\n", " \"current_count\": current_count,\n", " \"colors\": colors,\n", " \"occupancy\": occupancy,\n", " \"confidence\": confidence\n", " }\n", "\n", "def state_to_text(state):\n", " d = state[\"delta\"]\n", " if d > 0: change = f\"放入 {d} 件\"\n", " elif d < 0: change = f\"取出 {abs(d)} 件\"\n", " else: change = \"数量无变化\"\n", " colors = \"、\".join(state[\"colors\"]) if state[\"colors\"] else \"未知\"\n", " return f\"{change} | 当前 {state['current_count']} 件 | 颜色={colors} | 占用={state['occupancy']}% | 置信度={state['confidence']:.2f}\"\n", "\n", "@torch.inference_mode()\n", "def final_summary(history, final_state, final_max_tokens=128):\n", " if not history:\n", " return \"没有获得有效的衣物分析结果。\"\n", "\n", " prompt = FINAL_PROMPT.format(history=\"\\n\".join(history))\n", " prompt += f\"\"\"\n", "程序计算出的最终状态：\n", "数量：{final_state[\"current_count\"]} 件\n", "颜色：{\", \".join(final_state[\"colors\"]) if final_state[\"colors\"] else \"未知\"}\n", "空间占用：{final_state[\"occupancy\"]}%\n", "请以这个程序最终状态为主要依据。\n", "\"\"\"\n", " inputs = processor.tokenizer(prompt, return_tensors=\"pt\", truncation=True, max_length=4096)\n", " inputs = {k: v.to(DEVICE) for k, v in inputs.items()}\n", " out = None\n", " try:\n", " out = model.generate(**inputs, max_new_tokens=final_max_tokens, do_sample=False, use_cache=True)\n", " n = inputs[\"input_ids\"].shape[1]\n", " return processor.tokenizer.decode(out[0, n:], skip_special_tokens=True).strip()\n", " finally:\n", " cleanup(inputs)\n", " try: del out\n", " except: pass\n", "\n", "# ------------------------------\n", "# 4. 核心分析函数（完全参数化）\n", "# ------------------------------\n", "def run_analysis(\n", " video_path=None,\n", " prompt_template=None,\n", " segment_sec=None,\n", " max_segments=None,\n", " gate_threshold=None,\n", " max_new_tokens=None,\n", " simulate_realtime=None,\n", " show_raw=None,\n", " max_change_per_segment=None,\n", " always_run_llm=None\n", "):\n", " \"\"\"\n", " 可反复调用的主函数。\n", " 所有参数都有默认值，也可在调用时覆盖。\n", " \"\"\"\n", " video_path = video_path or DEFAULT_VIDEO\n", " prompt_template = prompt_template or PROMPT_TEMPLATE\n", " segment_sec = segment_sec if segment_sec is not None else DEFAULT_SEGMENT_SEC\n", " max_segments = max_segments if max_segments is not None else DEFAULT_MAX_SEGMENTS\n", " gate_threshold = gate_threshold if gate_threshold is not None else DEFAULT_GATE_THRESHOLD\n", " max_new_tokens = max_new_tokens if max_new_tokens is not None else DEFAULT_MAX_NEW_TOKENS\n", " simulate_realtime = simulate_realtime if simulate_realtime is not None else DEFAULT_SIMULATE_REALTIME\n", " show_raw = show_raw if show_raw is not None else DEFAULT_SHOW_RAW\n", " max_change = max_change_per_segment if max_change_per_segment is not None else DEFAULT_MAX_CHANGE_PER_SEGMENT\n", " always_run_llm = always_run_llm if always_run_llm is not None else DEFAULT_ALWAYS_RUN_LLM\n", "\n", " path = Path(video_path)\n", " if not path.exists():\n", " raise FileNotFoundError(f\"视频不存在: {video_path}\")\n", "\n", " total = duration(path)\n", " if total <= 0:\n", " raise RuntimeError(\"无法获取视频时长\")\n", "\n", " clip_dir = Path(os.environ[\"STREAMMIND_CLIP_CACHE\"])\n", " clip_dir.mkdir(parents=True, exist_ok=True)\n", "\n", " previous_count = 0\n", " previous_colors = []\n", " previous_occupancy = 0\n", " history = []\n", " results = []\n", " start = 0.0\n", " index = 0\n", " _, codec_override = get_codec_config(DEFAULT_CODEC_MAX_PIXELS, DEFAULT_CODEC_TARGET_CANVAS)\n", "\n", " print(f\"\\n开始分析 | 视频={path.name} | 时长={total:.1f}s | 窗口={segment_sec}s\")\n", " print(\"-\" * 60)\n", "\n", " while start < total - 0.05:\n", " if max_segments is not None and index >= max_segments:\n", " break\n", "\n", " end = min(total, start + segment_sec)\n", " length = end - start\n", " index += 1\n", " t0 = time.time()\n", "\n", " clip = clip_dir / f\"{path.stem}_{index:06d}_{start:.2f}_{end:.2f}.mp4\"\n", " inputs = None\n", " cuda_inputs = None\n", " said = None\n", "\n", " try:\n", " prompt = make_prompt(prompt_template, previous_count, previous_colors, previous_occupancy)\n", " extract_clip(path, start, length, clip)\n", " inputs, prob = analyze_clip(clip, prompt, codec_override, DEFAULT_CODEC_MAX_PIXELS)\n", "\n", " head = f\"**{start:.1f}–{end:.1f}s** Gate=`{prob:.2f}`\"\n", "\n", " # 即使 Gate 较低，也建议跑 LLM（因为你反馈 Gate 高时仍漏检）\n", " if not always_run_llm and prob < gate_threshold:\n", " line = f\"{head} → 🔇 无明显变化\"\n", " results.append(line)\n", " print(line)\n", " continue\n", "\n", " print(f\"{head} → 🧠 识别中...\")\n", " gt = time.time()\n", "\n", " cuda_inputs = to_cuda(inputs)\n", " said = generate(cuda_inputs, max_new_tokens=max_new_tokens)\n", " gen_time = time.time() - gt\n", "\n", " if show_raw:\n", " print(f\"[raw LLM] {said}\")\n", "\n", " raw_data = parse_json(said)\n", " state = normalize_state(raw_data, previous_count, previous_colors, previous_occupancy, max_change)\n", "\n", " previous_count = state[\"current_count\"]\n", " previous_colors = state[\"colors\"]\n", " previous_occupancy = state[\"occupancy\"]\n", "\n", " change = state[\"delta\"]\n", " if change > 0:\n", " change_text = f\"🟢 放入 {change} 件\"\n", " elif change < 0:\n", " change_text = f\"🔴 取出 {abs(change)} 件\"\n", " else:\n", " change_text = \"⚪ 数量无变化\"\n", "\n", " colors_text = \"、\".join(state[\"colors\"]) if state[\"colors\"] else \"未知\"\n", " elapsed = time.time() - t0\n", " speed = length / max(elapsed, 0.001)\n", "\n", " line = (\n", " f\"{head}\\n\"\n", " f\" {change_text} | 当前数量={state['current_count']}件\\n\"\n", " f\" 颜色={colors_text} | 占用={state['occupancy']}%\\n\"\n", " f\" 置信度={state['confidence']:.2f} | \"\n", " f\"Generate={gen_time:.2f}s | Total={elapsed:.2f}s | {speed:.2f}×\"\n", " )\n", " results.append(line)\n", " history.append(f\"{start:.1f}-{end:.1f}s：{state_to_text(state)}\")\n", " print(line)\n", "\n", " if simulate_realtime and elapsed < length:\n", " time.sleep(length - elapsed)\n", "\n", " except Exception as e:\n", " line = f\"**{start:.1f}–{end:.1f}s** → ❌ {type(e).__name__}: {e}\"\n", " results.append(line)\n", " print(line)\n", "\n", " finally:\n", " cleanup(cuda_inputs, inputs, said)\n", " if clip.exists():\n", " try: clip.unlink()\n", " except: pass\n", " start = end\n", " print(f\"[segment {index}] {start:.1f}/{total:.1f}s | {gpu_mem()}\", flush=True)\n", " try: torch.cuda.reset_peak_memory_stats()\n", " except: pass\n", "\n", " # 最终结果\n", " final_state = {\n", " \"current_count\": previous_count,\n", " \"colors\": previous_colors,\n", " \"occupancy\": previous_occupancy\n", " }\n", "\n", " colors_text = \"、\".join(previous_colors) if previous_colors else \"未知\"\n", " if previous_count == 0:\n", " occupancy_state = \"空\"\n", " elif previous_occupancy <= 20:\n", " occupancy_state = \"少量\"\n", " elif previous_occupancy <= 50:\n", " occupancy_state = \"中等\"\n", " elif previous_occupancy <= 80:\n", " occupancy_state = \"较多\"\n", " else:\n", " occupancy_state = \"接近装满\"\n", "\n", " print(\"\\n\" + \"=\" * 60)\n", " print(\"🧺 最终衣物状态\")\n", " print(\"=\" * 60)\n", " print(f\"【最终数量】 {previous_count} 件\")\n", " print(f\"【最终颜色】 {colors_text}\")\n", " print(f\"【空间占用】 {previous_occupancy}%\")\n", " print(f\"【状态】 {occupancy_state}\")\n", " print(\"=\" * 60)\n", "\n", " try:\n", " final = final_summary(history, final_state)\n", " print(\"\\n🧠 最终模型校验：\")\n", " print(final)\n", " except Exception as e:\n", " print(f\"\\n❌ 最终模型校验失败: {type(e).__name__}: {e}\")\n", "\n", " print(f\"\\n✅ 完成，共分析 {index} 个片段\")\n", " return {\n", " \"final_count\": previous_count,\n", " \"final_colors\": previous_colors,\n", " \"final_occupancy\": previous_occupancy,\n", " \"history\": history,\n", " \"results\": results\n", " }" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "开始分析 | 视频=test02.mp4 | 时长=14.1s | 窗口=1.0s\n", "------------------------------------------------------------\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[transformers] Setting `pad_token_id` to `eos_token_id`:151645 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "**0.0–1.0s** Gate=`0.79` → 🧠 识别中...\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[raw LLM] {\n", " \"delta\": 1,\n", " \"current_count\": 1,\n", " \"colors\": [\"灰色\"],\n", " \"occupancy\": 20,\n", " \"confidence\": 0.9\n", "}\n", "[debug] 原始 delta=1 | 原始 current=1 → 最终 delta=1, count=1\n", "**0.0–1.0s** Gate=`0.79`\n", " 🟢 放入 1 件 | 当前数量=1件\n", " 颜色=灰色 | 占用=20%\n", " 置信度=0.90 | Generate=6.01s | Total=8.80s | 0.11×\n", "[segment 1] 1.0/14.1s | GPU=9.84GB / reserved=9.87GB / peak=10.26GB\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[transformers] Setting `pad_token_id` to `eos_token_id`:151645 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "**1.0–2.0s** Gate=`0.40` → 🧠 识别中...\n", "[raw LLM] {\n", " \"delta\": 0,\n", " \"current_count\": 1,\n", " \"colors\": [\"灰色\"],\n", " \"occupancy\": 20,\n", " \"confidence\": 1.0\n", "}\n", "[debug] 原始 delta=0 | 原始 current=1 → 最终 delta=0, count=1\n", "**1.0–2.0s** Gate=`0.40`\n", " ⚪ 数量无变化 | 当前数量=1件\n", " 颜色=灰色 | 占用=20%\n", " 置信度=1.00 | Generate=6.02s | Total=9.37s | 0.11×\n", "[segment 2] 2.0/14.1s | GPU=9.84GB / reserved=9.87GB / peak=10.27GB\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[transformers] Setting `pad_token_id` to `eos_token_id`:151645 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "**2.0–3.0s** Gate=`0.59` → 🧠 识别中...\n", "[raw LLM] {\n", " \"delta\": 0,\n", " \"current_count\": 1,\n", " \"colors\": [\"灰色\"],\n", " \"occupancy\": 20,\n", " \"confidence\": 1.0\n", "}\n", "[debug] 原始 delta=0 | 原始 current=1 → 最终 delta=0, count=1\n", "**2.0–3.0s** Gate=`0.59`\n", " ⚪ 数量无变化 | 当前数量=1件\n", " 颜色=灰色 | 占用=20%\n", " 置信度=1.00 | Generate=6.08s | Total=9.46s | 0.11×\n", "[segment 3] 3.0/14.1s | GPU=9.84GB / reserved=9.87GB / peak=10.27GB\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[transformers] Setting `pad_token_id` to `eos_token_id`:151645 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "**3.0–4.0s** Gate=`0.76` → 🧠 识别中...\n", "[raw LLM] {\n", " \"delta\": 0,\n", " \"current_count\": 1,\n", " \"colors\": [\"灰色\", \"黑白格\"],\n", " \"occupancy\": 20,\n", " \"confidence\": 0.9\n", "}\n", "[debug] 原始 delta=0 | 原始 current=1 → 最终 delta=0, count=1\n", "**3.0–4.0s** Gate=`0.76`\n", " ⚪ 数量无变化 | 当前数量=1件\n", " 颜色=灰色 | 占用=20%\n", " 置信度=0.90 | Generate=6.31s | Total=9.06s | 0.11×\n", "[segment 4] 4.0/14.1s | GPU=9.84GB / reserved=9.87GB / peak=10.22GB\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[transformers] Setting `pad_token_id` to `eos_token_id`:151645 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "**4.0–5.0s** Gate=`0.66` → 🧠 识别中...\n", "[raw LLM] {\n", " \"delta\": 0,\n", " \"current_count\": 1,\n", " \"colors\": [\"灰色\"],\n", " \"occupancy\": 20,\n", " \"confidence\": 1.0\n", "}\n", "[debug] 原始 delta=0 | 原始 current=1 → 最终 delta=0, count=1\n", "**4.0–5.0s** Gate=`0.66`\n", " ⚪ 数量无变化 | 当前数量=1件\n", " 颜色=灰色 | 占用=20%\n", " 置信度=1.00 | Generate=6.21s | Total=9.22s | 0.11×\n", "[segment 5] 5.0/14.1s | GPU=9.84GB / reserved=9.87GB / peak=10.25GB\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[transformers] Setting `pad_token_id` to `eos_token_id`:151645 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "**5.0–6.0s** Gate=`0.55` → 🧠 识别中...\n", "[raw LLM] {\n", " \"delta\": 0,\n", " \"current_count\": 1,\n", " \"colors\": [\"灰色\"],\n", " \"occupancy\": 20,\n", " \"confidence\": 1.0\n", "}\n", "[debug] 原始 delta=0 | 原始 current=1 → 最终 delta=0, count=1\n", "**5.0–6.0s** Gate=`0.55`\n", " ⚪ 数量无变化 | 当前数量=1件\n", " 颜色=灰色 | 占用=20%\n", " 置信度=1.00 | Generate=5.55s | Total=8.85s | 0.11×\n", "[segment 6] 6.0/14.1s | GPU=9.84GB / reserved=9.87GB / peak=10.20GB\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[transformers] Setting `pad_token_id` to `eos_token_id`:151645 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "**6.0–7.0s** Gate=`0.81` → 🧠 识别中...\n", "[raw LLM] {\n", " \"delta\": 0,\n", " \"current_count\": 1,\n", " \"colors\": [\"灰色\"],\n", " \"occupancy\": 20,\n", " \"confidence\": 1.0\n", "}\n", "[debug] 原始 delta=0 | 原始 current=1 → 最终 delta=0, count=1\n", "**6.0–7.0s** Gate=`0.81`\n", " ⚪ 数量无变化 | 当前数量=1件\n", " 颜色=灰色 | 占用=20%\n", " 置信度=1.00 | Generate=5.85s | Total=9.37s | 0.11×\n", "[segment 7] 7.0/14.1s | GPU=9.84GB / reserved=9.87GB / peak=10.26GB\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[transformers] Setting `pad_token_id` to `eos_token_id`:151645 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "**7.0–8.0s** Gate=`0.69` → 🧠 识别中...\n", "[raw LLM] {\n", " \"delta\": 1,\n", " \"current_count\": 2,\n", " \"colors\": [\"灰色\", \"白色\"],\n", " \"occupancy\": 30,\n", " \"confidence\": 0.9\n", "}\n", "[debug] 原始 delta=1 | 原始 current=2 → 最终 delta=1, count=2\n", "**7.0–8.0s** Gate=`0.69`\n", " 🟢 放入 1 件 | 当前数量=2件\n", " 颜色=灰色、白色 | 占用=30%\n", " 置信度=0.90 | Generate=5.95s | Total=9.07s | 0.11×\n", "[segment 8] 8.0/14.1s | GPU=9.84GB / reserved=9.87GB / peak=10.22GB\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[transformers] Setting `pad_token_id` to `eos_token_id`:151645 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "**8.0–9.0s** Gate=`0.66` → 🧠 识别中...\n", "[raw LLM] {\n", " \"delta\": 0,\n", " \"current_count\": 2,\n", " \"colors\": [\"灰色\", \"白色\"],\n", " \"occupancy\": 30,\n", " \"confidence\": 1.0\n", "}\n", "[debug] 原始 delta=0 | 原始 current=2 → 最终 delta=0, count=2\n", "**8.0–9.0s** Gate=`0.66`\n", " ⚪ 数量无变化 | 当前数量=2件\n", " 颜色=灰色、白色 | 占用=30%\n", " 置信度=1.00 | Generate=6.37s | Total=9.49s | 0.11×\n", "[segment 9] 9.0/14.1s | GPU=9.84GB / reserved=9.87GB / peak=10.26GB\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[transformers] Setting `pad_token_id` to `eos_token_id`:151645 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "**9.0–10.0s** Gate=`0.74` → 🧠 识别中...\n", "[raw LLM] {\n", " \"delta\": 0,\n", " \"current_count\": 2,\n", " \"colors\": [\"灰色\", \"白色\"],\n", " \"occupancy\": 30,\n", " \"confidence\": 1.0\n", "}\n", "[debug] 原始 delta=0 | 原始 current=2 → 最终 delta=0, count=2\n", "**9.0–10.0s** Gate=`0.74`\n", " ⚪ 数量无变化 | 当前数量=2件\n", " 颜色=灰色、白色 | 占用=30%\n", " 置信度=1.00 | Generate=6.23s | Total=9.22s | 0.11×\n", "[segment 10] 10.0/14.1s | GPU=9.84GB / reserved=9.87GB / peak=10.27GB\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[transformers] Setting `pad_token_id` to `eos_token_id`:151645 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "**10.0–11.0s** Gate=`0.57` → 🧠 识别中...\n", "[raw LLM] {\n", " \"delta\": 0,\n", " \"current_count\": 2,\n", " \"colors\": [\"灰色\", \"白色\"],\n", " \"occupancy\": 30,\n", " \"confidence\": 0.9\n", "}\n", "[debug] 原始 delta=0 | 原始 current=2 → 最终 delta=0, count=2\n", "**10.0–11.0s** Gate=`0.57`\n", " ⚪ 数量无变化 | 当前数量=2件\n", " 颜色=灰色、白色 | 占用=30%\n", " 置信度=0.90 | Generate=5.69s | Total=9.61s | 0.10×\n", "[segment 11] 11.0/14.1s | GPU=9.84GB / reserved=9.87GB / peak=10.18GB\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[transformers] Setting `pad_token_id` to `eos_token_id`:151645 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "**11.0–12.0s** Gate=`0.40` → 🧠 识别中...\n", "[raw LLM] {\n", " \"delta\": 0,\n", " \"current_count\": 2,\n", " \"colors\": [\"灰色\", \"白色\"],\n", " \"occupancy\": 30,\n", " \"confidence\": 1.0\n", "}\n", "[debug] 原始 delta=0 | 原始 current=2 → 最终 delta=0, count=2\n", "**11.0–12.0s** Gate=`0.40`\n", " ⚪ 数量无变化 | 当前数量=2件\n", " 颜色=灰色、白色 | 占用=30%\n", " 置信度=1.00 | Generate=6.07s | Total=9.08s | 0.11×\n", "[segment 12] 12.0/14.1s | GPU=9.84GB / reserved=9.87GB / peak=10.26GB\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[transformers] Setting `pad_token_id` to `eos_token_id`:151645 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "**12.0–13.0s** Gate=`0.19` → 🧠 识别中...\n", "[raw LLM] {\n", " \"delta\": 0,\n", " \"current_count\": 2,\n", " \"colors\": [\"灰色\", \"白色\"],\n", " \"occupancy\": 30,\n", " \"confidence\": 0.9\n", "}\n", "[debug] 原始 delta=0 | 原始 current=2 → 最终 delta=0, count=2\n", "**12.0–13.0s** Gate=`0.19`\n", " ⚪ 数量无变化 | 当前数量=2件\n", " 颜色=灰色、白色 | 占用=30%\n", " 置信度=0.90 | Generate=6.51s | Total=9.51s | 0.11×\n", "[segment 13] 13.0/14.1s | GPU=9.84GB / reserved=9.87GB / peak=10.24GB\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[transformers] Setting `pad_token_id` to `eos_token_id`:151645 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "**13.0–14.0s** Gate=`0.19` → 🧠 识别中...\n", "[raw LLM] {\n", " \"delta\": 0,\n", " \"current_count\": 2,\n", " \"colors\": [\"灰色\", \"白色\"],\n", " \"occupancy\": 30,\n", " \"confidence\": 0.9\n", "}\n", "[debug] 原始 delta=0 | 原始 current=2 → 最终 delta=0, count=2\n", "**13.0–14.0s** Gate=`0.19`\n", " ⚪ 数量无变化 | 当前数量=2件\n", " 颜色=灰色、白色 | 占用=30%\n", " 置信度=0.90 | Generate=6.00s | Total=8.93s | 0.11×\n", "[segment 14] 14.0/14.1s | GPU=9.84GB / reserved=9.87GB / peak=10.24GB\n", "**14.0–14.1s** → ❌ RuntimeError: online codec failed rc=1: Traceback (most recent call last):\n", " File \"/usr/local/bin/cv-preinfer\", line 8, in \n", " sys.exit(main())\n", " ^^^^^^\n", " File \"/usr/local/lib/python3.12/dist-packages/codec_video_prep/cli.py\", line 85, in main\n", " result = run_preinfer(**kwargs)\n", " ^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/usr/local/lib/python3.12/dist-packages/codec_video_prep/api.py\", line 105, in run_preinfer\n", " return run_preinfer_config(config)\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/usr/local/lib/python3.12/dist-packages/codec_video_prep/api.py\", line 146, in run_preinfer_config\n", " result = run_bitcost_readiness(cfg)\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/usr/local/lib/python3.12/dist-packages/codec_selector/core/pipeline.py\", line 275, in run_bitcost_readiness\n", " raise RuntimeError(\"cannot read video metadata with ffprobe\")\n", "RuntimeError: cannot read video metadata with ffprobe\n", "\n", "[segment 15] 14.1/14.1s | GPU=9.84GB / reserved=9.87GB / peak=9.84GB\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[transformers] Setting `pad_token_id` to `eos_token_id`:151645 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "============================================================\n", "🧺 最终衣物状态\n", "============================================================\n", "【最终数量】 2 件\n", "【最终颜色】 灰色、白色\n", "【空间占用】 30%\n", "【状态】 中等\n", "============================================================\n", "\n", "🧠 最终模型校验：\n", "最终状态：中等\n", "\n", "最终数量\n", "2 件\n", "\n", "最终颜色\n", "灰色, 白色\n", "\n", "空间占用\n", "30%\n", "\n", "状态\n", "中等\n", "\n", "✅ 完成，共分析 15 个片段\n" ] } ], "source": [ "#调用\n", "# ------------------------------\n", "# 1. 默认参数（可在调用时覆盖）\n", "# ------------------------------\n", "DEFAULT_VIDEO = \"/content/test02.mp4\"\n", "DEFAULT_SEGMENT_SEC = 1.0\n", "DEFAULT_MAX_SEGMENTS = None\n", "DEFAULT_GATE_THRESHOLD = 0.45 # 稍微降低阈值\n", "DEFAULT_MAX_NEW_TOKENS = 128\n", "DEFAULT_FINAL_MAX_NEW_TOKENS = 128\n", "DEFAULT_CODEC_MAX_PIXELS = 48000\n", "DEFAULT_CODEC_TARGET_CANVAS = 20\n", "DEFAULT_SIMULATE_REALTIME = False\n", "DEFAULT_MAX_CHANGE_PER_SEGMENT = 8 # 允许一个片段内变化更多\n", "DEFAULT_ALWAYS_RUN_LLM = True\n", "DEFAULT_SHOW_RAW = True\n", "\n", "# ------------------------------\n", "# 2. 优化后的 Prompt（针对“只能识别1件”问题加强）\n", "# ------------------------------\n", "PROMPT_TEMPLATE = \"\"\"\n", "你是一个非常严格的洗衣机滚筒衣物数量变化检测器。\n", "\n", "你的唯一任务是准确判断：这个时间窗口内，有没有衣物被【放入】或【取出】滚筒。\n", "尤其要认真检查是否有衣物取出，分析视频取出衣物并计数。\n", "\n", "==================================================\n", "核心规则（必须严格遵守）\n", "==================================================\n", "1. 只要看到有衣物从外面进入滚筒，就必须增加数量（delta为正）。\n", "2. 只要看到有衣物从滚筒被拿出来，就必须减少数量（delta为负）。\n", "3. 一个片段内可以发生多次放入或取出，请累计计算。\n", "4. 衣服在滚筒内部移动、翻转、堆叠，不算数量变化（delta=0）。\n", "5. 如果衣服只是露出更多面积，也不算新增。\n", "6. 如果看不清，优先判断为没有变化（delta=0）。\n", "\n", "==================================================\n", "上一时刻状态\n", "==================================================\n", "上一时刻衣物数量：{previous_count}\n", "上一时刻颜色：{previous_colors}\n", "上一时刻空间占用：{previous_occupancy}%\n", "\n", "==================================================\n", "当前任务\n", "==================================================\n", "请仔细观察当前视频片段，判断：\n", "- 是否有新衣服被放进滚筒？放了几件？\n", "- 是否有衣服被从滚筒拿出来？拿了几件？\n", "- 当前滚筒内能看到的主要颜色是什么？\n", "- 当前衣物大约占滚筒可见空间的比例？\n", "\n", "==================================================\n", "输出要求（只输出JSON）\n", "==================================================\n", "{{\n", " \"delta\": 数字（正数表示放入，负数表示取出，0表示无变化）,\n", " \"current_count\": 上一时刻数量 + delta,\n", " \"colors\": [\"颜色1\", \"颜色2\"],\n", " \"occupancy\": 0到100之间的整数（10的倍数）,\n", " \"confidence\": 0到1之间的小数\n", "}}\n", "\n", "注意：\n", "- delta 可以是 +1、+2、+3、-1、-2 等。\n", "- 如果一个片段里连续放入多件，请如实累计。\n", "- 不要因为保守而把明显的放入动作判成0。\n", "\"\"\".strip()\n", "\n", "FINAL_PROMPT = \"\"\"\n", "你是洗衣机最终衣物状态统计模块。\n", "\n", "下面是视频播放过程中每个时间窗口得到的状态：\n", "\n", "{history}\n", "\n", "请根据以上过程，总结最终结果：\n", "\n", "【最终数量】\n", "X 件\n", "\n", "【最终颜色】\n", "颜色列表\n", "\n", "【空间占用】\n", "XX%\n", "\n", "【状态】\n", "空 / 少量 / 中等 / 较多 / 接近装满\n", "\n", "只输出以上内容，不要额外分析。\n", "\"\"\".strip()\n", "\n", "# ------------------------------\n", "# 5. 调用示例（可反复执行）\n", "# ------------------------------\n", "# 默认运行\n", "result = run_analysis()\n", "\n", "# 换视频示例：\n", "# result = run_analysis(video_path=\"/content/另一个视频.mp4\")\n", "\n", "# 只跑前几个片段快速测试：\n", "# result = run_analysis(max_segments=3)\n", "\n", "# 自定义 Prompt 示例：\n", "# result = run_analysis(prompt_template=你的新Prompt)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#===================================================" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "============================================================\n", "Mage-VL 实时洗衣机衣物状态识别\n", "============================================================\n", "[GPU] Tesla T4\n", "[CUDA] 12.8\n", "[BF16] True\n", "[MODEL DTYPE] torch.bfloat16\n", "[SEGMENT] 4.0\n", "[CODEC] 48000 20\n", "============================================================\n", "[init] 加载 Processor...\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "50c25601f54444c28a24b88aa0d07213", "version_major": 2, "version_minor": 0 }, "text/plain": [ "preprocessor_config.json: 0%| | 0.00/1.81k [00:00 3:\n", " print(f\"[warn] 数量偏差较大，强制纠正: 模型输出={current_count} → 期望={expected_count}\")\n", " current_count = expected_count\n", " # ==================================================\n", "\n", " colors = normalize_colors(data.get(\"colors\", previous_colors))\n", " if not colors:\n", " colors = previous_colors\n", "\n", " try:\n", " occupancy = int(round(float(\n", " data.get(\"occupancy\", previous_occupancy))))\n", " except Exception:\n", " occupancy = previous_occupancy\n", " occupancy = max(0, min(100, occupancy))\n", "\n", " try:\n", " confidence = float(data.get(\"confidence\", 0.0))\n", " except Exception:\n", " confidence = 0.0\n", " confidence = max(0.0, min(1.0, confidence))\n", "\n", " # 调试输出\n", " print(f\"[debug] 原始 delta={data.get('delta')} | 原始 current_count={data.get('current_count')} → 最终 delta={delta}, count={current_count}\")\n", "\n", " return {\n", " \"delta\": delta,\n", " \"current_count\": current_count,\n", " \"colors\": colors,\n", " \"occupancy\": occupancy,\n", " \"confidence\": confidence\n", " }\n", "\n", "def state_to_text(state):\n", " d = state[\"delta\"]\n", " if d > 0:\n", " change = f\"放入 {d} 件\"\n", " elif d < 0:\n", " change = f\"取出 {abs(d)} 件\"\n", " else:\n", " change = \"数量无变化\"\n", " colors = \"、\".join(state[\"colors\"]) if state[\"colors\"] else \"未知\"\n", " return f\"{change} | 当前 {state['current_count']} 件 | 颜色={colors} | 占用={state['occupancy']}% | 置信度={state['confidence']:.2f}\"\n", "\n", "# ============================================================\n", "# 最终总结\n", "# ============================================================\n", "@torch.inference_mode()\n", "def final_summary(history, final_state):\n", " if not history:\n", " return \"没有获得有效的衣物分析结果。\"\n", "\n", " prompt = FINAL_PROMPT.format(history=\"\\n\".join(history))\n", " prompt += f\"\"\"\n", "程序计算出的最终状态：\n", "数量：{final_state[\"current_count\"]} 件\n", "颜色：{\", \".join(final_state[\"colors\"]) if final_state[\"colors\"] else \"未知\"}\n", "空间占用：{final_state[\"occupancy\"]}%\n", "请以这个程序最终状态为主要依据。\n", "\"\"\"\n", " inputs = processor.tokenizer(\n", " prompt, return_tensors=\"pt\", truncation=True, max_length=4096)\n", " inputs = {k: v.to(DEVICE) for k, v in inputs.items()}\n", " out = None\n", " try:\n", " out = model.generate(\n", " **inputs,\n", " max_new_tokens=FINAL_MAX_NEW_TOKENS,\n", " do_sample=False, use_cache=True)\n", " n = inputs[\"input_ids\"].shape[1]\n", " return processor.tokenizer.decode(\n", " out[0, n:], skip_special_tokens=True).strip()\n", " finally:\n", " cleanup(inputs)\n", " try:\n", " del out\n", " except Exception:\n", " pass\n", "\n", "# ============================================================\n", "# 实时视频分析\n", "# ============================================================\n", "def stream_video_analysis(\n", " video_path,\n", " segment_sec=SEGMENT_SEC,\n", " gate_threshold=GATE_THRESHOLD,\n", " max_segments=MAX_SEGMENTS,\n", " max_new_tokens=MAX_NEW_TOKENS,\n", " simulate_realtime=SIMULATE_REALTIME):\n", "\n", " path = Path(video_path)\n", " if not path.exists():\n", " raise FileNotFoundError(video_path)\n", "\n", " total = duration(path)\n", " if total <= 0:\n", " raise RuntimeError(\"无法获取视频时长\")\n", "\n", " clip_dir = Path(os.environ[\"STREAMMIND_CLIP_CACHE\"])\n", " clip_dir.mkdir(parents=True, exist_ok=True)\n", "\n", " # 程序真正维护的状态\n", " previous_count = 0\n", " previous_colors = []\n", " previous_occupancy = 0\n", " history = []\n", " results = []\n", " start = 0.0\n", " index = 0\n", " _, codec_override = codec_config()\n", "\n", " yield f\"开始实时分析 | 时长={total:.1f}s | 窗口={segment_sec:.1f}s | Gate={gate_threshold:.2f}\"\n", "\n", " while start < total - 0.05:\n", " if max_segments is not None and index >= max_segments:\n", " break\n", "\n", " end = min(total, start + segment_sec)\n", " length = end - start\n", " index += 1\n", " t0 = time.time()\n", "\n", " clip = clip_dir / f\"{path.stem}_{index:06d}_{start:.2f}_{end:.2f}.mp4\"\n", " inputs = None\n", " cuda_inputs = None\n", " said = None\n", "\n", " try:\n", " prompt = make_prompt(\n", " previous_count,\n", " previous_colors,\n", " previous_occupancy)\n", "\n", " extract_clip(path, start, length, clip)\n", " inputs, prob = analyze_clip(\n", " clip, prompt, codec_override)\n", "\n", " head = f\"**{start:.1f}–{end:.1f}s** Gate=`{prob:.2f}`\"\n", "\n", " if not ALWAYS_RUN_LLM and prob < gate_threshold:\n", " line = f\"{head} → 🔇 无明显变化\"\n", " results.append(line)\n", " yield line\n", " continue\n", "\n", " yield f\"{head} → 🧠 识别衣物状态...\"\n", " gt = time.time()\n", "\n", " cuda_inputs = to_cuda(inputs)\n", " said = generate(cuda_inputs, max_new_tokens=max_new_tokens)\n", " gen_time = time.time() - gt\n", "\n", " # 打印原始模型输出，方便调试\n", " print(f\"[raw LLM] {said}\")\n", "\n", " raw_data = parse_json(said)\n", " state = normalize_state(\n", " raw_data,\n", " previous_count,\n", " previous_colors,\n", " previous_occupancy)\n", "\n", " previous_count = state[\"current_count\"]\n", " previous_colors = state[\"colors\"]\n", " previous_occupancy = state[\"occupancy\"]\n", "\n", " change = state[\"delta\"]\n", " if change > 0:\n", " change_text = f\"🟢 放入 {change} 件\"\n", " elif change < 0:\n", " change_text = f\"🔴 取出 {abs(change)} 件\"\n", " else:\n", " change_text = \"⚪ 数量无变化\"\n", "\n", " colors_text = \"、\".join(state[\"colors\"]) if state[\"colors\"] else \"未知\"\n", " elapsed = time.time() - t0\n", " speed = length / max(elapsed, 0.001)\n", "\n", " line = (\n", " f\"{head}\\n\"\n", " f\" {change_text} | 当前数量={state['current_count']}件\\n\"\n", " f\" 颜色={colors_text} | 占用={state['occupancy']}%\\n\"\n", " f\" 置信度={state['confidence']:.2f} | \"\n", " f\"Generate={gen_time:.2f}s | \"\n", " f\"Total={elapsed:.2f}s | Realtime={speed:.2f}×\"\n", " )\n", "\n", " results.append(line)\n", " history.append(\n", " f\"{start:.1f}-{end:.1f}s：{state_to_text(state)}\")\n", " yield line\n", "\n", " elapsed = time.time() - t0\n", " if simulate_realtime and elapsed < length:\n", " time.sleep(length - elapsed)\n", "\n", " except Exception as e:\n", " line = (\n", " f\"**{start:.1f}–{end:.1f}s** → ❌ \"\n", " f\"{type(e).__name__}: {e}\"\n", " )\n", " results.append(line)\n", " yield line\n", "\n", " finally:\n", " cleanup(cuda_inputs, inputs, said)\n", "\n", " if clip.exists():\n", " try:\n", " clip.unlink()\n", " except Exception:\n", " pass\n", "\n", " start = end\n", " print(\n", " f\"[segment {index}] {start:.1f}/{total:.1f}s | \"\n", " f\"{gpu_mem()}\",\n", " flush=True)\n", "\n", " try:\n", " torch.cuda.reset_peak_memory_stats()\n", " except Exception:\n", " pass\n", "\n", " # ========================================================\n", " # 视频结束\n", " # ========================================================\n", " final_state = {\n", " \"current_count\": previous_count,\n", " \"colors\": previous_colors,\n", " \"occupancy\": previous_occupancy\n", " }\n", "\n", " yield \"\\n========================================\\n🧺 视频播放完成\\n========================================\"\n", "\n", " colors_text = \"、\".join(previous_colors) if previous_colors else \"未知\"\n", "\n", " if previous_count == 0:\n", " occupancy_state = \"空\"\n", " elif previous_occupancy <= 20:\n", " occupancy_state = \"少量\"\n", " elif previous_occupancy <= 50:\n", " occupancy_state = \"中等\"\n", " elif previous_occupancy <= 80:\n", " occupancy_state = \"较多\"\n", " else:\n", " occupancy_state = \"接近装满\"\n", "\n", " final_direct = (\n", " \"========================================\\n\"\n", " \"🧺 最终衣物状态\\n\"\n", " \"========================================\\n\\n\"\n", " f\"【最终数量】\\n{previous_count} 件\\n\\n\"\n", " f\"【最终颜色】\\n{colors_text}\\n\\n\"\n", " f\"【空间占用】\\n{previous_occupancy}%\\n\\n\"\n", " f\"【状态】\\n{occupancy_state}\\n\"\n", " )\n", "\n", " try:\n", " final = final_summary(history, final_state)\n", " yield (\n", " \"\\n\\n\".join(results)\n", " + \"\\n\\n\"\n", " + final_direct\n", " + \"========================================\\n\"\n", " \"🧠 最终模型校验\\n\"\n", " \"========================================\\n\\n\"\n", " + final\n", " + f\"\\n\\n✅ 完成，共 {index} 个片段\"\n", " )\n", " except Exception as e:\n", " yield (\n", " \"\\n\\n\".join(results)\n", " + \"\\n\\n\"\n", " + final_direct\n", " + f\"\\n❌ 最终模型校验失败：{type(e).__name__}: {e}\"\n", " )\n", "\n", "# ============================================================\n", "# Main\n", "# ============================================================\n", "if __name__ == \"__main__\":\n", " for output in stream_video_analysis(\n", " VIDEO_FILE,\n", " segment_sec=SEGMENT_SEC,\n", " gate_threshold=GATE_THRESHOLD,\n", " max_segments=MAX_SEGMENTS,\n", " max_new_tokens=MAX_NEW_TOKENS,\n", " simulate_realtime=SIMULATE_REALTIME):\n", " print(output)\n", " print(\"\\n\" + \"-\" * 60)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "background_save": true, "base_uri": "https://localhost:8080/", "height": 666, "referenced_widgets": [ "2146c7055f0048ce8d21cb166f8e3d0f", "b14bc78b82d94246a335ed995ace3bc7", "01ba7d0ebc774b44a3b23fd2fa293e25", "77c543f3f01b4622a7c266699536befe", "790cc0377a3541dd8bb30020e8b81eb0", "f5045f2cdc7941d48d83ac977478ebcc", "655d5450206e46d2bbbb69c8337ace1d", "33e2524167ec464197adf82052ff1a28", "4f971939fd644d85b79eb9e5ac3fb254", "9c91f735ee154e3cb742e13cf250fbe7", "58a793c0c37c45c88cc7c28935dfd86e", "33d2f74a36cb4cff9b7c2a25f98fa9dc", "89f37aa8f21c40199e4311235fe50529", "e642be2c1e6a45bba3da59a2fcaa280c", "3654893da1a548bfab36c27c10ebb658", "c70aabc972a24fa5915b22f3dbea081e", "7113355aee024e53842940e7eaa4800b", "e369f44e6f8b4cd58bd06bbe968af7c0", "2e20d817d46947099cf317d70fde5cb4", "f7111982e4d445759be371c23a430637", "78a9e6fce42740f9be4c4d04936fb572", "01f8a89c1d0d4feab120b2e7be15ba7b", "003f3ae01dfc4a5dacbb3cd4f9c22b05", "fae6bbc264b9479a99f1255938686f06", "63f3d26dbd5045cfbd491395c7073a27", "d3756f253b8b4c1e9923828cfe03103c", "f4dedf2bc7bf40638056de24d895f10a", "fd6e57c3523b4527b257b30cafdfbc14", "8f9ce5dde93f4dbc843ac5ccb0d67a5e", "d6a6401cad6449c89085f4206d325bef", "3705670eef8645e38589f3e9f167f1c0", "91b455c557a045b0b7bf445588a9ee14", "9a7ea370e3d64b34bba25050cb454267", "4531fa06e24c41ad9e6ed6648a6dd9c8", "38adc087e2324488981e28a5c5aa9f8b", "9f141dc5f80f429d91fa4615748556c4", "c4e0a826cb1c4953974d98c1376f6f1f", "4f1ffbe0a4114c61b8c25a0515b97b78", "8cad1f9783534f669e2144f96fd22747", "dd9bdee5b3aa4ea6a7b4bbbd8cdca1d3", "7caccf6d8fda4dfaa92949c36f1fbcad", "3bf8b8434a6445f3a37a5412f02d9e3c", "4a716780f6da4ef8a4726723b546e41d", "8310c896c07248438f457c97fe49f0ed", "dd856863d5d24655b3ed46125d25ac06", "969428d2ec7b4034b683980c49a85967", "abb69e6625374498b6196f362325ab7e", "84777da5a0814f20b1991aec4443281c", "bd9b8a922acf43f4a04574615a4ff286", "26611c527e0d491e9948d3ec491b70a9", "0e9bbc59bfdf496b990b69ac13c339a0", "53cae6d330714ae1bfdc6da9070506f8", "8306364b2b5f42549e7a14068a25bd0a", "05c6bde7acdd43f2ab960cb4308b779d", "e5f527f1129948b3b5452cf8d39c6952" ] }, "id": "mLGjDxFjAPdb", "outputId": "84f41e77-faa3-4531-9bf1-4a3a484634a4" }, "outputs": [], "source": [ "# ============================================================\n", "# Mage-VL：洗衣机实时衣物状态识别\n", "# 只识别：数量变化 / 最终颜色 / 空间占用\n", "# ============================================================\n", "import os,gc,time,subprocess,importlib,warnings,json,sys\n", "from pathlib import Path\n", "\n", "\n", "\n", "# ---------- 环境 ----------\n", "os.environ.setdefault(\"PYTORCH_CUDA_ALLOC_CONF\",\"expandable_segments:True,max_split_size_mb:128\")\n", "os.environ.setdefault(\"ONLINE_CODEC_CACHE_DIR\",\"/tmp/mage_vl_codec_cache\")\n", "os.environ.setdefault(\"STREAMMIND_CLIP_CACHE\",\"/tmp/mage_vl_clips\")\n", "os.makedirs(os.environ[\"ONLINE_CODEC_CACHE_DIR\"],exist_ok=True)\n", "os.makedirs(os.environ[\"STREAMMIND_CLIP_CACHE\"],exist_ok=True)\n", "\n", "\n", "\n", "# ---------- Import ----------\n", "import torch\n", "from transformers import AutoModelForCausalLM,AutoProcessor\n", "\n", "# ============================================================\n", "# 配置\n", "# ============================================================\n", "MODEL_ID=\"microsoft/Mage-VL\"\n", "VIDEO_FILE=\"/content/test02.mp4\"\n", "SEGMENT_SEC=4.0\n", "MAX_SEGMENTS=None\n", "GATE_THRESHOLD=0.50\n", "MAX_NEW_TOKENS=96\n", "FINAL_MAX_NEW_TOKENS=128\n", "CODEC_MAX_PIXELS=48_000\n", "CODEC_TARGET_CANVAS=20\n", "SIMULATE_REALTIME=True\n", "MAX_CHANGE_PER_SEGMENT=5\n", "ALWAYS_RUN_LLM=True\n", "\n", "# ============================================================\n", "# CUDA\n", "# ============================================================\n", "if not torch.cuda.is_available():\n", " raise RuntimeError(\"未检测到 CUDA GPU\")\n", "DEVICE=\"cuda\"\n", "GPU_NAME=torch.cuda.get_device_name(0)\n", "CUDA_VERSION=torch.version.cuda\n", "try:\n", " BF16_SUPPORTED=torch.cuda.is_bf16_supported()\n", "except Exception:\n", " BF16_SUPPORTED=False\n", "MODEL_DTYPE=torch.bfloat16 if BF16_SUPPORTED else torch.float16\n", "\n", "print(\"=\"*60)\n", "print(\"Mage-VL 实时洗衣机衣物状态识别\")\n", "print(\"=\"*60)\n", "print(\"[GPU]\",GPU_NAME)\n", "print(\"[CUDA]\",CUDA_VERSION)\n", "print(\"[BF16]\",BF16_SUPPORTED)\n", "print(\"[MODEL DTYPE]\",MODEL_DTYPE)\n", "print(\"[SEGMENT]\",SEGMENT_SEC)\n", "print(\"[CODEC]\",CODEC_MAX_PIXELS,CODEC_TARGET_CANVAS)\n", "print(\"=\"*60)\n", "\n", "# ============================================================\n", "# 实时识别 Prompt\n", "# ============================================================\n", "PROMPT_TEMPLATE=\"\"\"\n", "你是洗衣机滚筒实时视觉状态检测器。\n", "\n", "你只判断以下三个信息：\n", "\n", "1. 衣物数量发生了什么变化\n", "2. 当前滚筒中的衣物数量\n", "3. 当前能看到的主要衣物颜色\n", "4. 当前衣物占滚筒可见内部空间的大约比例\n", "\n", "绝对不要分析：\n", "- 材质\n", "- 衣物类型\n", "- 厚薄\n", "- 重量\n", "- 洗涤程序\n", "- 水温\n", "- 漂洗\n", "- 脱水\n", "- 洗涤建议\n", "\n", "==================================================\n", "上一时刻状态\n", "==================================================\n", "上一时刻衣物数量：\n", "{previous_count}\n", "\n", "上一时刻颜色：\n", "{previous_colors}\n", "\n", "上一时刻空间占用：\n", "{previous_occupancy}%\n", "\n", "==================================================\n", "当前视频片段\n", "==================================================\n", "\n", "这是洗衣机连续视频中的一个时间窗口。\n", "\n", "重点观察：\n", "- 有没有新的衣物被放入滚筒\n", "- 有没有衣物被从滚筒取出\n", "- 有没有衣物只是移动位置\n", "- 不要因为衣物移动而重复计数\n", "- 不要因为同一件衣服露出更多面积而重复计数\n", "\n", "==================================================\n", "数量判断原则\n", "==================================================\n", "\n", "只有明确看到衣物进入或离开滚筒时，才改变数量。\n", "\n", "放入一件：delta=+1\n", "放入两件：delta=+2\n", "取出一件：delta=-1\n", "取出两件：delta=-2\n", "\n", "只是移动衣物：delta=0\n", "看不清：delta=0\n", "\n", "不要因为看见一件已经存在的衣物而再次增加数量。\n", "\n", "current_count = 上一时刻数量 + 本片段明确发生的数量变化。\n", "\n", "如果无法确定变化：\n", "delta=0\n", "\n", "==================================================\n", "颜色\n", "==================================================\n", "\n", "只识别当前明显看到的主要颜色。\n", "\n", "允许：\n", "黑色、白色、灰色、蓝色、深蓝色、浅蓝色、\n", "红色、粉色、黄色、绿色、棕色、米色、紫色、橙色\n", "\n", "颜色无法确定时不要猜。\n", "\n", "==================================================\n", "空间占用\n", "==================================================\n", "\n", "估计衣物占据滚筒可见内部空间的比例。\n", "\n", "只允许：\n", "0、10、20、30、40、50、60、70、80、90、100\n", "\n", "这不是精确物理体积，只是视觉估计。\n", "\n", "==================================================\n", "非常重要\n", "==================================================\n", "\n", "如果没有明确放入或取出：\n", "delta=0\n", "\n", "如果只是重新堆叠：\n", "delta=0\n", "\n", "如果无法判断：\n", "delta=0\n", "\n", "==================================================\n", "输出\n", "==================================================\n", "\n", "只输出 JSON，不要解释：\n", "\n", "{{\n", " \"delta\":0,\n", " \"current_count\":{previous_count},\n", " \"colors\":[],\n", " \"occupancy\":0,\n", " \"confidence\":0.0\n", "}}\n", "\"\"\".strip()\n", "\n", "# ============================================================\n", "# 最终总结 Prompt\n", "# ============================================================\n", "FINAL_PROMPT=\"\"\"\n", "你是洗衣机最终衣物状态统计模块。\n", "\n", "下面是视频播放过程中每个时间窗口得到的状态：\n", "\n", "{history}\n", "\n", "只总结三个信息：\n", "\n", "1. 最终滚筒中的衣物数量\n", "2. 最终滚筒中的主要衣物颜色\n", "3. 最终滚筒中的空间占用情况\n", "\n", "不要分析：\n", "- 材质\n", "- 衣物类型\n", "- 洗涤程序\n", "- 水温\n", "- 漂洗\n", "- 脱水\n", "- 洗涤建议\n", "\n", "注意：\n", "1. 放入后又取出的衣物不能计入最终数量。\n", "2. 同一件衣物不能重复计算。\n", "3. 移动、翻转、重新堆叠不算新增。\n", "4. 最终颜色只统计视频结束时仍在滚筒中的颜色。\n", "5. 最终空间占用以视频结束时为准。\n", "\n", "只输出：\n", "\n", "【最终数量】\n", "X 件\n", "\n", "【最终颜色】\n", "黑色、白色、蓝色\n", "\n", "【空间占用】\n", "XX%\n", "\n", "【状态】\n", "空 / 少量 / 中等 / 较多 / 接近装满\n", "\n", "不要输出其它分析。\n", "\"\"\".strip()\n", "\n", "# ============================================================\n", "# Processor / Model\n", "# ============================================================\n", "print(\"[init] 加载 Processor...\")\n", "processor=AutoProcessor.from_pretrained(MODEL_ID,trust_remote_code=True)\n", "print(\"[init] Processor 加载完成\")\n", "\n", "print(\"[init] 加载 Mage-VL...\")\n", "model=AutoModelForCausalLM.from_pretrained(\n", " MODEL_ID,\n", " trust_remote_code=True, \n", " torch_dtype=MODEL_DTYPE,\n", " attn_implementation=\"sdpa\",\n", " low_cpu_mem_usage=True\n", ").to(DEVICE).eval()\n", "print(\"[init] Mage-VL 加载完成\")\n", "\n", "# ============================================================\n", "# Codec\n", "# ============================================================\n", "_REMOTE_PKG=type(processor).__module__.rsplit(\".\",1)[0]\n", "codec_mod=importlib.import_module(_REMOTE_PKG+\".codec_video_processing_mage_vl\")\n", "CodecConfig=codec_mod.CodecConfig\n", "\n", "def codec_config():\n", " override={\"engine\":\"hevc\",\"target_canvas\":CODEC_TARGET_CANVAS,\"patch\":16}\n", " kwargs=dict(processor._codec_config_defaults)\n", " dcvc=dict(kwargs.get(\"dcvc\") or {})\n", " kwargs.update(override)\n", " if dcvc:\n", " kwargs[\"dcvc\"]=dcvc\n", " kwargs[\"max_pixels\"]=CODEC_MAX_PIXELS\n", " return CodecConfig(**kwargs),override\n", "\n", "# ============================================================\n", "# Gate\n", "# ============================================================\n", "print(\"[gate] 加载 Cognition Gate...\")\n", "try:\n", " gate=model.model._load_streammind_gate()\n", " print(\"[gate] Cognition Gate 加载成功\")\n", " try:\n", " print(\"[gate] 当前 dtype =\",next(gate.parameters()).dtype)\n", " except Exception:\n", " pass\n", "except Exception as e:\n", " gate=None\n", " print(\"[gate] 加载失败:\",type(e).__name__,e)\n", "\n", "if gate is not None and not BF16_SUPPORTED:\n", " try:\n", " gate=gate.float()\n", " model.model.streammind_gate=gate\n", " print(\"[gate] FP32 fallback 已启用\")\n", " except Exception as e:\n", " print(\"[gate] FP32 fallback 失败:\",e)\n", "\n", "# ============================================================\n", "# 工具\n", "# ============================================================\n", "def duration(path):\n", " try:\n", " r=subprocess.run(\n", " [\"ffprobe\",\"-v\",\"error\",\"-show_entries\",\"format=duration\",\n", " \"-of\",\"default=nw=1:nk=1\",str(path)],\n", " capture_output=True,text=True,check=True,timeout=60)\n", " return float(r.stdout.strip())\n", " except Exception:\n", " return 0.0\n", "\n", "def make_prompt(previous_count,previous_colors,previous_occupancy):\n", " text=PROMPT_TEMPLATE.format(\n", " previous_count=previous_count,\n", " previous_colors=\", \".join(previous_colors) if previous_colors else \"暂无\",\n", " previous_occupancy=previous_occupancy)\n", " messages=[{\"role\":\"user\",\"content\":[\n", " {\"type\":\"video\"},\n", " {\"type\":\"text\",\"text\":text}\n", " ]}]\n", " return processor.apply_chat_template(\n", " messages,tokenize=False,add_generation_prompt=True)\n", "\n", "def extract_clip(src,start,length,out):\n", " tmp=out.with_name(\".tmp_\"+out.name)\n", " if tmp.exists():\n", " try: tmp.unlink()\n", " except Exception: pass\n", " subprocess.run([\n", " \"ffmpeg\",\"-y\",\"-loglevel\",\"error\",\n", " \"-ss\",f\"{start:.3f}\",\"-i\",str(src),\n", " \"-t\",f\"{length:.3f}\",\n", " \"-c:v\",\"libx264\",\"-preset\",\"ultrafast\",\n", " \"-crf\",\"26\",\"-an\",\"-pix_fmt\",\"yuv420p\",str(tmp)\n", " ],check=True,capture_output=True,timeout=180)\n", " tmp.replace(out)\n", " return out\n", "\n", "def to_cuda(inputs):\n", " result={}\n", " for k,v in inputs.items():\n", " if not hasattr(v,\"to\"): continue\n", " v=v.to(DEVICE)\n", " if k==\"pixel_values\":\n", " v=v.to(MODEL_DTYPE)\n", " result[k]=v\n", " return result\n", "\n", "def cleanup(*objs):\n", " for x in objs:\n", " try: del x\n", " except Exception: pass\n", " gc.collect()\n", " try:\n", " torch.cuda.empty_cache()\n", " torch.cuda.ipc_collect()\n", " torch.cuda.synchronize()\n", " except Exception: pass\n", "\n", "def gpu_mem():\n", " a=torch.cuda.memory_allocated()/1024**3\n", " r=torch.cuda.memory_reserved()/1024**3\n", " p=torch.cuda.max_memory_allocated()/1024**3\n", " return f\"GPU={a:.2f}GB / reserved={r:.2f}GB / peak={p:.2f}GB\"\n", "\n", "# ============================================================\n", "# Gate\n", "# ============================================================\n", "@torch.inference_mode()\n", "def run_gate(inputs):\n", " if gate is None:\n", " return 1.0\n", " feat={}\n", " logits=None\n", " try:\n", " for k in (\"pixel_values\",\"image_grid_thw\",\"patch_positions\"):\n", " if k not in inputs: continue\n", " v=inputs[k].to(DEVICE)\n", " if k==\"pixel_values\":\n", " v=v.to(MODEL_DTYPE)\n", " feat[k]=v\n", "\n", " if BF16_SUPPORTED:\n", " logits=model.model.streammind_gate_forward_segments([feat])\n", " else:\n", " visual_tokens=model.model._streammind_vision_tokens(\n", " feat[\"pixel_values\"],\n", " feat[\"image_grid_thw\"],\n", " patch_positions=feat.get(\"patch_positions\"))\n", " if not torch.isfinite(visual_tokens).all():\n", " print(\"[gate] visual tokens NaN/Inf\")\n", " return 0.0\n", " logits=gate(torch.cat([visual_tokens.float()],dim=1))\n", "\n", " if not isinstance(logits,torch.Tensor) or logits.ndim!=3:\n", " print(\"[gate] Unexpected shape:\",getattr(logits,\"shape\",None))\n", " return 0.0\n", "\n", " gate_logits=logits[0]\n", " length=int(inputs[\"image_grid_thw\"][:,0].sum().item())\n", " if length<=0: return 0.0\n", "\n", " pos=min(length-1,gate_logits.shape[0]-1)\n", " last_logits=gate_logits[pos].float()\n", " if not torch.isfinite(last_logits).all():\n", " return 0.0\n", "\n", " prob=torch.exp(torch.log_softmax(last_logits,dim=-1)[1])\n", " if not torch.isfinite(prob):\n", " return 0.0\n", " return float(prob.item())\n", " except Exception as e:\n", " print(\"[gate] ERROR:\",type(e).__name__,e)\n", " return 0.0\n", " finally:\n", " cleanup(feat,logits)\n", "\n", "# ============================================================\n", "# Codec + Gate\n", "# ============================================================\n", "@torch.inference_mode()\n", "def analyze_clip(clip,prompt,codec_override):\n", " with warnings.catch_warnings():\n", " warnings.simplefilter(\"ignore\")\n", " inputs=processor(\n", " text=[prompt],\n", " videos=[str(clip)],\n", " video_backend=\"codec\",\n", " max_pixels=CODEC_MAX_PIXELS,\n", " codec_config=codec_override,\n", " return_tensors=\"pt\",\n", " padding=False)\n", " return inputs,run_gate(inputs)\n", "\n", "# ============================================================\n", "# LLM\n", "# ============================================================\n", "@torch.inference_mode()\n", "def generate(inputs,max_new_tokens=MAX_NEW_TOKENS):\n", " out=model.generate(\n", " **inputs,\n", " max_new_tokens=max_new_tokens,\n", " do_sample=False,\n", " use_cache=True)\n", " n=inputs[\"input_ids\"].shape[1]\n", " text=processor.tokenizer.decode(\n", " out[0,n:],skip_special_tokens=True).strip()\n", " del out\n", " return text\n", "\n", "# ============================================================\n", "# JSON\n", "# ============================================================\n", "def parse_json(text):\n", " if not text: return None\n", " text=text.strip().replace(\"```json\",\"\").replace(\"```\",\"\").strip()\n", " start=text.find(\"{\")\n", " end=text.rfind(\"}\")\n", " if start<0 or end<0: return None\n", " try:\n", " data=json.loads(text[start:end+1])\n", " return data if isinstance(data,dict) else None\n", " except Exception:\n", " return None\n", "\n", "VALID_COLORS={\n", " \"黑色\",\"白色\",\"灰色\",\"蓝色\",\"深蓝色\",\"浅蓝色\",\n", " \"红色\",\"粉色\",\"黄色\",\"绿色\",\"棕色\",\"米色\",\"紫色\",\"橙色\"\n", "}\n", "\n", "def normalize_colors(colors):\n", " if not isinstance(colors,list): return []\n", " result=[]\n", " for c in colors:\n", " if isinstance(c,str):\n", " c=c.strip()\n", " if c in VALID_COLORS and c not in result:\n", " result.append(c)\n", " return result\n", "\n", "def normalize_state(data,previous_count,previous_colors,previous_occupancy):\n", " if not isinstance(data,dict):\n", " return {\n", " \"delta\":0,\n", " \"current_count\":previous_count,\n", " \"colors\":previous_colors,\n", " \"occupancy\":previous_occupancy,\n", " \"confidence\":0.0\n", " }\n", "\n", " try:\n", " delta=int(round(float(data.get(\"delta\",0))))\n", " except Exception:\n", " delta=0\n", " delta=max(-MAX_CHANGE_PER_SEGMENT,min(MAX_CHANGE_PER_SEGMENT,delta))\n", "\n", " try:\n", " current_count=int(round(float(\n", " data.get(\"current_count\",previous_count+delta))))\n", " except Exception:\n", " current_count=previous_count+delta\n", "\n", " current_count=max(0,current_count)\n", " expected_count=max(0,previous_count+delta)\n", " if abs(current_count-expected_count)>1:\n", " current_count=expected_count\n", "\n", " colors=normalize_colors(data.get(\"colors\",previous_colors))\n", " if not colors:\n", " colors=previous_colors\n", "\n", " try:\n", " occupancy=int(round(float(\n", " data.get(\"occupancy\",previous_occupancy))))\n", " except Exception:\n", " occupancy=previous_occupancy\n", " occupancy=max(0,min(100,occupancy))\n", "\n", " try:\n", " confidence=float(data.get(\"confidence\",0.0))\n", " except Exception:\n", " confidence=0.0\n", " confidence=max(0.0,min(1.0,confidence))\n", "\n", " return {\n", " \"delta\":delta,\n", " \"current_count\":current_count,\n", " \"colors\":colors,\n", " \"occupancy\":occupancy,\n", " \"confidence\":confidence\n", " }\n", "\n", "def state_to_text(state):\n", " d=state[\"delta\"]\n", " if d>0: change=f\"放入 {d} 件\"\n", " elif d<0: change=f\"取出 {abs(d)} 件\"\n", " else: change=\"数量无变化\"\n", " colors=\"、\".join(state[\"colors\"]) if state[\"colors\"] else \"未知\"\n", " return f\"{change} | 当前 {state['current_count']} 件 | 颜色={colors} | 占用={state['occupancy']}% | 置信度={state['confidence']:.2f}\"\n", "\n", "# ============================================================\n", "# 最终总结\n", "# ============================================================\n", "@torch.inference_mode()\n", "def final_summary(history,final_state):\n", " if not history:\n", " return \"没有获得有效的衣物分析结果。\"\n", "\n", " prompt=FINAL_PROMPT.format(history=\"\\n\".join(history))\n", " prompt+=f\"\"\"\n", "程序计算出的最终状态：\n", "数量：{final_state[\"current_count\"]} 件\n", "颜色：{\", \".join(final_state[\"colors\"]) if final_state[\"colors\"] else \"未知\"}\n", "空间占用：{final_state[\"occupancy\"]}%\n", "请以这个程序最终状态为主要依据。\n", "\"\"\"\n", " inputs=processor.tokenizer(\n", " prompt,return_tensors=\"pt\",truncation=True,max_length=4096)\n", " inputs={k:v.to(DEVICE) for k,v in inputs.items()}\n", " out=None\n", " try:\n", " out=model.generate(\n", " **inputs,\n", " max_new_tokens=FINAL_MAX_NEW_TOKENS,\n", " do_sample=False,use_cache=True)\n", " n=inputs[\"input_ids\"].shape[1]\n", " return processor.tokenizer.decode(\n", " out[0,n:],skip_special_tokens=True).strip()\n", " finally:\n", " cleanup(inputs)\n", " try: del out\n", " except Exception: pass\n", "\n", "# ============================================================\n", "# 实时视频分析\n", "# ============================================================\n", "def stream_video_analysis(\n", " video_path,\n", " segment_sec=SEGMENT_SEC,\n", " gate_threshold=GATE_THRESHOLD,\n", " max_segments=MAX_SEGMENTS,\n", " max_new_tokens=MAX_NEW_TOKENS,\n", " simulate_realtime=SIMULATE_REALTIME):\n", "\n", " path=Path(video_path)\n", " if not path.exists():\n", " raise FileNotFoundError(video_path)\n", "\n", " total=duration(path)\n", " if total<=0:\n", " raise RuntimeError(\"无法获取视频时长\")\n", "\n", " clip_dir=Path(os.environ[\"STREAMMIND_CLIP_CACHE\"])\n", " clip_dir.mkdir(parents=True,exist_ok=True)\n", "\n", " # 程序真正维护的状态\n", " previous_count=0\n", " previous_colors=[]\n", " previous_occupancy=0\n", " history=[]\n", " results=[]\n", " start=0.0\n", " index=0\n", " _,codec_override=codec_config()\n", "\n", " yield f\"开始实时分析 | 时长={total:.1f}s | 窗口={segment_sec:.1f}s | Gate={gate_threshold:.2f}\"\n", "\n", " while start=max_segments:\n", " break\n", "\n", " end=min(total,start+segment_sec)\n", " length=end-start\n", " index+=1\n", " t0=time.time()\n", "\n", " clip=clip_dir/f\"{path.stem}_{index:06d}_{start:.2f}_{end:.2f}.mp4\"\n", " inputs=None\n", " cuda_inputs=None\n", " said=None\n", "\n", " try:\n", " prompt=make_prompt(\n", " previous_count,\n", " previous_colors,\n", " previous_occupancy)\n", "\n", " extract_clip(path,start,length,clip)\n", " inputs,prob=analyze_clip(\n", " clip,prompt,codec_override)\n", "\n", " head=f\"**{start:.1f}–{end:.1f}s** Gate=`{prob:.2f}`\"\n", "\n", " # 保留原逻辑：ALWAYS_RUN_LLM=True 时 Gate 不阻止 LLM\n", " if not ALWAYS_RUN_LLM and prob0:\n", " change_text=f\"🟢 放入 {change} 件\"\n", " elif change<0:\n", " change_text=f\"🔴 取出 {abs(change)} 件\"\n", " else:\n", " change_text=\"⚪ 数量无变化\"\n", "\n", " colors_text=\"、\".join(state[\"colors\"]) if state[\"colors\"] else \"未知\"\n", " elapsed=time.time()-t0\n", " speed=length/max(elapsed,0.001)\n", "\n", " line=(\n", " f\"{head}\\n\"\n", " f\" {change_text} | 当前数量={state['current_count']}件\\n\"\n", " f\" 颜色={colors_text} | 占用={state['occupancy']}%\\n\"\n", " f\" 置信度={state['confidence']:.2f} | \"\n", " f\"Generate={gen_time:.2f}s | \"\n", " f\"Total={elapsed:.2f}s | Realtime={speed:.2f}×\"\n", " )\n", "\n", " results.append(line)\n", " history.append(\n", " f\"{start:.1f}-{end:.1f}s：{state_to_text(state)}\")\n", " yield line\n", "\n", " elapsed=time.time()-t0\n", " if simulate_realtime and elapsed