{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "52685911", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m100.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m0:01\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m394.3/394.3 kB\u001b[0m \u001b[31m45.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.1/43.1 MB\u001b[0m \u001b[31m20.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m106.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m35.8/35.8 MB\u001b[0m \u001b[31m30.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.6/13.6 MB\u001b[0m \u001b[31m110.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25h✅ 环境安装完成\n" ] } ], "source": [ "# ============================================================\n", "# 代码块 1：环境安装\n", "# ============================================================\n", "!pip install -U transformers accelerate qwen-vl-utils[decord] bitsandbytes -q\n", "!pip install -U torch torchvision --index-url https://download.pytorch.org/whl/cu124 -q # 根据你的CUDA版本调整\n", "\n", "import os\n", "os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True,max_split_size_mb:128\"\n", "print(\"✅ 环境安装完成\")" ] }, { "cell_type": "code", "execution_count": null, "id": "70fffa36", "metadata": {}, "outputs": [], "source": [ "from huggingface_hub import login\n", "\n", "login(token=\"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\") " ] }, { "cell_type": "code", "execution_count": 2, "id": "f6914f9a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2026-09-11 02:40:28-- http://172.245.243.238:8000/test02.mp4\n", "Connecting to 172.245.243.238:8000... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 30270635 (29M) [video/mp4]\n", "Saving to: ‘test02.mp4’\n", "\n", "test02.mp4 0%[ ] 0 --.-KB/s " ] }, { "name": "stdout", "output_type": "stream", "text": [ "test02.mp4 100%[===================>] 28.87M 55.1MB/s in 0.5s \n", "\n", "2026-09-11 02:40:29 (55.1 MB/s) - ‘test02.mp4’ saved [30270635/30270635]\n", "\n", "--2026-09-11 02:40:29-- http://172.245.243.238:8000/test03.mp4\n", "Connecting to 172.245.243.238:8000... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 13549203 (13M) [video/mp4]\n", "Saving to: ‘test03.mp4’\n", "\n", "test03.mp4 100%[===================>] 12.92M 35.6MB/s in 0.4s \n", "\n", "2026-09-11 02:40:30 (35.6 MB/s) - ‘test03.mp4’ saved [13549203/13549203]\n", "\n", "--2026-09-11 02:40:30-- http://172.245.243.238:8000/test04.mp4\n", "Connecting to 172.245.243.238:8000... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 27143071 (26M) [video/mp4]\n", "Saving to: ‘test04.mp4’\n", "\n", "test04.mp4 100%[===================>] 25.88M 53.0MB/s in 0.5s \n", "\n", "2026-09-11 02:40:30 (53.0 MB/s) - ‘test04.mp4’ saved [27143071/27143071]\n", "\n", "--2026-09-11 02:40:30-- http://172.245.243.238:8000/test05.mp4\n", "Connecting to 172.245.243.238:8000... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 36972551 (35M) [video/mp4]\n", "Saving to: ‘test05.mp4’\n", "\n", "test05.mp4 100%[===================>] 35.26M 41.0MB/s in 0.9s \n", "\n", "2026-09-11 02:40:31 (41.0 MB/s) - ‘test05.mp4’ saved [36972551/36972551]\n", "\n" ] } ], "source": [ "!wget http://172.245.243.238:8000/test02.mp4\n", "!wget http://172.245.243.238:8000/test03.mp4\n", "!wget http://172.245.243.238:8000/test04.mp4\n", "!wget http://172.245.243.238:8000/test05.mp4" ] }, { "cell_type": "code", "execution_count": 3, "id": "9622b9f4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "============================================================\n", "正在加载 Qwen3-VL-8B-Instruct（4-bit 量化）...\n", "============================================================\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.12/dist-packages/huggingface_hub/utils/_auth.py:138: UserWarning: \n", "Error while fetching `HF_TOKEN` secret value from your vault: 'Requesting secret HF_TOKEN timed out. Secrets can only be fetched when running from the Colab UI.'.\n", " warnings.warn(f\"\\nError while fetching `HF_TOKEN` secret value from your vault: '{str(e)}'.\")\n", "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n", "WARNING:huggingface_hub.utils._http:Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "63e98862b6b141e7a4dd379c25c51af9", "version_major": 2, "version_minor": 0 }, "text/plain": [ "config.json: 0%| | 0.00/1.47k [00:00 数当前可见衣物 -> current_count - previous_count\n", "#\n", "# 而是：\n", "# BEFORE + AFTER\n", "# ↓\n", "# 判断有没有“放入/取出”动作\n", "# ↓\n", "# Python 累计数量\n", "#\n", "# 这样可以避免：\n", "# 衣物遮挡\n", "# 衣物翻转\n", "# 滚筒旋转\n", "# 可见衣物数量变化\n", "# 被误认为衣物真的增加/减少\n", "# ============================================================\n", "\n", "import cv2\n", "import time\n", "import json\n", "import re\n", "import gc\n", "import torch\n", "import numpy as np\n", "\n", "from pathlib import Path\n", "\n", "\n", "# ============================================================\n", "# 1. 基本工具\n", "# ============================================================\n", "\n", "def duration(path):\n", " \"\"\"\n", " 获取视频时长。\n", " 优先使用 OpenCV，失败时返回 0。\n", " \"\"\"\n", "\n", " cap = cv2.VideoCapture(str(path))\n", "\n", " if not cap.isOpened():\n", " return 0.0\n", "\n", " fps = cap.get(cv2.CAP_PROP_FPS)\n", " frames = cap.get(cv2.CAP_PROP_FRAME_COUNT)\n", "\n", " cap.release()\n", "\n", " if fps and fps > 0 and frames and frames > 0:\n", " return frames / fps\n", "\n", " return 0.0\n", "\n", "\n", "def get_video_info(path):\n", " \"\"\"\n", " 获取视频 FPS、总帧数、时长。\n", " \"\"\"\n", "\n", " cap = cv2.VideoCapture(str(path))\n", "\n", " if not cap.isOpened():\n", " raise RuntimeError(f\"无法打开视频：{path}\")\n", "\n", " fps = cap.get(cv2.CAP_PROP_FPS)\n", " frame_count = cap.get(cv2.CAP_PROP_FRAME_COUNT)\n", "\n", " if fps <= 0:\n", " fps = 30.0\n", "\n", " duration_sec = frame_count / fps\n", "\n", " cap.release()\n", "\n", " return fps, frame_count, duration_sec\n", "\n", "\n", "# ============================================================\n", "# 2. 精确读取某一时间点的视频帧\n", "# ============================================================\n", "\n", "def read_frame(cap, timestamp):\n", " \"\"\"\n", " 读取 timestamp 秒位置的帧。\n", "\n", " 返回：\n", " RGB numpy array\n", " \"\"\"\n", "\n", " cap.set(\n", " cv2.CAP_PROP_POS_MSEC,\n", " float(timestamp) * 1000.0\n", " )\n", "\n", " ok, frame = cap.read()\n", "\n", " if not ok or frame is None:\n", " return None\n", "\n", " # OpenCV BGR -> RGB\n", " frame = cv2.cvtColor(\n", " frame,\n", " cv2.COLOR_BGR2RGB\n", " )\n", "\n", " return frame\n", "\n", "\n", "# ============================================================\n", "# 3. 图像变化检测\n", "#\n", "# 这里只负责：\n", "# “有没有明显变化？”\n", "#\n", "# 不负责识别衣服。\n", "#\n", "# 如果变化很小，就不用调用 Qwen。\n", "# ============================================================\n", "\n", "def image_difference(frame1, frame2):\n", "\n", " if frame1 is None or frame2 is None:\n", " return 1.0\n", "\n", " # 转灰度\n", " gray1 = cv2.cvtColor(\n", " frame1,\n", " cv2.COLOR_RGB2GRAY\n", " )\n", "\n", " gray2 = cv2.cvtColor(\n", " frame2,\n", " cv2.COLOR_RGB2GRAY\n", " )\n", "\n", " # 缩小，降低计算量\n", " gray1 = cv2.resize(\n", " gray1,\n", " (160, 240)\n", " )\n", "\n", " gray2 = cv2.resize(\n", " gray2,\n", " (160, 240)\n", " )\n", "\n", " diff = cv2.absdiff(\n", " gray1,\n", " gray2\n", " )\n", "\n", " score = float(\n", " diff.mean()\n", " ) / 255.0\n", "\n", " return score\n", "\n", "\n", "# ============================================================\n", "# 4. JSON解析\n", "# ============================================================\n", "\n", "def parse_json(text):\n", "\n", " if not text:\n", " return None\n", "\n", " text = str(text).strip()\n", "\n", " # 去掉 markdown\n", " text = re.sub(\n", " r\"```json\",\n", " \"\",\n", " text,\n", " flags=re.IGNORECASE\n", " )\n", "\n", " text = text.replace(\n", " \"```\",\n", " \"\"\n", " ).strip()\n", "\n", " # 直接解析\n", " try:\n", "\n", " data = json.loads(text)\n", "\n", " if isinstance(data, dict):\n", " return data\n", "\n", " except Exception:\n", " pass\n", "\n", " # 尝试从其他文本中提取 JSON\n", " match = re.search(\n", " r\"\\{.*\\}\",\n", " text,\n", " flags=re.S\n", " )\n", "\n", " if match:\n", "\n", " try:\n", "\n", " data = json.loads(\n", " match.group()\n", " )\n", "\n", " if isinstance(data, dict):\n", " return data\n", "\n", " except Exception:\n", " pass\n", "\n", " return None\n", "\n", "\n", "# ============================================================\n", "# 5. 标准化事件\n", "# ============================================================\n", "\n", "def normalize_event(data):\n", "\n", " if not isinstance(data, dict):\n", " return None\n", "\n", " event = str(\n", " data.get(\n", " \"event\",\n", " \"uncertain\"\n", " )\n", " ).strip().lower()\n", "\n", " valid_events = {\n", " \"put_in\",\n", " \"take_out\",\n", " \"none\",\n", " \"uncertain\"\n", " }\n", "\n", " if event not in valid_events:\n", " event = \"uncertain\"\n", "\n", " # confidence\n", " try:\n", "\n", " confidence = float(\n", " data.get(\n", " \"confidence\",\n", " 0.0\n", " )\n", " )\n", "\n", " except Exception:\n", "\n", " confidence = 0.0\n", "\n", " confidence = max(\n", " 0.0,\n", " min(\n", " 1.0,\n", " confidence\n", " )\n", " )\n", "\n", " # count change\n", " try:\n", "\n", " count_change = int(\n", " round(\n", " float(\n", " data.get(\n", " \"count_change\",\n", " 0\n", " )\n", " )\n", " )\n", " )\n", "\n", " except Exception:\n", "\n", " count_change = 0\n", "\n", " # 事件方向约束\n", " if event == \"put_in\":\n", "\n", " count_change = max(\n", " 1,\n", " count_change\n", " )\n", "\n", " elif event == \"take_out\":\n", "\n", " count_change = min(\n", " -1,\n", " count_change\n", " )\n", "\n", " else:\n", "\n", " count_change = 0\n", "\n", " # 防止模型一次说几十件\n", " count_change = max(\n", " -10,\n", " min(\n", " 10,\n", " count_change\n", " )\n", " )\n", "\n", " return {\n", " \"event\": event,\n", " \"count_change\": count_change,\n", " \"item_color\": data.get(\n", " \"item_color\"\n", " ),\n", " \"item_pattern\": data.get(\n", " \"item_pattern\"\n", " ),\n", " \"confidence\": confidence,\n", " \"reason\": str(\n", " data.get(\n", " \"reason\",\n", " \"\"\n", " )\n", " ),\n", " }\n", "\n", "\n", "# ============================================================\n", "# 6. 事件确认器\n", "#\n", "# 一个动作可能被多个相邻窗口看到。\n", "#\n", "# 例如：\n", "#\n", "# 2.0-4.0s -> put_in\n", "# 3.0-5.0s -> put_in\n", "#\n", "# 实际上只是放了一件衣服。\n", "#\n", "# 必须经过连续确认才真正 +1。\n", "# ============================================================\n", "\n", "class EventStabilizer:\n", "\n", " def __init__(\n", " self,\n", " required_confirmations=2\n", " ):\n", "\n", " self.required = (\n", " required_confirmations\n", " )\n", "\n", " self.pending_event = None\n", " self.pending_change = 0\n", " self.pending_hits = 0\n", "\n", " def reset(self):\n", "\n", " self.pending_event = None\n", " self.pending_change = 0\n", " self.pending_hits = 0\n", "\n", " def update(self, result):\n", "\n", " if result is None:\n", " return None\n", "\n", " event = result[\"event\"]\n", " change = result[\"count_change\"]\n", "\n", " # none / uncertain\n", " # 不参与累计确认\n", " if event not in {\n", " \"put_in\",\n", " \"take_out\"\n", " }:\n", "\n", " # 注意：\n", " # 不立即 reset 也可以，但这里选择 reset，\n", " # 防止间隔很久后误累计。\n", " self.reset()\n", "\n", " return None\n", "\n", " # 低置信度不确认\n", " if result[\"confidence\"] < 0.70:\n", "\n", " return None\n", "\n", " # 与上一次候选事件相同\n", " if (\n", " event == self.pending_event\n", " and change == self.pending_change\n", " ):\n", "\n", " self.pending_hits += 1\n", "\n", " else:\n", "\n", " self.pending_event = event\n", " self.pending_change = change\n", " self.pending_hits = 1\n", "\n", " # 达到确认次数\n", " if self.pending_hits >= self.required:\n", "\n", " confirmed = result.copy()\n", "\n", " self.reset()\n", "\n", " return confirmed\n", "\n", " return None\n", "\n", "\n", "# ============================================================\n", "# 7. Qwen3-VL：\n", "#\n", "# BEFORE + AFTER\n", "#\n", "# 注意：\n", "# 这里不再使用 video 输入。\n", "#\n", "# 因此：\n", "# 不需要 fps\n", "# 不需要 process_vision_info\n", "# 不会出现 fps=[xxx]\n", "# 不会出现 Defaulting to fps=24\n", "# ============================================================\n", "\n", "@torch.inference_mode()\n", "def compare_frames(\n", " before_frame,\n", " after_frame,\n", " before_time,\n", " after_time,\n", " max_pixels=360 * 420,\n", " max_new_tokens=64,\n", "):\n", " \"\"\"\n", " 使用 Qwen3-VL 比较两个时间点的画面。\n", "\n", " 只判断：\n", " put_in\n", " take_out\n", " none\n", " uncertain\n", "\n", " 不让模型负责累计总数量。\n", " \"\"\"\n", "\n", " if before_frame is None:\n", " return None\n", "\n", " if after_frame is None:\n", " return None\n", "\n", " prompt = f\"\"\"\n", "你是一个高精度的“洗衣机滚筒衣物变化检测专家”。\n", "\n", "现在给你同一个洗衣机滚筒在两个时间点的画面：\n", "\n", "BEFORE：\n", "约 {before_time:.1f} 秒\n", "\n", "AFTER：\n", "约 {after_time:.1f} 秒\n", "\n", "请比较 BEFORE 和 AFTER。\n", "\n", "============================================================\n", "最重要的任务\n", "============================================================\n", "\n", "判断这两个时间点之间，是否真的发生了：\n", "\n", "1. 放入衣物\n", "2. 取出衣物\n", "3. 没有衣物数量变化\n", "4. 无法确定\n", "\n", "============================================================\n", "非常重要的判断规则\n", "============================================================\n", "\n", "【规则1】\n", "不要计算“当前画面能看到几件衣服”。\n", "\n", "这个任务不是数当前可见衣物。\n", "\n", "【规则2】\n", "衣物发生翻转、移动、滚动、折叠、展开、被遮挡，\n", "都不能认为衣物数量发生变化。\n", "\n", "【规则3】\n", "如果 BEFORE 中可以看到一件衣物，\n", "而 AFTER 中因为其他衣物遮挡看不到，\n", "不能判断为取出。\n", "\n", "【规则4】\n", "只有有明显证据证明：\n", "一件新的衣物进入滚筒，\n", "才判断 put_in。\n", "\n", "【规则5】\n", "只有有明显证据证明：\n", "一件原本在滚筒中的衣物离开滚筒，\n", "才判断 take_out。\n", "\n", "【规则6】\n", "如果看到人的手、衣物正在被推进滚筒，\n", "这是非常强的 put_in 证据。\n", "\n", "【规则7】\n", "如果看到人的手把衣物从滚筒中拉出去，\n", "这是非常强的 take_out 证据。\n", "\n", "【规则8】\n", "如果只是滚筒中的衣服发生位置变化，\n", "返回 none。\n", "\n", "【规则9】\n", "严重遮挡时宁可 uncertain，\n", "不要猜。\n", "\n", "【规则10】\n", "一件衣物的多个部分只算一件。\n", "\n", "【规则11】\n", "颜色和纹理只作为辅助证据，\n", "不能单独因为颜色变化就判断新增衣物。\n", "\n", "例如：\n", "\n", "BEFORE：\n", "黑白格纹衣物 + 灰色衣物\n", "\n", "AFTER：\n", "灰色衣物 + 黑白格纹衣物\n", "\n", "这是 none。\n", "\n", "因为只是位置变化。\n", "\n", "------------------------------------------------------------\n", "\n", "另一个例子：\n", "\n", "BEFORE：\n", "只有灰色衣物\n", "\n", "AFTER：\n", "灰色衣物 + 明显出现新的黑白格纹衣物，\n", "并且能够看到黑白格纹衣物进入滚筒的过程\n", "\n", "这是：\n", "\n", "put_in\n", "\n", "------------------------------------------------------------\n", "\n", "如果无法确定：\n", "\n", "uncertain\n", "\n", "============================================================\n", "数量变化\n", "============================================================\n", "\n", "通常：\n", "\n", "put_in = +1\n", "take_out = -1\n", "none = 0\n", "uncertain = 0\n", "\n", "如果明显一次放入两件衣物，\n", "可以使用 +2。\n", "\n", "但是不要根据“画面里看到了两件衣服”\n", "就判断 +2。\n", "\n", "必须有证据证明是两件新的衣物进入。\n", "\n", "============================================================\n", "输出\n", "============================================================\n", "\n", "只输出纯 JSON，不要输出其他文字：\n", "\n", "{{\n", " \"event\": \"put_in|take_out|none|uncertain\",\n", " \"count_change\": 0,\n", " \"item_color\": null,\n", " \"item_pattern\": null,\n", " \"confidence\": 0.0,\n", " \"reason\": \"\"\n", "}}\n", "\n", "============================================================\n", "置信度\n", "============================================================\n", "\n", "真正看到衣物进入/离开：\n", "0.90-1.00\n", "\n", "有比较强的视觉证据：\n", "0.80-0.89\n", "\n", "存在明显遮挡：\n", "0.60-0.79\n", "\n", "无法判断：\n", "0.00-0.59\n", "\"\"\"\n", "\n", " # --------------------------------------------------------\n", " # Qwen3-VL messages\n", " # --------------------------------------------------------\n", "\n", " messages = [\n", " {\n", " \"role\": \"user\",\n", " \"content\": [\n", "\n", " {\n", " \"type\": \"image\",\n", " \"image\": before_frame,\n", " \"max_pixels\": max_pixels,\n", " },\n", "\n", " {\n", " \"type\": \"text\",\n", " \"text\": (\n", " f\"\\n\\n\"\n", " f\"上面第一张图片是 BEFORE，\"\n", " f\"时间约 {before_time:.1f}s。\"\n", " ),\n", " },\n", "\n", " {\n", " \"type\": \"image\",\n", " \"image\": after_frame,\n", " \"max_pixels\": max_pixels,\n", " },\n", "\n", " {\n", " \"type\": \"text\",\n", " \"text\": (\n", " f\"\\n\\n\"\n", " f\"上面第二张图片是 AFTER，\"\n", " f\"时间约 {after_time:.1f}s。\\n\\n\"\n", " + prompt\n", " ),\n", " },\n", "\n", " ],\n", " }\n", " ]\n", "\n", " # --------------------------------------------------------\n", " # chat template\n", " # --------------------------------------------------------\n", "\n", " text = processor.apply_chat_template(\n", " messages,\n", " tokenize=False,\n", " add_generation_prompt=True,\n", " )\n", "\n", " # --------------------------------------------------------\n", " # 这里直接把图片交给 processor\n", " #\n", " # 不再调用：\n", " # process_vision_info\n", " # --------------------------------------------------------\n", "\n", " inputs = processor(\n", " text=[text],\n", " images=[\n", " before_frame,\n", " after_frame\n", " ],\n", " padding=True,\n", " return_tensors=\"pt\",\n", " )\n", "\n", " inputs = inputs.to(DEVICE)\n", "\n", " # --------------------------------------------------------\n", " # Qwen推理\n", " # --------------------------------------------------------\n", "\n", " t0 = time.time()\n", "\n", " generated_ids = model.generate(\n", " **inputs,\n", " max_new_tokens=max_new_tokens,\n", " do_sample=False,\n", " )\n", "\n", " elapsed = time.time() - t0\n", "\n", " # --------------------------------------------------------\n", " # 去掉输入 token\n", " # --------------------------------------------------------\n", "\n", " generated_ids_trimmed = [\n", " out_ids[len(in_ids):]\n", " for in_ids, out_ids\n", " in zip(\n", " inputs.input_ids,\n", " generated_ids\n", " )\n", " ]\n", "\n", " output_text = processor.batch_decode(\n", " generated_ids_trimmed,\n", " skip_special_tokens=True,\n", " clean_up_tokenization_spaces=False,\n", " )[0]\n", "\n", " output_text = output_text.strip()\n", "\n", " # --------------------------------------------------------\n", " # 释放本次推理显存\n", " # --------------------------------------------------------\n", "\n", " del inputs\n", " del generated_ids\n", " del generated_ids_trimmed\n", "\n", " gc.collect()\n", "\n", " if torch.cuda.is_available():\n", " torch.cuda.empty_cache()\n", "\n", " return output_text, elapsed\n", "\n", "\n", "# ============================================================\n", "# 8. 主分析函数\n", "# ============================================================\n", "\n", "def run_analysis(\n", " video_path=\"/content/test02.mp4\",\n", "\n", " # 每隔多少秒取一个检测点\n", " sample_interval=1.0,\n", "\n", " # BEFORE 与 AFTER 相隔多少秒\n", " compare_interval=2.0,\n", "\n", " # 图像变化阈值\n", " #\n", " # 太低：\n", " # 会大量调用 Qwen\n", " #\n", " # 太高：\n", " # 可能漏掉缓慢放衣服\n", " #\n", " change_threshold=0.045,\n", "\n", " # Qwen图片最大像素\n", " max_pixels=360 * 420,\n", "\n", " # JSON很短，64已经足够\n", " max_new_tokens=64,\n", "\n", " # 连续确认次数\n", " required_confirmations=2,\n", "\n", " # 初始数量\n", " #\n", " # 你的 test02.mp4 是从空滚筒开始，\n", " # 所以设置 0。\n", " initial_count=0,\n", "\n", " show_raw=True,\n", "):\n", " \"\"\"\n", " 洗衣机衣物事件检测主函数。\n", " \"\"\"\n", "\n", " path = Path(video_path)\n", "\n", " if not path.exists():\n", "\n", " raise FileNotFoundError(\n", " f\"视频不存在：{video_path}\"\n", " )\n", "\n", " # --------------------------------------------------------\n", " # 视频信息\n", " # --------------------------------------------------------\n", "\n", " fps, frame_count, total = get_video_info(\n", " path\n", " )\n", "\n", " print()\n", " print(\"=\" * 70)\n", " print(\"🧺 Qwen3-VL 洗衣机衣物事件检测\")\n", " print(\"=\" * 70)\n", "\n", " print(\n", " f\"视频 : {path.name}\"\n", " )\n", "\n", " print(\n", " f\"FPS : {fps:.2f}\"\n", " )\n", "\n", " print(\n", " f\"帧数 : {frame_count:.0f}\"\n", " )\n", "\n", " print(\n", " f\"时长 : {total:.2f}s\"\n", " )\n", "\n", " print(\n", " f\"采样间隔 : {sample_interval:.2f}s\"\n", " )\n", "\n", " print(\n", " f\"比较时间跨度 : {compare_interval:.2f}s\"\n", " )\n", "\n", " print(\n", " f\"变化阈值 : {change_threshold:.3f}\"\n", " )\n", "\n", " print(\n", " f\"图像最大像素 : {max_pixels}\"\n", " )\n", "\n", " print(\n", " f\"最大输出 token : {max_new_tokens}\"\n", " )\n", "\n", " print(\n", " f\"事件确认次数 : {required_confirmations}\"\n", " )\n", "\n", " print(\n", " f\"初始衣物数量 : {initial_count}\"\n", " )\n", "\n", " print(\"=\" * 70)\n", "\n", " # --------------------------------------------------------\n", " # 打开视频\n", " # --------------------------------------------------------\n", "\n", " cap = cv2.VideoCapture(\n", " str(path)\n", " )\n", "\n", " if not cap.isOpened():\n", "\n", " raise RuntimeError(\n", " f\"无法打开视频：{path}\"\n", " )\n", "\n", " # --------------------------------------------------------\n", " # Python自己维护的累计状态\n", " # --------------------------------------------------------\n", "\n", " total_count = int(\n", " max(\n", " 0,\n", " initial_count\n", " )\n", " )\n", "\n", " events = []\n", "\n", " # 保存最终确认的衣物\n", " confirmed_items = []\n", "\n", " # --------------------------------------------------------\n", " # 事件确认器\n", " # --------------------------------------------------------\n", "\n", " stabilizer = EventStabilizer(\n", " required_confirmations\n", " )\n", "\n", " # --------------------------------------------------------\n", " # 第一帧\n", " #\n", " # 只是 baseline\n", " #\n", " # 不进行“放入”判断\n", " # --------------------------------------------------------\n", "\n", " baseline_frame = read_frame(\n", " cap,\n", " 0.0\n", " )\n", "\n", " if baseline_frame is None:\n", "\n", " cap.release()\n", "\n", " raise RuntimeError(\n", " \"无法读取视频第一帧\"\n", " )\n", "\n", " previous_sample_frame = baseline_frame\n", " previous_sample_time = 0.0\n", "\n", " print()\n", " print(\n", " \"🟡 0.0s：建立 baseline\"\n", " )\n", "\n", " print(\n", " f\" 当前累计数量 = {total_count}\"\n", " )\n", "\n", " # --------------------------------------------------------\n", " # 主循环\n", " # --------------------------------------------------------\n", "\n", " current_time = sample_interval\n", "\n", " sample_index = 0\n", "\n", " qwen_calls = 0\n", "\n", " while current_time <= total + 0.001:\n", "\n", " # ----------------------------------------------------\n", " # 读取当前帧\n", " # ----------------------------------------------------\n", "\n", " frame = read_frame(\n", " cap,\n", " min(\n", " current_time,\n", " total - 0.001\n", " )\n", " )\n", "\n", " if frame is None:\n", " break\n", "\n", " sample_index += 1\n", "\n", " # ----------------------------------------------------\n", " # 计算画面变化\n", " # ----------------------------------------------------\n", "\n", " diff = image_difference(\n", " previous_sample_frame,\n", " frame\n", " )\n", "\n", " print()\n", " print(\n", " f\"[{previous_sample_time:5.1f}s → \"\n", " f\"{current_time:5.1f}s]\"\n", " )\n", "\n", " print(\n", " f\" 画面变化 = {diff:.4f}\"\n", " )\n", "\n", " print(\n", " f\" 当前累计 = {total_count} 件\"\n", " )\n", "\n", " # ----------------------------------------------------\n", " # 是否需要调用 Qwen？\n", " # ----------------------------------------------------\n", "\n", " if diff < change_threshold:\n", "\n", " print(\n", " \" → 画面变化较小，跳过 Qwen\"\n", " )\n", "\n", " else:\n", "\n", " # ------------------------------------------------\n", " # BEFORE 时间\n", " #\n", " # 比当前时间往前 compare_interval\n", " # ------------------------------------------------\n", "\n", " before_time = max(\n", " 0.0,\n", " current_time - compare_interval\n", " )\n", "\n", " before_frame = read_frame(\n", " cap,\n", " before_time\n", " )\n", "\n", " if before_frame is None:\n", "\n", " print(\n", " \" → 无法读取 BEFORE，跳过\"\n", " )\n", "\n", " else:\n", "\n", " print(\n", " f\" → 疑似状态变化\"\n", " )\n", "\n", " print(\n", " f\" → BEFORE = \"\n", " f\"{before_time:.1f}s\"\n", " )\n", "\n", " print(\n", " f\" → AFTER = \"\n", " f\"{current_time:.1f}s\"\n", " )\n", "\n", " print(\n", " \" → 🧠 Qwen3-VL 判断中...\"\n", " )\n", "\n", " try:\n", "\n", " qwen_calls += 1\n", "\n", " raw, qwen_time = compare_frames(\n", " before_frame,\n", " frame,\n", " before_time,\n", " current_time,\n", " max_pixels=max_pixels,\n", " max_new_tokens=max_new_tokens,\n", " )\n", "\n", " if show_raw:\n", "\n", " print(\n", " f\" [raw] {raw}\"\n", " )\n", "\n", " print(\n", " f\" → Qwen耗时 = \"\n", " f\"{qwen_time:.2f}s\"\n", " )\n", "\n", " # ----------------------------------------\n", " # JSON\n", " # ----------------------------------------\n", "\n", " data = parse_json(\n", " raw\n", " )\n", "\n", " result = normalize_event(\n", " data\n", " )\n", "\n", " # ----------------------------------------\n", " # 无法解析\n", " # ----------------------------------------\n", "\n", " if result is None:\n", "\n", " print(\n", " \" → ⚠️ JSON解析失败\"\n", " )\n", "\n", " else:\n", "\n", " print(\n", " f\" → event = \"\n", " f\"{result['event']}\"\n", " )\n", "\n", " print(\n", " f\" → change = \"\n", " f\"{result['count_change']}\"\n", " )\n", "\n", " print(\n", " f\" → confidence = \"\n", " f\"{result['confidence']:.2f}\"\n", " )\n", "\n", " if result[\"item_color\"]:\n", "\n", " print(\n", " f\" → 颜色 = \"\n", " f\"{result['item_color']}\"\n", " )\n", "\n", " if result[\"item_pattern\"]:\n", "\n", " print(\n", " f\" → 纹理 = \"\n", " f\"{result['item_pattern']}\"\n", " )\n", "\n", " if result[\"reason\"]:\n", "\n", " print(\n", " f\" → 原因 = \"\n", " f\"{result['reason']}\"\n", " )\n", "\n", " # ------------------------------------\n", " # 事件确认\n", " # ------------------------------------\n", "\n", " confirmed = (\n", " stabilizer.update(\n", " result\n", " )\n", " )\n", "\n", " if confirmed is None:\n", "\n", " if result[\"event\"] in {\n", " \"put_in\",\n", " \"take_out\"\n", " }:\n", "\n", " print(\n", " \" → ⏳ \"\n", " \"候选事件，等待再次确认\"\n", " )\n", "\n", " else:\n", "\n", " # --------------------------------\n", " # 真正确认\n", " # --------------------------------\n", "\n", " change = (\n", " confirmed[\n", " \"count_change\"\n", " ]\n", " )\n", "\n", " old_count = (\n", " total_count\n", " )\n", "\n", " total_count += change\n", "\n", " # 不允许小于 0\n", " total_count = max(\n", " 0,\n", " total_count\n", " )\n", "\n", " # --------------------------------\n", " # 保存衣物信息\n", " # --------------------------------\n", "\n", " item = {\n", " \"color\":\n", " confirmed[\n", " \"item_color\"\n", " ],\n", "\n", " \"pattern\":\n", " confirmed[\n", " \"item_pattern\"\n", " ],\n", " }\n", "\n", " # --------------------------------\n", " # 放入\n", " # --------------------------------\n", "\n", " if (\n", " confirmed[\n", " \"event\"\n", " ]\n", " == \"put_in\"\n", " ):\n", "\n", " confirmed_items.append(\n", " item\n", " )\n", "\n", " # --------------------------------\n", " # 取出\n", " # --------------------------------\n", "\n", " elif (\n", " confirmed[\n", " \"event\"\n", " ]\n", " == \"take_out\"\n", " ):\n", "\n", " # 如果有已记录衣物，\n", " # 这里只做近似移除。\n", " if confirmed_items:\n", "\n", " confirmed_items.pop()\n", "\n", " # --------------------------------\n", " # 保存事件\n", " # --------------------------------\n", "\n", " event_record = {\n", "\n", " \"time\":\n", " current_time,\n", "\n", " \"event\":\n", " confirmed[\n", " \"event\"\n", " ],\n", "\n", " \"change\":\n", " change,\n", "\n", " \"old_count\":\n", " old_count,\n", "\n", " \"new_count\":\n", " total_count,\n", "\n", " \"color\":\n", " confirmed[\n", " \"item_color\"\n", " ],\n", "\n", " \"pattern\":\n", " confirmed[\n", " \"item_pattern\"\n", " ],\n", "\n", " \"confidence\":\n", " confirmed[\n", " \"confidence\"\n", " ],\n", "\n", " \"reason\":\n", " confirmed[\n", " \"reason\"\n", " ],\n", " }\n", "\n", " events.append(\n", " event_record\n", " )\n", "\n", " # --------------------------------\n", " # 打印醒目结果\n", " # --------------------------------\n", "\n", " print()\n", " print(\n", " \" \"\n", " + \"★\" * 55\n", " )\n", "\n", " if change > 0:\n", "\n", " print(\n", " \" 🟢 \"\n", " f\"确认放入 \"\n", " f\"{change} 件\"\n", " )\n", "\n", " else:\n", "\n", " print(\n", " \" 🔴 \"\n", " f\"确认取出 \"\n", " f\"{abs(change)} 件\"\n", " )\n", "\n", " print(\n", " f\" 数量：\"\n", " f\"{old_count}\"\n", " f\" → \"\n", " f\"{total_count}\"\n", " )\n", "\n", " print(\n", " f\" 衣物：\"\n", " f\"{confirmed['item_color']} \"\n", " f\"{confirmed['item_pattern']}\"\n", " )\n", "\n", " print(\n", " f\" 置信度：\"\n", " f\"{confirmed['confidence']:.2f}\"\n", " )\n", "\n", " print(\n", " \" \"\n", " + \"★\" * 55\n", " )\n", "\n", " except Exception as e:\n", "\n", " print(\n", " f\" → ❌ Qwen分析失败：\"\n", " f\"{type(e).__name__}: {e}\"\n", " )\n", "\n", " # ----------------------------------------------------\n", " # 保存当前帧\n", " # ----------------------------------------------------\n", "\n", " previous_sample_frame = frame\n", " previous_sample_time = current_time\n", "\n", " # ----------------------------------------------------\n", " # 时间推进\n", " # ----------------------------------------------------\n", "\n", " current_time += sample_interval\n", "\n", " # --------------------------------------------------------\n", " # 释放\n", " # --------------------------------------------------------\n", "\n", " cap.release()\n", "\n", " gc.collect()\n", "\n", " if torch.cuda.is_available():\n", " torch.cuda.empty_cache()\n", "\n", " # ========================================================\n", " # 最终结果\n", " # ========================================================\n", "\n", " print()\n", " print(\"=\" * 70)\n", " print(\"🧺 最终衣物状态\")\n", " print(\"=\" * 70)\n", "\n", " print(\n", " f\"【最终数量】 {total_count} 件\"\n", " )\n", "\n", " print(\n", " f\"【Qwen调用次数】 {qwen_calls} 次\"\n", " )\n", "\n", " print()\n", "\n", " print(\n", " \"【确认事件】\"\n", " )\n", "\n", " if not events:\n", "\n", " print(\n", " \" 无\"\n", " )\n", "\n", " else:\n", "\n", " for i, event in enumerate(\n", " events,\n", " 1\n", " ):\n", "\n", " if event[\"change\"] > 0:\n", "\n", " action = (\n", " f\"放入 \"\n", " f\"{event['change']} 件\"\n", " )\n", "\n", " else:\n", "\n", " action = (\n", " f\"取出 \"\n", " f\"{abs(event['change'])} 件\"\n", " )\n", "\n", " print(\n", " f\" {i}. \"\n", " f\"{event['time']:.1f}s \"\n", " f\"{action} \"\n", " f\"{event['old_count']} → \"\n", " f\"{event['new_count']} \"\n", " f\"颜色={event['color']} \"\n", " f\"置信度={event['confidence']:.2f}\"\n", " )\n", "\n", " print()\n", "\n", " print(\n", " \"【累计衣物信息】\"\n", " )\n", "\n", " if not confirmed_items:\n", "\n", " print(\n", " \" 暂无\"\n", " )\n", "\n", " else:\n", "\n", " for i, item in enumerate(\n", " confirmed_items,\n", " 1\n", " ):\n", "\n", " print(\n", " f\" {i}. \"\n", " f\"{item['color']} \"\n", " f\"{item['pattern']}\"\n", " )\n", "\n", " print(\"=\" * 70)\n", "\n", " return {\n", "\n", " \"final_count\":\n", " total_count,\n", "\n", " \"events\":\n", " events,\n", "\n", " \"items\":\n", " confirmed_items,\n", "\n", " \"qwen_calls\":\n", " qwen_calls,\n", " }" ] }, { "cell_type": "code", "execution_count": 9, "id": "25e29805", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "======================================================================\n", "🧺 Qwen3-VL 洗衣机衣物事件检测\n", "======================================================================\n", "视频 : test02.mp4\n", "FPS : 29.97\n", "帧数 : 420\n", "时长 : 14.02s\n", "采样间隔 : 1.00s\n", "比较时间跨度 : 2.00s\n", "变化阈值 : 0.045\n", "图像最大像素 : 151200\n", "最大输出 token : 64\n", "事件确认次数 : 2\n", "初始衣物数量 : 0\n", "======================================================================\n", "\n", "🟡 0.0s：建立 baseline\n", " 当前累计数量 = 0\n", "\n", "[ 0.0s → 1.0s]\n", " 画面变化 = 0.2496\n", " 当前累计 = 0 件\n", " → 疑似状态变化\n", " → BEFORE = 0.0s\n", " → AFTER = 1.0s\n", " → 🧠 Qwen3-VL 判断中...\n", " → ❌ Qwen分析失败：OutOfMemoryError: CUDA out of memory. Tried to allocate 3.97 GiB. GPU 0 has a total capacity of 14.56 GiB of which 3.73 GiB is free. Including non-PyTorch memory, this process has 10.83 GiB memory in use. Of the allocated memory 10.56 GiB is allocated by PyTorch, and 146.42 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf)\n", "\n", "[ 1.0s → 2.0s]\n", " 画面变化 = 0.1371\n", " 当前累计 = 0 件\n", " → 疑似状态变化\n", " → BEFORE = 0.0s\n", " → AFTER = 2.0s\n", " → 🧠 Qwen3-VL 判断中...\n" ] }, { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mCannot execute code, session has been disposed. Please try restarting the Kernel." ] }, { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mCannot execute code, session has been disposed. Please try restarting the Kernel. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "# ============================================================\n", "# 代码块 4：执行优化后的衣物事件分析\n", "# ============================================================\n", "\n", "result = run_analysis(\n", " video_path=\"/content/test02.mp4\",\n", "\n", " # 每 1 秒检查一次\n", " sample_interval=1.0,\n", "\n", " # 比较当前画面与 2 秒前画面\n", " compare_interval=2.0,\n", "\n", " # 画面变化超过这个值才调用 Qwen\n", " change_threshold=0.045,\n", "\n", " # 你的 T4 可以先保持这个\n", " max_pixels=360 * 420,\n", "\n", " # JSON 输出很短，64 足够\n", " max_new_tokens=64,\n", "\n", " # 同一个事件至少连续确认 2 次\n", " required_confirmations=2,\n", "\n", " # test02.mp4 是从空滚筒开始\n", " initial_count=0,\n", "\n", " show_raw=True,\n", ")\n", "\n", "\n", "print(\"\\n\")\n", "print(\"=\" * 70)\n", "print(\"最终结果摘要\")\n", "print(\"=\" * 70)\n", "\n", "print(\n", " f\"最终衣物数量：{result['final_count']} 件\"\n", ")\n", "\n", "print(\n", " f\"Qwen调用次数：{result['qwen_calls']} 次\"\n", ")\n", "\n", "print()\n", "\n", "print(\"确认的事件：\")\n", "\n", "for i, event in enumerate(\n", " result[\"events\"],\n", " 1\n", "):\n", "\n", " if event[\"change\"] > 0:\n", " action = f\"放入 {event['change']} 件\"\n", " else:\n", " action = f\"取出 {abs(event['change'])} 件\"\n", "\n", " print(\n", " f\"{i}. \"\n", " f\"{event['time']:.1f}s \"\n", " f\"{action} | \"\n", " f\"{event['old_count']} → \"\n", " f\"{event['new_count']} | \"\n", " f\"{event['color']} \"\n", " f\"{event['pattern']} | \"\n", " f\"confidence={event['confidence']:.2f}\"\n", " )\n", "\n", "print(\"=\" * 70)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 5 }