Lcsts.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e172ee14-7693-4c0a-8079-f9cc0cba2c50",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-19T09:52:47.028184Z",
     "iopub.status.busy": "2024-10-19T09:52:47.027701Z",
     "iopub.status.idle": "2024-10-19T09:52:50.265915Z",
     "shell.execute_reply": "2024-10-19T09:52:50.265156Z",
     "shell.execute_reply.started": "2024-10-19T09:52:47.028152Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from datasets import Dataset\n",
    "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, DataCollatorForSeq2Seq, Seq2SeqTrainer, Seq2SeqTrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "10877255-d316-4300-ab4f-d855dbe827e1",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-19T09:52:52.580431Z",
     "iopub.status.busy": "2024-10-19T09:52:52.579653Z",
     "iopub.status.idle": "2024-10-19T09:52:59.549459Z",
     "shell.execute_reply": "2024-10-19T09:52:59.548826Z",
     "shell.execute_reply.started": "2024-10-19T09:52:52.580399Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'title', 'content'],\n",
       "        num_rows: 4900\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['id', 'title', 'content'],\n",
       "        num_rows: 100\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "import random\n",
    "# 读取train.json文件\n",
    "# train_data = []\n",
    "\n",
    "# 逐行读取train.json文件\n",
    "# with open('./LCSTS/train.json', 'r', encoding='utf-8') as f:\n",
    "#     for line in f:\n",
    "#         data = json.loads(line)\n",
    "#         train_data.append(data)\n",
    "        \n",
    "# 读取train.json文件并随机选择5000条数据\n",
    "train_data = []\n",
    "num_samples = 5000\n",
    "\n",
    "with open('./LCSTS/train.json', 'r', encoding='utf-8') as f:\n",
    "    for line in f:\n",
    "        data = json.loads(line)\n",
    "        train_data.append(data)\n",
    "\n",
    "# 随机选择5000条数据\n",
    "train_data = random.sample(train_data, num_samples)        \n",
    "\n",
    "\n",
    "# # 读取dev.json文件\n",
    "# with open('./LCSTS/dev.json', 'r', encoding='utf-8') as f:\n",
    "#     dev_data = json.load(f)\n",
    "\n",
    "# 处理数据集，提取id、summary和content\n",
    "def process_data(data):\n",
    "    processed_data = []\n",
    "    for item in data:\n",
    "        processed_item = {\n",
    "            \"id\": item[\"id\"],\n",
    "            \"summary\": item[\"summary\"],\n",
    "            \"content\": item[\"content\"]\n",
    "        }\n",
    "        processed_data.append(processed_item)\n",
    "    return processed_data\n",
    "\n",
    "# # 处理train数据\n",
    "processed_train_data = process_data(train_data)\n",
    "\n",
    "# # 处理dev数据\n",
    "# processed_dev_data = process_data(dev_data)\n",
    "\n",
    "# 将数据整理成Dataset格式\n",
    "train_dataset = Dataset.from_dict({\"id\": [item[\"id\"] for item in processed_train_data],\n",
    "                                   \"title\": [item[\"summary\"] for item in processed_train_data],\n",
    "                                   \"content\": [item[\"content\"] for item in processed_train_data]})\n",
    "\n",
    "# # 划分训练集和测试集\n",
    "ds = train_dataset.train_test_split(100, seed=42)\n",
    "ds\n",
    "\n",
    "# train_size = int(0.7 * len(train_dataset))\n",
    "# val_size = int(0.1 * len(train_dataset))\n",
    "# test_size = len(train_dataset) - train_size - val_size\n",
    "\n",
    "# ds = train_dataset.train_test_split(train_size=train_size, test_size=test_size, seed=42)\n",
    "# ds\n",
    "\n",
    "# 现在你可以像之前处理nlpcc_2017数据集一样使用train_dataset和test_dataset了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b122c151-2079-46d1-a24e-6309b796d874",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T07:17:13.396700Z",
     "iopub.status.busy": "2024-10-03T07:17:13.396221Z",
     "iopub.status.idle": "2024-10-03T07:17:13.402539Z",
     "shell.execute_reply": "2024-10-03T07:17:13.401592Z",
     "shell.execute_reply.started": "2024-10-03T07:17:13.396675Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 455698,\n",
       " 'title': '宝龙许华芳：未来十年是商业地产黄金十年',\n",
       " 'content': '当前电商来势汹汹，对传统商业的冲击已经看得见。许华芳也同意电商带来的变化，称电商分流掉一部分传统商业是毋庸置疑的。不过他也明确表示，“但这并不意味着它(电商)可以取代传统商业。'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds[\"train\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cae87334-2990-4cb5-8702-ba09154575dc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T07:17:16.179042Z",
     "iopub.status.busy": "2024-10-03T07:17:16.178566Z",
     "iopub.status.idle": "2024-10-03T07:17:16.183856Z",
     "shell.execute_reply": "2024-10-03T07:17:16.183267Z",
     "shell.execute_reply.started": "2024-10-03T07:17:16.179013Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 757222,\n",
       " 'title': '路虎发现运动版配置曝光19日国内首发',\n",
       " 'content': '近日我们从官方获悉，路虎发现运动版(DiscoverySport)将于9月19日在国内正式亮相并公布中文新名称及官方预售价。与此同时的是我们才相关渠道获取到了一份相关配置表。根据我们的了解，路虎发现运动版将于2015年初上市发售。'}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds[\"train\"][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "18b5919c-a787-408e-8569-9c2559ac0871",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-01T15:01:13.047577Z",
     "iopub.status.busy": "2024-10-01T15:01:13.047155Z",
     "iopub.status.idle": "2024-10-01T15:01:13.810670Z",
     "shell.execute_reply": "2024-10-01T15:01:13.809922Z",
     "shell.execute_reply.started": "2024-10-01T15:01:13.047554Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model2/ZhipuAI/glm-large-chinese\n"
     ]
    }
   ],
   "source": [
    "#模型下载\n",
    "from modelscope import snapshot_download\n",
    "# model_dir = snapshot_download('langboat/mengzi-t5-base',revision='v1.0.0')\n",
    "model_dir = snapshot_download('ZhipuAI/glm-large-chinese',cache_dir='model2')\n",
    "print(model_dir)\n",
    "\n",
    "# #模型下载\n",
    "# from modelscope import snapshot_download\n",
    "# model_dir = snapshot_download('ZhipuAI/glm-large-chinese')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e19c16b4-ccbf-4737-ae29-ed14bfa4fb77",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-01T15:01:25.227104Z",
     "iopub.status.busy": "2024-10-01T15:01:25.226636Z",
     "iopub.status.idle": "2024-10-01T15:01:25.398377Z",
     "shell.execute_reply": "2024-10-01T15:01:25.397650Z",
     "shell.execute_reply.started": "2024-10-01T15:01:25.227071Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 对于高版本的Transformers加载会报错，需要修改源码\n",
    "# 文件地址 ~/.cache\\huggingface\\modules\\transformers_modules\\THUDM\\glm-large-chinese\\230f54e413fab4bc8f29bd3508aab301d757ef3e\\tokenization_glm.py\n",
    "# 231行 super().__init__(**kwargs) 移动至 235行，放至self.sp_model.Load(vocab_file)的后面一行\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"model2/ZhipuAI/glm-large-chinese\", trust_remote_code=True)\n",
    "# tokenizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "aa12957e-ce64-4da1-b783-3fcb84c1fecd",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-01T15:02:47.493801Z",
     "iopub.status.busy": "2024-10-01T15:02:47.493381Z",
     "iopub.status.idle": "2024-10-01T15:02:47.498300Z",
     "shell.execute_reply": "2024-10-01T15:02:47.497468Z",
     "shell.execute_reply.started": "2024-10-01T15:02:47.493777Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def process_func(exmaples):\n",
    "    contents = [\"摘要生成: \\n\" + e + tokenizer.mask_token for e in exmaples[\"content\"]]\n",
    "    inputs = tokenizer(contents, max_length=256, truncation=True, padding=\"max_length\", return_tensors=\"pt\")\n",
    "    inputs = tokenizer.build_inputs_for_generation(inputs, targets=exmaples['title'], padding=True, max_gen_length=64)\n",
    "    return inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "89cbe6be-2654-47f3-8527-c2f2b9e9410c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-01T15:02:49.729289Z",
     "iopub.status.busy": "2024-10-01T15:02:49.728895Z",
     "iopub.status.idle": "2024-10-01T15:03:12.991783Z",
     "shell.execute_reply": "2024-10-01T15:03:12.991191Z",
     "shell.execute_reply.started": "2024-10-01T15:02:49.729264Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c12159a4d9c42ad8909c2821c146e94",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/4500 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e69b8b58e7c74a489d12230614b54956",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/500 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_ids', 'position_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 4500\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['input_ids', 'position_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 500\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_ds = ds.map(process_func, batched=True, remove_columns=ds[\"train\"].column_names)\n",
    "tokenized_ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "aedf6b13-dbd0-4bd5-b47b-5d3cbed58dff",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-01T15:03:15.489922Z",
     "iopub.status.busy": "2024-10-01T15:03:15.489579Z",
     "iopub.status.idle": "2024-10-01T15:03:15.526981Z",
     "shell.execute_reply": "2024-10-01T15:03:15.526388Z",
     "shell.execute_reply.started": "2024-10-01T15:03:15.489903Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[CLS] 摘要生成: 吴敬琏警示道,目前宏观与微观经济两方面正开始显现危机。吴敬琏指出,随着资产负债率的日益攀升,一些地区出现的危机很快传到其他地方,比如温州、苏南等地相继出现的“跑路”事件,包括6月的钱荒,其实都是给人们发预警信号。[MASK]<|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|> <|startofpiece|> 三中全会临近吴敬琏警示危机:资产负债率太高 <|endofpiece|> <|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|>'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(tokenized_ds[\"train\"][0][\"input_ids\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c6141cbb-ae3e-4642-b608-9674d61e6682",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-01T15:03:32.804314Z",
     "iopub.status.busy": "2024-10-01T15:03:32.803925Z",
     "iopub.status.idle": "2024-10-01T15:03:32.842410Z",
     "shell.execute_reply": "2024-10-01T15:03:32.841661Z",
     "shell.execute_reply.started": "2024-10-01T15:03:32.804291Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " 603,\n",
       " 43378,\n",
       " 25781,\n",
       " 19048,\n",
       " 44720,\n",
       " 44957,\n",
       " 48354,\n",
       " 24511,\n",
       " 3129,\n",
       " 43383,\n",
       " 32701,\n",
       " 43923,\n",
       " 7511,\n",
       " 50007,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_ds[\"train\"][0][\"labels\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "09429885-8bec-4bc3-9021-853ea7e324e8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-01T15:03:45.086234Z",
     "iopub.status.busy": "2024-10-01T15:03:45.085898Z",
     "iopub.status.idle": "2024-10-01T15:03:45.122778Z",
     "shell.execute_reply": "2024-10-01T15:03:45.122124Z",
     "shell.execute_reply.started": "2024-10-01T15:03:45.086214Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64]]\n"
     ]
    }
   ],
   "source": [
    "print(tokenized_ds[\"train\"][0][\"position_ids\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7615f77e-c333-421a-8615-135d34aa6499",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-01T15:03:55.578454Z",
     "iopub.status.busy": "2024-10-01T15:03:55.578112Z",
     "iopub.status.idle": "2024-10-01T15:03:57.490546Z",
     "shell.execute_reply": "2024-10-01T15:03:57.489682Z",
     "shell.execute_reply.started": "2024-10-01T15:03:55.578435Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = AutoModelForSeq2SeqLM.from_pretrained(\"model2/ZhipuAI/glm-large-chinese\", trust_remote_code=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "1933e77b-76ec-489d-b9f5-d7a364a22651",
   "metadata": {
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2024-10-01T15:05:33.031175Z",
     "iopub.status.busy": "2024-10-01T15:05:33.030783Z",
     "iopub.status.idle": "2024-10-01T15:24:51.643448Z",
     "shell.execute_reply": "2024-10-01T15:24:51.642765Z",
     "shell.execute_reply.started": "2024-10-01T15:05:33.031154Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Detected kernel version 4.19.91, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2024-10-01 23:05:36,427] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "df: /root/.triton/autotune: 没有那个文件或目录\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[93m [WARNING] \u001b[0m Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH\n",
      "\u001b[93m [WARNING] \u001b[0m sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3\n",
      "\u001b[93m [WARNING] \u001b[0m using untested triton version (2.3.1), only 1.0.0 is known to be compatible\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='140' max='140' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [140/140 19:07, Epoch 0/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>3.200500</td>\n",
       "      <td>2.899231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>3.075400</td>\n",
       "      <td>2.816801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>2.905100</td>\n",
       "      <td>2.798641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>2.922300</td>\n",
       "      <td>2.805028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>3.018500</td>\n",
       "      <td>2.776683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>2.937300</td>\n",
       "      <td>2.763617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56</td>\n",
       "      <td>2.822900</td>\n",
       "      <td>2.764317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>64</td>\n",
       "      <td>2.831700</td>\n",
       "      <td>2.754272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>72</td>\n",
       "      <td>2.843800</td>\n",
       "      <td>2.747238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80</td>\n",
       "      <td>2.902200</td>\n",
       "      <td>2.735222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>88</td>\n",
       "      <td>2.905900</td>\n",
       "      <td>2.731628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>96</td>\n",
       "      <td>2.948700</td>\n",
       "      <td>2.728689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>104</td>\n",
       "      <td>2.873600</td>\n",
       "      <td>2.725428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>112</td>\n",
       "      <td>2.945500</td>\n",
       "      <td>2.716777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>120</td>\n",
       "      <td>2.889400</td>\n",
       "      <td>2.714005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>128</td>\n",
       "      <td>2.783500</td>\n",
       "      <td>2.713423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>136</td>\n",
       "      <td>2.947900</td>\n",
       "      <td>2.711239</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=140, training_loss=2.9211345400129045, metrics={'train_runtime': 1152.5461, 'train_samples_per_second': 3.904, 'train_steps_per_second': 0.121, 'total_flos': 3041186650521600.0, 'train_loss': 2.9211345400129045, 'epoch': 0.9955555555555555})"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import torch\n",
    "# from datasets import Dataset\n",
    "# from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, DataCollatorForSeq2Seq, Seq2SeqTrainer, Seq2SeqTrainingArguments\n",
    "\n",
    "# # 加载数据集\n",
    "# ds = Dataset.load_from_disk(\"./nlpcc_2017/\")\n",
    "# ds = ds.train_test_split(100, seed=42)\n",
    "\n",
    "# # 加载分词器\n",
    "# tokenizer = AutoTokenizer.from_pretrained(\"model2/ZhipuAI/glm-large-chinese\", trust_remote_code=True)\n",
    "\n",
    "# # 数据预处理函数\n",
    "# def process_func(examples):\n",
    "#     contents = [\"摘要生成: \\n\" + e + tokenizer.mask_token for e in examples[\"content\"]]\n",
    "#     inputs = tokenizer(contents, max_length=384, truncation=True, padding=\"max_length\", return_tensors=\"pt\")\n",
    "#     inputs = tokenizer.build_inputs_for_generation(inputs, targets=examples['title'], padding=True, max_gen_length=64)\n",
    "#     return inputs\n",
    "\n",
    "# # 处理数据集\n",
    "# tokenized_ds = ds.map(process_func, batched=True, remove_columns=ds[\"train\"].column_names)\n",
    "\n",
    "# # 加载模型\n",
    "# model = AutoModelForSeq2SeqLM.from_pretrained(\"model2/ZhipuAI/glm-large-chinese\", trust_remote_code=True)\n",
    "\n",
    "# 设置训练参数\n",
    "args = Seq2SeqTrainingArguments(\n",
    "    output_dir=\"./summary_glm\",\n",
    "    per_device_train_batch_size=4,\n",
    "    per_device_eval_batch_size=8,\n",
    "    gradient_accumulation_steps=8,\n",
    "    logging_steps=8,\n",
    "    num_train_epochs=1,\n",
    "    save_total_limit=1,  # 只保留最近的一个模型\n",
    "    save_steps=512,       # 每500步保存一次模型\n",
    "    # eval_strategy=\"steps\",  # 评估的策略\n",
    "    load_best_model_at_end=True,  # 在训练结束时加载最佳模型\n",
    "    eval_strategy=\"steps\",\n",
    ")\n",
    "\n",
    "# 创建训练器，添加 eval_dataset\n",
    "trainer = Seq2SeqTrainer(\n",
    "    args=args,\n",
    "    model=model,\n",
    "    train_dataset=tokenized_ds[\"train\"],  # 训练集\n",
    "    eval_dataset=tokenized_ds['test'],  # 评估集\n",
    "    tokenizer=tokenizer,\n",
    ")\n",
    "# 开始训练\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "26647095-1792-4ac9-9fe8-deda4381b144",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-02T07:27:34.230173Z",
     "iopub.status.busy": "2024-10-02T07:27:34.229662Z",
     "iopub.status.idle": "2024-10-02T07:27:36.021379Z",
     "shell.execute_reply": "2024-10-02T07:27:36.020615Z",
     "shell.execute_reply.started": "2024-10-02T07:27:34.230142Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('./summary_glm/checkpoint-140/tokenizer_config.json',\n",
       " './summary_glm/checkpoint-140/special_tokens_map.json',\n",
       " './summary_glm/checkpoint-140/cog-pretrain.model',\n",
       " './summary_glm/checkpoint-140/added_tokens.json')"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# # 保存模型和分词器\n",
    "# trainer.save_model(args.output_dir)  # 保存模型\n",
    "# tokenizer.save_pretrained(args.output_dir)  # 保存分词器\n",
    "\n",
    "# 保存模型\n",
    "path='./summary_glm/checkpoint-140'\n",
    "\n",
    "model.save_pretrained('./summary_glm/checkpoint-140')\n",
    "# 保存分词器\n",
    "tokenizer.save_pretrained('./summary_glm/checkpoint-140')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e1f52f15-438d-4e17-b1f1-0b88ae2683f4",
   "metadata": {
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2024-10-19T06:10:59.212104Z",
     "iopub.status.busy": "2024-10-19T06:10:59.211610Z",
     "iopub.status.idle": "2024-10-19T06:11:00.109103Z",
     "shell.execute_reply": "2024-10-19T06:11:00.108229Z",
     "shell.execute_reply.started": "2024-10-19T06:10:59.212075Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GLMForConditionalGeneration(\n",
       "  (glm): GLMModel(\n",
       "    (word_embeddings): VocabEmbedding()\n",
       "    (transformer): GLMStack(\n",
       "      (embedding_dropout): Dropout(p=0.1, inplace=False)\n",
       "      (position_embeddings): Embedding(1025, 1024)\n",
       "      (block_position_embeddings): Embedding(1025, 1024)\n",
       "      (layers): ModuleList(\n",
       "        (0-23): 24 x GLMBlock(\n",
       "          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          (attention): SelfAttention(\n",
       "            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n",
       "            (attention_dropout): Dropout(p=0.1, inplace=False)\n",
       "            (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            (output_dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): MLP(\n",
       "            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (final_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 加载模型进行推理\n",
    "\n",
    "path='./checkpoint-140'\n",
    "# 检查是否有可用的 GPU\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(path,trust_remote_code=True)\n",
    "tokenizer = AutoTokenizer.from_pretrained(path,trust_remote_code=True)\n",
    "# 将模型移动到 GPU\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4e25784c-4481-4c50-bb5e-d044115caea3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T04:41:40.438372Z",
     "iopub.status.busy": "2024-10-09T04:41:40.437889Z",
     "iopub.status.idle": "2024-10-09T04:41:41.314180Z",
     "shell.execute_reply": "2024-10-09T04:41:41.313496Z",
     "shell.execute_reply.started": "2024-10-09T04:41:40.438342Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
      "Setting `pad_token_id` to `eos_token_id`:None for open-end generation.\n",
      "The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'[CLS] 摘要生成: 中国10月工业、投资和消费同比增速齐齐下滑,其中工业增速再次回落至8%以下,投资增速受房地产市场疲弱影响更是录得近13年最低,反映经济下行压力仍大,经济调整态势不改,料四季度将进一步放缓。[MASK]<|endoftext|> <|startofpiece|> 中金:经济“二次探底”进程仍在进行中 <|endofpiece|>'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# with torch.no_grad():\n",
    "#     output = model.generate(**inputs, max_new_tokens=64, eos_token_id=tokenizer.eop_token_id, do_sample=True)\n",
    "\n",
    "# # 解码并输出摘要\n",
    "# summary = tokenizer.decode(output[0].tolist(), skip_special_tokens=True)\n",
    "# print(\"生成的摘要:\", summary)\n",
    "\n",
    "\n",
    "# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "# # 将模型移动到 GPU\n",
    "# model.to(device)\n",
    "\n",
    "input_text = ds[\"test\"][0][\"content\"]\n",
    "inputs = tokenizer(\"摘要生成: \\n\" + input_text + tokenizer.mask_token, return_tensors=\"pt\")\n",
    "inputs = tokenizer.build_inputs_for_generation(inputs, max_gen_length=64)\n",
    "# inputs = inputs.to(\"cpu\")\n",
    "# 将输入张量移动到与模型相同的设备\n",
    "inputs = {key: value.to(device) for key, value in inputs.items()}\n",
    "\n",
    "\n",
    "output = model.generate(**inputs, max_new_tokens=64, eos_token_id=tokenizer.eop_token_id, do_sample=True)\n",
    "tokenizer.decode(output[0].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4fe2d381-8d0a-4871-9a71-fa60f8798f9c",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-19T06:11:04.631490Z",
     "iopub.status.busy": "2024-10-19T06:11:04.631005Z",
     "iopub.status.idle": "2024-10-19T06:11:04.639099Z",
     "shell.execute_reply": "2024-10-19T06:11:04.638109Z",
     "shell.execute_reply.started": "2024-10-19T06:11:04.631460Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "model = model.eval()\n",
    "\n",
    "def predict_test():\n",
    "    predict = []\n",
    "    with torch.inference_mode():\n",
    "        for d in ds[\"test\"]:\n",
    "            inputs = tokenizer(\"摘要生成: \\n\" + d[\"content\"] + tokenizer.mask_token, return_tensors=\"pt\")\n",
    "            inputs = tokenizer.build_inputs_for_generation(inputs, max_gen_length=64)\n",
    "            inputs = inputs.to(\"cuda\")\n",
    "            output = model.generate(**inputs, max_new_tokens=64, eos_token_id=tokenizer.eop_token_id, do_sample=True)\n",
    "            predict.append(tokenizer.decode(output[0].tolist()).split(\"<|startofpiece|>\")[1].replace(\"<|endofpiece|>\", \"\").strip())\n",
    "            print(\"curID:\", len(predict))\n",
    "    return predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "35ce900b-bb29-4e83-9e27-442d10750c42",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-19T10:01:55.960794Z",
     "iopub.status.busy": "2024-10-19T10:01:55.960305Z",
     "iopub.status.idle": "2024-10-19T10:01:56.563943Z",
     "shell.execute_reply": "2024-10-19T10:01:56.563253Z",
     "shell.execute_reply.started": "2024-10-19T10:01:55.960749Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "专家表示，从2...。\n",
      "\n",
      "curID: 1\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "相关负责人表示界旅游城市排名是国际上首个以城市综合旅游服务能力作为主要依据的排行榜。\n",
      "\n",
      "curID: 2\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "在众多游戏发展中，中国移动游戏市场的发展最为耀眼，截至9月份移动游戏用户接近3亿人。\n",
      "\n",
      "curID: 3\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "周末是欣赏美术作品的好时机，昨日，两个书画、篆刻作品展开幕：“文以载道·湖南中国画学会诗书画印邀请展”在湖南省画院美术馆开幕，“山东·湖南篆刻联展”在长沙简牍博物馆开幕，两个展览均持续到12月15日，向市民免费开放。\n",
      "\n",
      "curID: 4\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "这次“海巡01轮”出海，携带了多套海测设备，包括1套黑匣子搜寻仪、2个潜水机器人、1套旁侧声纳和磁力仪，这些设备完全是国外引进的，水下最佳探测范围是2000米，最大探测范围是5000米。\n",
      "\n",
      "curID: 5\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "”在约四十分钟的演说中，安倍首相连喊七次“世界第一”。\n",
      "\n",
      "curID: 6\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "由于可能产生的污染问题，正引起来自民间的强烈反对，包括140余名离退休老干部。\n",
      "\n",
      "curID: 7\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "纳智捷，这个来自于台湾的汽车品牌随着一部《痞子英雄》开始被国内消费者所认识，通过一部大空间、高配置的SUV让消费者开始了解这个品牌，之后其又慢慢地将MPV和轿车等车型引入国内汽车市场。\n",
      "\n",
      "curID: 8\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "二手车电商O2O，不仅面临着电商流量、产品、IT系统的线上问题，更考验其线下渠道、检测团队管理的能力。\n",
      "\n",
      "curID: 9\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "电力增速走势对经济波动反映准确，对工业增加值变动更为敏感。\n",
      "\n",
      "curID: 10\n",
      "---------开始----------------------------------------\n",
      "迭代14次，收敛\n",
      "最大的1个值： [1.61056694]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "①英特尔会重新对智能手机感兴趣。\n",
      "\n",
      "curID: 11\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "这部法规在保护老年人权益方面作出了多项规定，包括禁止有独立生活能力的子女“啃老”，对于虐待老人的养老机构最高可罚款３万元。\n",
      "\n",
      "curID: 12\n",
      "---------开始----------------------------------------\n",
      "迭代11次，收敛\n",
      "最大的1个值： [1.12931198]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "谷歌被控侵犯七项专利权。\n",
      "\n",
      "curID: 13\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "韩联社援引“朝鲜总联”机关报《朝鲜新报》4月25日报道称，朝鲜计划将位于朝鲜咸镜南道的端川港发展成为国际贸易港口，为实现经济建设和核武建设并进的奋斗目标打下经济基础。\n",
      "\n",
      "curID: 14\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "与此同时，辉瑞从本月初开始真正一分为三，其中包括一个仿制药部门。\n",
      "\n",
      "curID: 15\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "12月13日晚19点35分，中国男足将在湖南郴州与吉尔吉斯斯坦队进行热身赛，由于球队刚刚集训一周的时间，考虑到球员们的身体情况，佩兰也决定在这场比赛多安排一些球员出场，在战前进行的发布会上佩兰表示，会在上、下半时采取两套不同的阵容。\n",
      "\n",
      "curID: 16\n",
      "---------开始----------------------------------------\n",
      "迭代17次，收敛\n",
      "最大的1个值： [1.13969179]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "凭借过去8年的完美答卷，默克尔帮助联盟党在9月22日的联邦议院选举中夺得超过41%的选票。\n",
      "\n",
      "curID: 17\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "日本东京都知事石原慎太郎在24日的记者会上就都政府以“购买”钓鱼岛为由向中央政府申请登岛进行“现场调查”一事表示，已申请的“现场调查”成行后，都政府将于10月再次实施所谓“调查”，届时他也将前往。\n",
      "\n",
      "curID: 18\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "把茅台和腐败简单联系起来毫无道理。\n",
      "\n",
      "curID: 19\n",
      "---------开始----------------------------------------\n",
      "迭代43次，收敛\n",
      "最大的1个值： [1.36660071]\n",
      "对应的索引： [2]\n",
      "结果：\n",
      "截至26日两市共235个交易日，平均每一个交易日减持参考总市值约为6.7亿元，有3次减持市值金额超过10亿元。\n",
      "\n",
      "curID: 20\n",
      "---------开始----------------------------------------\n",
      "迭代12次，收敛\n",
      "最大的1个值： [1.1904904]\n",
      "对应的索引： [3]\n",
      "结果：\n",
      "相比，北京净租金回报率仅仅3%左右。\n",
      "\n",
      "curID: 21\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "#ETtoday新聞雲#報導：　雙眼皮眼鏡好高招。\n",
      "\n",
      "curID: 22\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "形式主义，一个干部模样的胖子被一堆公章文件包围着；官僚主义，胖子坐在一个高高椅子上，另外一个群众站在下面仰望他；享乐主义，胖子躺进了酒杯里脸喝得红扑扑；奢靡之风，摆满大餐的桌前，胖子大吃大喝。\n",
      "\n",
      "curID: 23\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "由于时间上的限制，小股东们如今已经失去了补救的机会。\n",
      "\n",
      "curID: 24\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "它以人类汗液、分泌物、脱落的皮屑为营养源，最喜欢温暖潮湿阴暗的环境（20~30摄氏度的温度、50%~70%的湿度，与人类生活环境相似），且繁殖能力惊人。\n",
      "\n",
      "curID: 25\n",
      "---------开始----------------------------------------\n",
      "迭代18次，收敛\n",
      "最大的1个值： [1.15378506]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "“现在租房太贵了，40平方米的住房要2000多元。\n",
      "\n",
      "curID: 26\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "在随后2个月的行程中，他们将横跨亚欧大陆，进行一次行程长达15000公里的行进式采访。\n",
      "\n",
      "curID: 27\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "北京直飞曼谷，免费WIFI境外无线网，特色美食，舒适酒店贴心奉上。\n",
      "\n",
      "curID: 28\n",
      "---------开始----------------------------------------\n",
      "迭代10次，收敛\n",
      "最大的1个值： [1.05707384]\n",
      "对应的索引： [2]\n",
      "结果：\n",
      "2月起，上图启动iPad借阅服务，首次70台iPad2整装待发，可供持参考外借证的读者借阅，时长是28天。\n",
      "\n",
      "curID: 29\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [2]\n",
      "结果：\n",
      "15组监控抓拍摄像头分布在S115线机场高速公路双向车道（阿勒泰路与木材厂交界处—小地窝堡立交桥）五处地段。\n",
      "\n",
      "curID: 30\n",
      "---------开始----------------------------------------\n",
      "迭代12次，收敛\n",
      "最大的1个值： [1.03598786]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "近两年，有关零售业话题谈论最多的恐怕是百货业的转型。\n",
      "\n",
      "curID: 31\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "以市值计算，京东在中国上市互联网公司中排名第三，仅次于腾讯和百度。\n",
      "\n",
      "curID: 32\n",
      "---------开始----------------------------------------\n",
      "迭代27次，收敛\n",
      "最大的1个值： [1.26692815]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "面对汹涌澎湃的在线教育浪潮，K12上市教育机构学大教育用X计划布局在线教育。\n",
      "\n",
      "curID: 33\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "之前曾有传闻，SE准备把《最终幻想3》搬上PC，现在几张泄露的画面已经证实这一消息属实，虽然具体发售时间尚不确定，不过Steam平台上《最终幻想3》页面已经公开并确定会有成就列表……。\n",
      "\n",
      "curID: 34\n",
      "---------开始----------------------------------------\n",
      "迭代21次，收敛\n",
      "最大的1个值： [1.20545215]\n",
      "对应的索引： [2]\n",
      "结果：\n",
      "全国人大代表、著名作家二月河从历史角度谈反腐倡廉，王岐山笑称“知音”。\n",
      "\n",
      "curID: 35\n",
      "---------开始----------------------------------------\n",
      "迭代76次，收敛\n",
      "最大的1个值： [1.91891494]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "据易观智库监测数据显示，2011年3季度，中国工商银行在信用卡、借记卡、新业务以及理财产品方面都进行了相关产品的创新。\n",
      "\n",
      "curID: 36\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "苏宁“免费贴膜”活动，使移动端应用得以推广，引来大批关注；各商家借APP、微信等手段，主动推广线上和线下服务，打通流通渠道，其例不胜枚举……物流从移动端开始，也将从此延续、深入下去。\n",
      "\n",
      "curID: 37\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [3]\n",
      "结果：\n",
      "巴基斯坦和阿富汗的困难则是当地的动荡局势，去年12月就有9名疫苗接种员在巴基斯坦被杀害。\n",
      "\n",
      "curID: 38\n",
      "---------开始----------------------------------------\n",
      "迭代9次，收敛\n",
      "最大的1个值： [1.54208429]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "哪些行业会是私募挖掘业绩超预期个股的“主矿脉”。\n",
      "\n",
      "curID: 39\n",
      "---------开始----------------------------------------\n",
      "迭代13次，收敛\n",
      "最大的1个值： [1.22162259]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "沈先生通过软件获取丢失手机的通讯录时，发现多了王某亲属的号码。\n",
      "\n",
      "curID: 40\n",
      "---------开始----------------------------------------\n",
      "迭代13次，收敛\n",
      "最大的1个值： [1.07270491]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "宇龙通信、摩托罗拉、魅族、三星和中兴通讯排名均与2011年12月相同，且宇龙通信、摩托罗拉、魅族仍分列前3。\n",
      "\n",
      "curID: 41\n",
      "---------开始----------------------------------------\n",
      "迭代72次，收敛\n",
      "最大的1个值： [1.45945565]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "公司拟以7.81元/股的价格向激励对象发行738.9万股限制性股票，占总股本的3.08%。\n",
      "\n",
      "curID: 42\n",
      "---------开始----------------------------------------\n",
      "迭代28次，收敛\n",
      "最大的1个值： [1.27556707]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "在为切尔西出战的第400场英超比赛中，特里头球破门，帮助球队逆转南安普敦，赛后特里表示正是里程碑时刻给了他进球的动力。\n",
      "\n",
      "curID: 43\n",
      "---------开始----------------------------------------\n",
      "迭代24次，收敛\n",
      "最大的1个值： [1.24696725]\n",
      "对应的索引： [2]\n",
      "结果：\n",
      "他又来到6号楼，走进102号。\n",
      "\n",
      "curID: 44\n",
      "---------开始----------------------------------------\n",
      "迭代25次，收敛\n",
      "最大的1个值： [1.2518613]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "大额可转让定期存单在正式面世之前，获得了中国主要银行不同的态度反馈--有的摩拳擦掌，有的意兴阑珊。\n",
      "\n",
      "curID: 45\n",
      "---------开始----------------------------------------\n",
      "迭代21次，收敛\n",
      "最大的1个值： [1.21988743]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "明年1月1日起，全市禁行黄标车（高速公路、佛山“一环”、国道除外）。\n",
      "\n",
      "curID: 46\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "现在，它希望通过NBA来彰显地位，挑战耐克阿迪，李宁没做到的它能做到吗。\n",
      "\n",
      "curID: 47\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "美国国际贸易委员会昨日宣布，对华为、中兴、三星和诺基亚公司的3G和4G无线设备发起“337调查”，以确定这些产品是否侵犯美国公司专利权，这是今年以来美国对中国产品发起的第4起“337调查”。\n",
      "\n",
      "curID: 48\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "戳。\n",
      "\n",
      "curID: 49\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "目前仅推测寒流导致向电车传输动力的电池电压下降可能是其中原因之一，但还未查明。\n",
      "\n",
      "curID: 50\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "据日本媒体报道，为纪念中日邦交正常化40周年，同时也为修补因日本政府非法“国有化”钓鱼岛而急速恶化的中日关系，日本七个日中友好团体的负责人26号开始陆续到中国访问。\n",
      "\n",
      "curID: 51\n",
      "---------开始----------------------------------------\n",
      "迭代24次，收敛\n",
      "最大的1个值： [1.24549784]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "一个自称是国内最全、每日更新300兆的社工库网站，公开出售互联网用户的账号和密码，这一幕赫然出现在了互联网世界之中。\n",
      "\n",
      "curID: 52\n",
      "---------开始----------------------------------------\n",
      "迭代9次，收敛\n",
      "最大的1个值： [1.11039091]\n",
      "对应的索引： [3]\n",
      "结果：\n",
      "苹果同时富含维生素和矿物质，能够提高人体免疫力，改善心血管功能。\n",
      "\n",
      "curID: 53\n",
      "---------开始----------------------------------------\n",
      "迭代25次，收敛\n",
      "最大的1个值： [1.25954534]\n",
      "对应的索引： [2]\n",
      "结果：\n",
      "车内乘客说，中巴起火后他们从车内撤离，大火在3分钟就蔓延开来。\n",
      "\n",
      "curID: 54\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "此举意味着，今后法院对“老赖”的执行力度将更加严苛。\n",
      "\n",
      "curID: 55\n",
      "---------开始----------------------------------------\n",
      "迭代9次，收敛\n",
      "最大的1个值： [1.00816284]\n",
      "对应的索引： [2]\n",
      "结果：\n",
      "因涉嫌非法集资和集资诈骗受审，检方建议量刑无期及以上。\n",
      "\n",
      "curID: 56\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "为了保护金牛岭公园的完整性和园内的生态环境不因金宇西路的修建而被被破坏，这座公园附近的一千多名市民联名上书海口市代市长倪强，恳请留住这个“全体市民最好的私人空间”。\n",
      "\n",
      "curID: 57\n",
      "---------开始----------------------------------------\n",
      "迭代17次，收敛\n",
      "最大的1个值： [1.14012182]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "具体面试时间详见各市公务员主管部门和自治区招录机关的面试公告。\n",
      "\n",
      "curID: 58\n",
      "---------开始----------------------------------------\n",
      "迭代20次，收敛\n",
      "最大的1个值： [1.18552353]\n",
      "对应的索引： [2]\n",
      "结果：\n",
      "这就意味着到2020年，中国人均收入水平很可能达到12700美元，成为高收入国家。\n",
      "\n",
      "curID: 59\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "10月24日，温州建筑工人贺武带着幼儿爬上20楼天台，打算以跳楼为由引起对方注意，但此举并未了结他的追讨之路。\n",
      "\n",
      "curID: 60\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "动车坐席号的统一有望使网购火车票的选座功能得以实现。\n",
      "\n",
      "curID: 61\n",
      "---------开始----------------------------------------\n",
      "迭代18次，收敛\n",
      "最大的1个值： [1.37413657]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "微软中国确认，四地办公室遭到中国国家工商总局调查。\n",
      "\n",
      "curID: 62\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "Milking的好处是将调温、量取奶粉、搅拌、试温这些过程交给了机器以简化流程，加入奶粉和水后，10秒后就能够得到一杯40度的奶。\n",
      "\n",
      "curID: 63\n",
      "---------开始----------------------------------------\n",
      "迭代10次，收敛\n",
      "最大的1个值： [1.06411932]\n",
      "对应的索引： [2]\n",
      "结果：\n",
      "昨天，因不忍拖累儿子，余师傅服下大量农药，让儿子不知所措。\n",
      "\n",
      "curID: 64\n",
      "---------开始----------------------------------------\n",
      "迭代10次，收敛\n",
      "最大的1个值： [1.02167068]\n",
      "对应的索引： [2]\n",
      "结果：\n",
      "宜家的挑战是，需要拥有更加便宜的物流解决方案，并把电子商务与实体店配送结合起来。\n",
      "\n",
      "curID: 65\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "据了解，这块土地的起拍价创南昌土地市场历史新高，如果不流拍，即使底价成交也将成为南昌12个月以来单价最高地块。\n",
      "\n",
      "curID: 66\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "马云回应称自己“躺枪”，此举考虑到目前中国公益慈善基金设置制度尚未完善，而且实际上捐款的绝大部分将用于中国公益事业。\n",
      "\n",
      "curID: 67\n",
      "---------开始----------------------------------------\n",
      "迭代11次，收敛\n",
      "最大的1个值： [1.16965567]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "年轻父母束手无策，他们表示车是新买的，不愿砸车窗。\n",
      "\n",
      "curID: 68\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "如今正设法把电动汽车及私营太空产业变成真正的产业，他的SpaecX公司成功发射了太空巴士，他现在正筹划建立火星殖民地，详见。\n",
      "\n",
      "curID: 69\n",
      "---------开始----------------------------------------\n",
      "迭代18次，收敛\n",
      "最大的1个值： [1.34239637]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "2012年，中国巩固了作为全球超级大国的地位，政治和经济领域都发生了标志性的历史事件。\n",
      "\n",
      "curID: 70\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "部分问题目前基本查清：翟家（8个身份证）曾有31处房产，无一经适房；弄虚作假骗取二胎生育指标；南溪苑项目涉嫌倒卖经适房；二七区房管局两名工作人员通过翟妻取得27套经适房源并牟利。\n",
      "\n",
      "curID: 71\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "买官卖官问题突出，跑官要官、拉票贿选问题一度也比较严重。\n",
      "\n",
      "curID: 72\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "截至8月22日15:30，本周有88家银行共发行841款产品，平均预期收益率为5.08%。\n",
      "\n",
      "curID: 73\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "1、韩启朴槿惠模式经济外交困局挑战新时代目标；2、安倍访美促经济合作日本近期将加入TPP谈判；3、美国自动减支进入倒计时；4、欧盟预计今年欧元区经济萎缩0.3%；5、英国痛失3A评级财政大臣挨“重拳”。\n",
      "\n",
      "curID: 74\n",
      "---------开始----------------------------------------\n",
      "迭代72次，收敛\n",
      "最大的1个值： [1.45945565]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "汤森路透中国固定收益市场展望的参与机构预期，央行9月将维持存款基准利率及准备金率不变，当月流动性也相对稳定。\n",
      "\n",
      "curID: 75\n",
      "---------开始----------------------------------------\n",
      "迭代22次，收敛\n",
      "最大的1个值： [1.21040751]\n",
      "对应的索引： [2]\n",
      "结果：\n",
      "目前，女子的情绪在民警连日来的疏导下已恢复稳定。\n",
      "\n",
      "curID: 76\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "李扬强调，上述数据显示中国全社会的杠杆率已经很高，去杠杆再所难免。\n",
      "\n",
      "curID: 77\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "此次习奥会带给世界太多惊喜，中美双边投资协定、两国军事关系、气候变化安排等27项主要共识与重大成果，让外界不仅看到了中美探索构建新型大国关系的持续努力，更充分感受到中国领导人在复杂外部环境中积极塑造中美关系的战略视野、智慧与能力。\n",
      "\n",
      "curID: 78\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "资金挪用和分配不公是最大的问题。\n",
      "\n",
      "curID: 79\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "此外未与客户签约，银行不得收短信提醒费；跨行转账手续费，每笔最高50元。\n",
      "\n",
      "curID: 80\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "2015年2月28日前，使用中国银行发行的银联标识信用卡（卡号以62开头）通过银联在线支付渠道成功消费，单笔交易金额满等值500元人民币，可享5%现金返还礼遇，每卡每月最高返还金额不超过等值100元人民币，不限名额哦。\n",
      "\n",
      "curID: 81\n",
      "---------开始----------------------------------------\n",
      "迭代18次，收敛\n",
      "最大的1个值： [1.1563549]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "21世纪经济报道获悉，随着三中全会中央全面深化改革的决定，在2014年进入执行年，收入分配改革加快的步伐可能会加快。\n",
      "\n",
      "curID: 82\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "陈一筠称，在广西南宁调研时发现有9岁少女堕胎的案例，而表示对此负责的是一个13岁男孩。\n",
      "\n",
      "curID: 83\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "【川贝蜂蜜炖雪耳、川贝梨糖水、苹果川贝糖水】三款做法戳。\n",
      "\n",
      "curID: 84\n",
      "---------开始----------------------------------------\n",
      "迭代18次，收敛\n",
      "最大的1个值： [1.16985164]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "法庭认定，宋文代利用职务侵吞公共财物6500多万，非法占有黄金百余公斤、白银将近1吨。\n",
      "\n",
      "curID: 85\n",
      "---------开始----------------------------------------\n",
      "迭代28次，收敛\n",
      "最大的1个值： [1.48136716]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "担心父母在家缺少陪伴，安庆一大学生竟通过网络花300多元买了一条国家一级保护动物“暹罗鳄”，快递给父母当宠物。\n",
      "\n",
      "curID: 86\n",
      "---------开始----------------------------------------\n",
      "迭代72次，收敛\n",
      "最大的1个值： [1.45945565]\n",
      "对应的索引： [2]\n",
      "结果：\n",
      "而Mandy与叶翠翠现时虽然单身无拖拍，但也不急于找个马主做男友，笑言不怕做「盛女」。\n",
      "\n",
      "curID: 87\n",
      "---------开始----------------------------------------\n",
      "迭代14次，收敛\n",
      "最大的1个值： [1.09318667]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "然而大陆的成品油价格尚未达到下调的条件。\n",
      "\n",
      "curID: 88\n",
      "---------开始----------------------------------------\n",
      "迭代14次，收敛\n",
      "最大的1个值： [1.08385036]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "青岛市公布2014年“3+4”本科录取线及市内三区普通高中学校录取线。\n",
      "\n",
      "curID: 89\n",
      "---------开始----------------------------------------\n",
      "迭代15次，收敛\n",
      "最大的1个值： [1.10855733]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "加拿大一直是海外移民投资中的热门选项，该投资移民项目已达28之久。\n",
      "\n",
      "curID: 90\n",
      "---------开始----------------------------------------\n",
      "迭代72次，收敛\n",
      "最大的1个值： [1.45945565]\n",
      "对应的索引： [2]\n",
      "结果：\n",
      "随后，上海、郑州、洛阳等多家公司也纷纷致电海璐璐，表示时刻为她敞开就业大门。\n",
      "\n",
      "curID: 91\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "2013年初，工委召开专题会议，就落实八项规定精神作出安排部署。\n",
      "\n",
      "curID: 92\n",
      "---------开始----------------------------------------\n",
      "迭代8次，收敛\n",
      "最大的1个值： [1.16015532]\n",
      "对应的索引： [2]\n",
      "结果：\n",
      "经调查，其真实身份为某啤酒厂打工青年，目前已被拘留，案件正在进一步审理。\n",
      "\n",
      "curID: 93\n",
      "---------开始----------------------------------------\n",
      "迭代33次，收敛\n",
      "最大的1个值： [1.49192692]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "昨日，有报告称我国城镇职工5年涨薪近7成。\n",
      "\n",
      "curID: 94\n",
      "---------开始----------------------------------------\n",
      "迭代17次，收敛\n",
      "最大的1个值： [1.1407815]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "明天开始，高温范围将迅速扩大。\n",
      "\n",
      "curID: 95\n",
      "---------开始----------------------------------------\n",
      "迭代11次，收敛\n",
      "最大的1个值： [1.2740357]\n",
      "对应的索引： [4]\n",
      "结果：\n",
      "⑤决名子茶（清热、明目、镇肝气、益筋骨）。\n",
      "\n",
      "curID: 96\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "更有意思的是，公司董事长孟凯“十一”长假后便开始在国外出差，主要进行为偿付公司债等事项筹集资金，为处置有关资产寻找收购方。\n",
      "\n",
      "curID: 97\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "由于长期营养不良，她如今身高1.6米，体重仅44.5公斤。\n",
      "\n",
      "curID: 98\n",
      "---------开始----------------------------------------\n",
      "迭代1次，收敛\n",
      "最大的1个值： [0.15]\n",
      "对应的索引： [0]\n",
      "结果：\n",
      "中钢协统计，3月上旬重点企业粗钢日均产量158.58万吨，旬环比增长2.49%，钢材市场新年以来出现持续回暖，钢厂产能也得到释放，短期内钢铁行业的估值有望明显提升。\n",
      "\n",
      "curID: 99\n",
      "---------开始----------------------------------------\n",
      "迭代0次，收敛\n",
      "最大的1个值： [1.]\n",
      "对应的索引： [1]\n",
      "结果：\n",
      "而在近十年间，虽然有些国人还对中国道路迷茫懵懂，或对现状忧心忡忡，或面对西方自卑不安，厚积薄发的“醒狮”已经大步流星地走在了“列强”的前面。\n",
      "\n",
      "curID: 100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['专家表示，从2...。\\n',\n",
       " '相关负责人表示界旅游城市排名是国际上首个以城市综合旅游服务能力作为主要依据的排行榜。\\n',\n",
       " '在众多游戏发展中，中国移动游戏市场的发展最为耀眼，截至9月份移动游戏用户接近3亿人。\\n',\n",
       " '周末是欣赏美术作品的好时机，昨日，两个书画、篆刻作品展开幕：“文以载道·湖南中国画学会诗书画印邀请展”在湖南省画院美术馆开幕，“山东·湖南篆刻联展”在长沙简牍博物馆开幕，两个展览均持续到12月15日，向市民免费开放。\\n',\n",
       " '这次“海巡01轮”出海，携带了多套海测设备，包括1套黑匣子搜寻仪、2个潜水机器人、1套旁侧声纳和磁力仪，这些设备完全是国外引进的，水下最佳探测范围是2000米，最大探测范围是5000米。\\n',\n",
       " '”在约四十分钟的演说中，安倍首相连喊七次“世界第一”。\\n',\n",
       " '由于可能产生的污染问题，正引起来自民间的强烈反对，包括140余名离退休老干部。\\n',\n",
       " '纳智捷，这个来自于台湾的汽车品牌随着一部《痞子英雄》开始被国内消费者所认识，通过一部大空间、高配置的SUV让消费者开始了解这个品牌，之后其又慢慢地将MPV和轿车等车型引入国内汽车市场。\\n',\n",
       " '二手车电商O2O，不仅面临着电商流量、产品、IT系统的线上问题，更考验其线下渠道、检测团队管理的能力。\\n',\n",
       " '电力增速走势对经济波动反映准确，对工业增加值变动更为敏感。\\n',\n",
       " '①英特尔会重新对智能手机感兴趣。\\n',\n",
       " '这部法规在保护老年人权益方面作出了多项规定，包括禁止有独立生活能力的子女“啃老”，对于虐待老人的养老机构最高可罚款３万元。\\n',\n",
       " '谷歌被控侵犯七项专利权。\\n',\n",
       " '韩联社援引“朝鲜总联”机关报《朝鲜新报》4月25日报道称，朝鲜计划将位于朝鲜咸镜南道的端川港发展成为国际贸易港口，为实现经济建设和核武建设并进的奋斗目标打下经济基础。\\n',\n",
       " '与此同时，辉瑞从本月初开始真正一分为三，其中包括一个仿制药部门。\\n',\n",
       " '12月13日晚19点35分，中国男足将在湖南郴州与吉尔吉斯斯坦队进行热身赛，由于球队刚刚集训一周的时间，考虑到球员们的身体情况，佩兰也决定在这场比赛多安排一些球员出场，在战前进行的发布会上佩兰表示，会在上、下半时采取两套不同的阵容。\\n',\n",
       " '凭借过去8年的完美答卷，默克尔帮助联盟党在9月22日的联邦议院选举中夺得超过41%的选票。\\n',\n",
       " '日本东京都知事石原慎太郎在24日的记者会上就都政府以“购买”钓鱼岛为由向中央政府申请登岛进行“现场调查”一事表示，已申请的“现场调查”成行后，都政府将于10月再次实施所谓“调查”，届时他也将前往。\\n',\n",
       " '把茅台和腐败简单联系起来毫无道理。\\n',\n",
       " '截至26日两市共235个交易日，平均每一个交易日减持参考总市值约为6.7亿元，有3次减持市值金额超过10亿元。\\n',\n",
       " '相比，北京净租金回报率仅仅3%左右。\\n',\n",
       " '#ETtoday新聞雲#報導：\\u3000雙眼皮眼鏡好高招。\\n',\n",
       " '形式主义，一个干部模样的胖子被一堆公章文件包围着；官僚主义，胖子坐在一个高高椅子上，另外一个群众站在下面仰望他；享乐主义，胖子躺进了酒杯里脸喝得红扑扑；奢靡之风，摆满大餐的桌前，胖子大吃大喝。\\n',\n",
       " '由于时间上的限制，小股东们如今已经失去了补救的机会。\\n',\n",
       " '它以人类汗液、分泌物、脱落的皮屑为营养源，最喜欢温暖潮湿阴暗的环境（20~30摄氏度的温度、50%~70%的湿度，与人类生活环境相似），且繁殖能力惊人。\\n',\n",
       " '“现在租房太贵了，40平方米的住房要2000多元。\\n',\n",
       " '在随后2个月的行程中，他们将横跨亚欧大陆，进行一次行程长达15000公里的行进式采访。\\n',\n",
       " '北京直飞曼谷，免费WIFI境外无线网，特色美食，舒适酒店贴心奉上。\\n',\n",
       " '2月起，上图启动iPad借阅服务，首次70台iPad2整装待发，可供持参考外借证的读者借阅，时长是28天。\\n',\n",
       " '15组监控抓拍摄像头分布在S115线机场高速公路双向车道（阿勒泰路与木材厂交界处—小地窝堡立交桥）五处地段。\\n',\n",
       " '近两年，有关零售业话题谈论最多的恐怕是百货业的转型。\\n',\n",
       " '以市值计算，京东在中国上市互联网公司中排名第三，仅次于腾讯和百度。\\n',\n",
       " '面对汹涌澎湃的在线教育浪潮，K12上市教育机构学大教育用X计划布局在线教育。\\n',\n",
       " '之前曾有传闻，SE准备把《最终幻想3》搬上PC，现在几张泄露的画面已经证实这一消息属实，虽然具体发售时间尚不确定，不过Steam平台上《最终幻想3》页面已经公开并确定会有成就列表……。\\n',\n",
       " '全国人大代表、著名作家二月河从历史角度谈反腐倡廉，王岐山笑称“知音”。\\n',\n",
       " '据易观智库监测数据显示，2011年3季度，中国工商银行在信用卡、借记卡、新业务以及理财产品方面都进行了相关产品的创新。\\n',\n",
       " '苏宁“免费贴膜”活动，使移动端应用得以推广，引来大批关注；各商家借APP、微信等手段，主动推广线上和线下服务，打通流通渠道，其例不胜枚举……物流从移动端开始，也将从此延续、深入下去。\\n',\n",
       " '巴基斯坦和阿富汗的困难则是当地的动荡局势，去年12月就有9名疫苗接种员在巴基斯坦被杀害。\\n',\n",
       " '哪些行业会是私募挖掘业绩超预期个股的“主矿脉”。\\n',\n",
       " '沈先生通过软件获取丢失手机的通讯录时，发现多了王某亲属的号码。\\n',\n",
       " '宇龙通信、摩托罗拉、魅族、三星和中兴通讯排名均与2011年12月相同，且宇龙通信、摩托罗拉、魅族仍分列前3。\\n',\n",
       " '公司拟以7.81元/股的价格向激励对象发行738.9万股限制性股票，占总股本的3.08%。\\n',\n",
       " '在为切尔西出战的第400场英超比赛中，特里头球破门，帮助球队逆转南安普敦，赛后特里表示正是里程碑时刻给了他进球的动力。\\n',\n",
       " '他又来到6号楼，走进102号。\\n',\n",
       " '大额可转让定期存单在正式面世之前，获得了中国主要银行不同的态度反馈--有的摩拳擦掌，有的意兴阑珊。\\n',\n",
       " '明年1月1日起，全市禁行黄标车（高速公路、佛山“一环”、国道除外）。\\n',\n",
       " '现在，它希望通过NBA来彰显地位，挑战耐克阿迪，李宁没做到的它能做到吗。\\n',\n",
       " '美国国际贸易委员会昨日宣布，对华为、中兴、三星和诺基亚公司的3G和4G无线设备发起“337调查”，以确定这些产品是否侵犯美国公司专利权，这是今年以来美国对中国产品发起的第4起“337调查”。\\n',\n",
       " '戳。\\n',\n",
       " '目前仅推测寒流导致向电车传输动力的电池电压下降可能是其中原因之一，但还未查明。\\n',\n",
       " '据日本媒体报道，为纪念中日邦交正常化40周年，同时也为修补因日本政府非法“国有化”钓鱼岛而急速恶化的中日关系，日本七个日中友好团体的负责人26号开始陆续到中国访问。\\n',\n",
       " '一个自称是国内最全、每日更新300兆的社工库网站，公开出售互联网用户的账号和密码，这一幕赫然出现在了互联网世界之中。\\n',\n",
       " '苹果同时富含维生素和矿物质，能够提高人体免疫力，改善心血管功能。\\n',\n",
       " '车内乘客说，中巴起火后他们从车内撤离，大火在3分钟就蔓延开来。\\n',\n",
       " '此举意味着，今后法院对“老赖”的执行力度将更加严苛。\\n',\n",
       " '因涉嫌非法集资和集资诈骗受审，检方建议量刑无期及以上。\\n',\n",
       " '为了保护金牛岭公园的完整性和园内的生态环境不因金宇西路的修建而被被破坏，这座公园附近的一千多名市民联名上书海口市代市长倪强，恳请留住这个“全体市民最好的私人空间”。\\n',\n",
       " '具体面试时间详见各市公务员主管部门和自治区招录机关的面试公告。\\n',\n",
       " '这就意味着到2020年，中国人均收入水平很可能达到12700美元，成为高收入国家。\\n',\n",
       " '10月24日，温州建筑工人贺武带着幼儿爬上20楼天台，打算以跳楼为由引起对方注意，但此举并未了结他的追讨之路。\\n',\n",
       " '动车坐席号的统一有望使网购火车票的选座功能得以实现。\\n',\n",
       " '微软中国确认，四地办公室遭到中国国家工商总局调查。\\n',\n",
       " 'Milking的好处是将调温、量取奶粉、搅拌、试温这些过程交给了机器以简化流程，加入奶粉和水后，10秒后就能够得到一杯40度的奶。\\n',\n",
       " '昨天，因不忍拖累儿子，余师傅服下大量农药，让儿子不知所措。\\n',\n",
       " '宜家的挑战是，需要拥有更加便宜的物流解决方案，并把电子商务与实体店配送结合起来。\\n',\n",
       " '据了解，这块土地的起拍价创南昌土地市场历史新高，如果不流拍，即使底价成交也将成为南昌12个月以来单价最高地块。\\n',\n",
       " '马云回应称自己“躺枪”，此举考虑到目前中国公益慈善基金设置制度尚未完善，而且实际上捐款的绝大部分将用于中国公益事业。\\n',\n",
       " '年轻父母束手无策，他们表示车是新买的，不愿砸车窗。\\n',\n",
       " '如今正设法把电动汽车及私营太空产业变成真正的产业，他的SpaecX公司成功发射了太空巴士，他现在正筹划建立火星殖民地，详见。\\n',\n",
       " '2012年，中国巩固了作为全球超级大国的地位，政治和经济领域都发生了标志性的历史事件。\\n',\n",
       " '部分问题目前基本查清：翟家（8个身份证）曾有31处房产，无一经适房；弄虚作假骗取二胎生育指标；南溪苑项目涉嫌倒卖经适房；二七区房管局两名工作人员通过翟妻取得27套经适房源并牟利。\\n',\n",
       " '买官卖官问题突出，跑官要官、拉票贿选问题一度也比较严重。\\n',\n",
       " '截至8月22日15:30，本周有88家银行共发行841款产品，平均预期收益率为5.08%。\\n',\n",
       " '1、韩启朴槿惠模式经济外交困局挑战新时代目标；2、安倍访美促经济合作日本近期将加入TPP谈判；3、美国自动减支进入倒计时；4、欧盟预计今年欧元区经济萎缩0.3%；5、英国痛失3A评级财政大臣挨“重拳”。\\n',\n",
       " '汤森路透中国固定收益市场展望的参与机构预期，央行9月将维持存款基准利率及准备金率不变，当月流动性也相对稳定。\\n',\n",
       " '目前，女子的情绪在民警连日来的疏导下已恢复稳定。\\n',\n",
       " '李扬强调，上述数据显示中国全社会的杠杆率已经很高，去杠杆再所难免。\\n',\n",
       " '此次习奥会带给世界太多惊喜，中美双边投资协定、两国军事关系、气候变化安排等27项主要共识与重大成果，让外界不仅看到了中美探索构建新型大国关系的持续努力，更充分感受到中国领导人在复杂外部环境中积极塑造中美关系的战略视野、智慧与能力。\\n',\n",
       " '资金挪用和分配不公是最大的问题。\\n',\n",
       " '此外未与客户签约，银行不得收短信提醒费；跨行转账手续费，每笔最高50元。\\n',\n",
       " '2015年2月28日前，使用中国银行发行的银联标识信用卡（卡号以62开头）通过银联在线支付渠道成功消费，单笔交易金额满等值500元人民币，可享5%现金返还礼遇，每卡每月最高返还金额不超过等值100元人民币，不限名额哦。\\n',\n",
       " '21世纪经济报道获悉，随着三中全会中央全面深化改革的决定，在2014年进入执行年，收入分配改革加快的步伐可能会加快。\\n',\n",
       " '陈一筠称，在广西南宁调研时发现有9岁少女堕胎的案例，而表示对此负责的是一个13岁男孩。\\n',\n",
       " '【川贝蜂蜜炖雪耳、川贝梨糖水、苹果川贝糖水】三款做法戳。\\n',\n",
       " '法庭认定，宋文代利用职务侵吞公共财物6500多万，非法占有黄金百余公斤、白银将近1吨。\\n',\n",
       " '担心父母在家缺少陪伴，安庆一大学生竟通过网络花300多元买了一条国家一级保护动物“暹罗鳄”，快递给父母当宠物。\\n',\n",
       " '而Mandy与叶翠翠现时虽然单身无拖拍，但也不急于找个马主做男友，笑言不怕做「盛女」。\\n',\n",
       " '然而大陆的成品油价格尚未达到下调的条件。\\n',\n",
       " '青岛市公布2014年“3+4”本科录取线及市内三区普通高中学校录取线。\\n',\n",
       " '加拿大一直是海外移民投资中的热门选项，该投资移民项目已达28之久。\\n',\n",
       " '随后，上海、郑州、洛阳等多家公司也纷纷致电海璐璐，表示时刻为她敞开就业大门。\\n',\n",
       " '2013年初，工委召开专题会议，就落实八项规定精神作出安排部署。\\n',\n",
       " '经调查，其真实身份为某啤酒厂打工青年，目前已被拘留，案件正在进一步审理。\\n',\n",
       " '昨日，有报告称我国城镇职工5年涨薪近7成。\\n',\n",
       " '明天开始，高温范围将迅速扩大。\\n',\n",
       " '⑤决名子茶（清热、明目、镇肝气、益筋骨）。\\n',\n",
       " '更有意思的是，公司董事长孟凯“十一”长假后便开始在国外出差，主要进行为偿付公司债等事项筹集资金，为处置有关资产寻找收购方。\\n',\n",
       " '由于长期营养不良，她如今身高1.6米，体重仅44.5公斤。\\n',\n",
       " '中钢协统计，3月上旬重点企业粗钢日均产量158.58万吨，旬环比增长2.49%，钢材市场新年以来出现持续回暖，钢厂产能也得到释放，短期内钢铁行业的估值有望明显提升。\\n',\n",
       " '而在近十年间，虽然有些国人还对中国道路迷茫懵懂，或对现状忧心忡忡，或面对西方自卑不安，厚积薄发的“醒狮”已经大步流星地走在了“列强”的前面。\\n']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = predict_test()\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fe4f5fe6-91e2-4690-8701-812d72264ff2",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-19T10:02:00.583170Z",
     "iopub.status.busy": "2024-10-19T10:02:00.582353Z",
     "iopub.status.idle": "2024-10-19T10:02:00.656031Z",
     "shell.execute_reply": "2024-10-19T10:02:00.655394Z",
     "shell.execute_reply.started": "2024-10-19T10:02:00.583140Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'rouge-1': 0.2403395384550552,\n",
       " 'rouge-2': 0.1218037292673829,\n",
       " 'rouge-l': 0.18250341080199725}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from rouge_chinese import Rouge\n",
    "\n",
    "rouge = Rouge()\n",
    "\n",
    "docode_preds = [\" \".join(p) for p in result]\n",
    "decode_labels = [\" \".join(l) for l in ds[\"test\"][\"title\"]]\n",
    "scores = rouge.get_scores(docode_preds, decode_labels, avg=True)\n",
    "{\n",
    "    \"rouge-1\": scores[\"rouge-1\"][\"f\"],\n",
    "    \"rouge-2\": scores[\"rouge-2\"][\"f\"],\n",
    "    \"rouge-l\": scores[\"rouge-l\"][\"f\"],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "06af551d-c0fb-4fc0-ad9c-f9ab0db1ae06",
   "metadata": {
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2024-10-19T10:02:03.857806Z",
     "iopub.status.busy": "2024-10-19T10:02:03.857301Z",
     "iopub.status.idle": "2024-10-19T10:02:03.932040Z",
     "shell.execute_reply": "2024-10-19T10:02:03.931272Z",
     "shell.execute_reply.started": "2024-10-19T10:02:03.857775Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'rouge-1': {'precision': 0.18880349664387153, 'recall': 0.3940944136805994, 'f1': 0.2403395384550552}, 'rouge-2': {'precision': 0.09491139369605771, 'recall': 0.20992371675788346, 'f1': 0.1218037292673829}, 'rouge-l': {'precision': 0.14293111552242682, 'recall': 0.3136475881923821, 'f1': 0.18250341080199725}}\n"
     ]
    }
   ],
   "source": [
    "from rouge_chinese import Rouge\n",
    "\n",
    "rouge = Rouge()\n",
    "\n",
    "docode_preds = [\" \".join(p) for p in result]\n",
    "decode_labels = [\" \".join(l) for l in ds[\"test\"][\"title\"]]\n",
    "\n",
    "                \n",
    "# 计算 ROUGE 分数\n",
    "scores = rouge.get_scores(docode_preds, decode_labels, avg=True)\n",
    "\n",
    "# 提取 ROUGE 的 Precision、Recall 和 F1 值\n",
    "rouge_scores = {\n",
    "    \"rouge-1\": {\n",
    "        \"precision\": scores[\"rouge-1\"][\"p\"],\n",
    "        \"recall\": scores[\"rouge-1\"][\"r\"],\n",
    "        \"f1\": scores[\"rouge-1\"][\"f\"]\n",
    "    },\n",
    "    \"rouge-2\": {\n",
    "        \"precision\": scores[\"rouge-2\"][\"p\"],\n",
    "        \"recall\": scores[\"rouge-2\"][\"r\"],\n",
    "        \"f1\": scores[\"rouge-2\"][\"f\"]\n",
    "    },\n",
    "    \"rouge-l\": {\n",
    "        \"precision\": scores[\"rouge-l\"][\"p\"],\n",
    "        \"recall\": scores[\"rouge-l\"][\"r\"],\n",
    "        \"f1\": scores[\"rouge-l\"][\"f\"]\n",
    "    }\n",
    "}\n",
    "print(rouge_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e64332ab-8420-40b9-ae4a-0278cef401c4",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-19T10:02:12.506307Z",
     "iopub.status.busy": "2024-10-19T10:02:12.505833Z",
     "iopub.status.idle": "2024-10-19T10:02:14.448986Z",
     "shell.execute_reply": "2024-10-19T10:02:14.448136Z",
     "shell.execute_reply.started": "2024-10-19T10:02:12.506280Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-chinese were not used when initializing BertForMaskedLM: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average BERTScore:\n",
      "Average Precision: 0.6132225662469863\n",
      "Average Recall: 0.661447462439537\n",
      "Average F1: 0.634861501455307\n"
     ]
    }
   ],
   "source": [
    "from bert_score import BERTScorer\n",
    "from rouge import Rouge\n",
    "from transformers import BertTokenizer, BertForMaskedLM\n",
    "from bert_score import BERTScorer\n",
    "import jieba\n",
    "model_name = 'bert-base-chinese'\n",
    "tokenizer_A = BertTokenizer.from_pretrained(model_name)\n",
    "model_A = BertForMaskedLM.from_pretrained(model_name)\n",
    "\n",
    "scorer = BERTScorer(model_type=model_name, num_layers=model_A.config.num_hidden_layers)\n",
    "bert_scores = []\n",
    "for pred, ref in zip(result,ds[\"test\"][\"title\"]):\n",
    "    P, R, F1 = scorer.score([pred], [ref])\n",
    "\n",
    "    # 存储Precision、Recall和F1值\n",
    "    bert_scores.append({\n",
    "        \"bert-precision\": P.item(),\n",
    "        \"bert-recall\": R.item(),\n",
    "        \"bert-f1\": F1.item()\n",
    "    })\n",
    "# 计算平均BERTScore的Precision、Recall和F1值\n",
    "avg_precision = sum(score[\"bert-precision\"] for score in bert_scores) / len(bert_scores)\n",
    "avg_recall = sum(score[\"bert-recall\"] for score in bert_scores) / len(bert_scores)\n",
    "avg_f1 = sum(score[\"bert-f1\"] for score in bert_scores) / len(bert_scores)\n",
    "\n",
    "print(\"\\nAverage BERTScore:\")\n",
    "print(\"Average Precision:\", avg_precision)\n",
    "print(\"Average Recall:\", avg_recall)\n",
    "print(\"Average F1:\", avg_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "55faa9f4-68dd-47d1-90ce-663ff5aa9f13",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-09T04:41:52.770999Z",
     "iopub.status.busy": "2024-10-09T04:41:52.770501Z",
     "iopub.status.idle": "2024-10-09T04:41:53.173735Z",
     "shell.execute_reply": "2024-10-09T04:41:53.172980Z",
     "shell.execute_reply.started": "2024-10-09T04:41:52.770968Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
      "Setting `pad_token_id` to `eos_token_id`:None for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CLS] 摘要生成: 昨日,“湘约启航”李湘加盟深圳卫视新闻发布会在京举行,李湘正式与深圳卫视签约,成为主管品牌推广的副总监。李湘说跳槽最重要的原因不是钱,“对于我来说最重要的是发展机会,还有快乐的工作环境。“李湘透露,自己的人事关系也会从湖南迁往深圳。[MASK]<|endoftext|> <|startofpiece|> 李湘加盟深圳卫视:跳槽最主要原因为钱 <|endofpiece|>\n"
     ]
    }
   ],
   "source": [
    "# input_text=''\n",
    "import jieba\n",
    "from rouge_chinese import Rouge\n",
    "\n",
    "input_text = ds[\"test\"][-1][\"content\"]\n",
    "title = ds[\"test\"][-1][\"title\"]\n",
    "\n",
    "inputs = tokenizer(\"摘要生成: \\n\" + input_text + tokenizer.mask_token, return_tensors=\"pt\")\n",
    "inputs = tokenizer.build_inputs_for_generation(inputs, max_gen_length=64)\n",
    "# inputs = inputs.to(\"cpu\")\n",
    "# 将输入张量移动到与模型相同的设备\n",
    "inputs = {key: value.to(device) for key, value in inputs.items()}\n",
    "\n",
    "output = model.generate(**inputs, max_new_tokens=64, eos_token_id=tokenizer.eop_token_id,do_sample=True)\n",
    "print(tokenizer.decode(output[0].tolist()))\n",
    "\n",
    "# 原始文本\n",
    "text = tokenizer.decode(output[0].tolist())\n",
    "# 提取数据\n",
    "start_marker = '<|startofpiece|>'\n",
    "end_marker = '<|endofpiece|>'\n",
    "# 找到起始和结束位置\n",
    "start_index = text.find(start_marker) + len(start_marker)\n",
    "end_index = text.find(end_marker)\n",
    "# 提取所需数据\n",
    "extracted_data = text[start_index:end_index].strip()\n",
    "# 打印结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d576c8a6-2a62-4aec-932a-be9835b8a4b4",
   "metadata": {
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2024-10-09T04:42:08.224815Z",
     "iopub.status.busy": "2024-10-09T04:42:08.224087Z",
     "iopub.status.idle": "2024-10-09T04:42:08.830092Z",
     "shell.execute_reply": "2024-10-09T04:42:08.829363Z",
     "shell.execute_reply.started": "2024-10-09T04:42:08.224781Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at ./bert-base-chinese were not used when initializing BertForMaskedLM: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "原文title内容为:  李湘加盟深圳卫视任副总监\n",
      "\n",
      "\n",
      "文本摘要内容为:  李湘加盟深圳卫视:跳槽最主要原因为钱\n",
      "\n",
      "分词情况\n",
      "\n",
      "李湘 加盟 深圳 卫视 任 副 总监\n",
      "李湘 加盟 深圳 卫视 : 跳槽 最 主要 原因 为 钱\n",
      "\n",
      "ROUGE 分数:\n",
      "{'rouge-1': 0.44444443969135805, 'rouge-2': 0.3749999953125, 'rouge-l': 0.44444443969135805}\n",
      "\n",
      "BERTScore:\n",
      "{'bert-precision': 0.7679480314254761, 'bert-recall': 0.8269060850143433, 'bert-f1': 0.7963373064994812}\n"
     ]
    }
   ],
   "source": [
    "from rouge import Rouge\n",
    "from transformers import BertTokenizer, BertForMaskedLM\n",
    "from bert_score import BERTScorer\n",
    "import jieba\n",
    "\n",
    "# 定义预测和实际值\n",
    "result = title\n",
    "predict = extracted_data\n",
    "\n",
    "# 加载BERT模型和tokenizer\n",
    "model_name = './bert-base-chinese'\n",
    "tokenizer_A = BertTokenizer.from_pretrained(model_name)\n",
    "model_A = BertForMaskedLM.from_pretrained(model_name)\n",
    "# 计算BERTScore\n",
    "scorer = BERTScorer(model_type=model_name, num_layers=model_A.config.num_hidden_layers, batch_size=1)\n",
    "P, R, F1 = scorer.score([predict], [result])\n",
    "\n",
    "# 初始化 ROUGE 评分器\n",
    "rouge = Rouge()\n",
    "# 使用 jieba 进行分词\n",
    "result_tokens = \" \".join(jieba.cut(result))  # 实际值分词\n",
    "predict_tokens = \" \".join(jieba.cut(predict))  # 预测值分词\n",
    "\n",
    "\n",
    "print('\\n原文title内容为:  '+title)\n",
    "print(\"\\n\")\n",
    "print('文本摘要内容为:  '+extracted_data)\n",
    "print(\"\\n分词情况\\n\")\n",
    "print(result_tokens)\n",
    "print(predict_tokens)\n",
    "\n",
    "# 将实际值和预测值放入列表\n",
    "decode_preds = [predict_tokens]  # 预测值\n",
    "decode_labels = [result_tokens]  # 实际值\n",
    "\n",
    "# 计算 ROUGE 分数\n",
    "scores = rouge.get_scores(decode_preds, decode_labels, avg=True)\n",
    "\n",
    "# 提取 ROUGE 分数\n",
    "rouge_scores = {\n",
    "    \"rouge-1\": scores['rouge-1']['f'],\n",
    "    \"rouge-2\": scores['rouge-2']['f'],\n",
    "    \"rouge-l\": scores['rouge-l']['f'],\n",
    "}\n",
    "# 输出 ROUGE 和 BERTScore 结果\n",
    "print(\"\\nROUGE 分数:\")\n",
    "print(rouge_scores)\n",
    "\n",
    "print(\"\\nBERTScore:\")\n",
    "bert_scores = {\n",
    "    \"bert-precision\": P.item(),\n",
    "    \"bert-recall\": R.item(),\n",
    "    \"bert-f1\": F1.item()\n",
    "}\n",
    "# 输出转换后的 BERTScore 结果\n",
    "print(bert_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a84fc289-137b-40d6-8f86-2f450477e414",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-09T04:42:16.156015Z",
     "iopub.status.busy": "2024-10-09T04:42:16.155508Z",
     "iopub.status.idle": "2024-10-09T04:42:16.447523Z",
     "shell.execute_reply": "2024-10-09T04:42:16.446838Z",
     "shell.execute_reply.started": "2024-10-09T04:42:16.155986Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
      "Setting `pad_token_id` to `eos_token_id`:None for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "原文内容： 近阶段苹果的诸多投资者无疑仿佛徜徉在九霄云外。根据苹果官方公布的总利润再次刷新历史，公司的股票也再创新高。此图表非常清晰的展示了苹果股票的“孤独”，其中Y轴代表这些公司目前的市值，X轴代表年增长率涨幅情况。\n",
      "\n",
      "文本摘要内容为: 图解《寂寞乔布斯》:苹果的“黄金单身汉”\n",
      "文本标题内容: “孤独”的苹果：一张图说明竞争对手和苹果的差距\n"
     ]
    }
   ],
   "source": [
    "def generate_summary(input_text, tokenizer=tokenizer, model=model, device=device):\n",
    "    # 生成摘要的输入\n",
    "    inputs = tokenizer(\"摘要生成: \\n\" + input_text + tokenizer.mask_token, return_tensors=\"pt\")\n",
    "    inputs = tokenizer.build_inputs_for_generation(inputs, max_gen_length=64)\n",
    "\n",
    "    # 将输入张量移动到与模型相同的设备\n",
    "    inputs = {key: value.to(device) for key, value in inputs.items()}\n",
    "\n",
    "    # 生成输出\n",
    "    output = model.generate(**inputs, max_new_tokens=64, eos_token_id=tokenizer.eop_token_id, do_sample=True)\n",
    "\n",
    "    # 解码生成的文本\n",
    "    text = tokenizer.decode(output[0].tolist())\n",
    "    # print(text)\n",
    "\n",
    "    # 提取数据\n",
    "    start_marker = '<|startofpiece|>'\n",
    "    end_marker = '<|endofpiece|>'\n",
    "    # 找到起始和结束位置\n",
    "    start_index = text.find(start_marker) + len(start_marker)\n",
    "    end_index = text.find(end_marker)\n",
    "    # 提取所需数据\n",
    "    extracted_data = text[start_index:end_index].strip()\n",
    "\n",
    "    return extracted_data\n",
    "\n",
    "# input_text = ds[\"test\"][5][\"content\"]\n",
    "# generate_summary(input_text)\n",
    "\n",
    "\n",
    "input_text = ds[\"test\"][5][\"content\"]\n",
    "print('原文内容： '+input_text)\n",
    "summary = generate_summary(input_text, tokenizer, model, device)\n",
    "print('\\n文本摘要内容为: ' + summary)\n",
    "print('文本标题内容: ' + ds[\"test\"][5][\"title\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9186d54d-c6e7-4d33-8591-895d2c40bd0a",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-10-09T04:43:37.180556Z",
     "iopub.status.busy": "2024-10-09T04:43:37.179959Z",
     "iopub.status.idle": "2024-10-09T04:43:37.199021Z",
     "shell.execute_reply": "2024-10-09T04:43:37.198404Z",
     "shell.execute_reply.started": "2024-10-09T04:43:37.180526Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "faa5364d7e654ac3a4117167f1f77f3f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Textarea(value='', description='文本输入:', layout=Layout(height='300px', width='600px'), placehold…"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from ipywidgets import Button, Textarea, Output, HBox, VBox\n",
    "\n",
    "import string\n",
    "# 数字转换为汉字的字典\n",
    "num_to_chinese = {\n",
    "    \"0\": \"零\",\n",
    "    \"1\": \"一\",\n",
    "    \"2\": \"二\",\n",
    "    \"3\": \"三\",\n",
    "    \"4\": \"四\",\n",
    "    \"5\": \"五\",\n",
    "    \"6\": \"六\",\n",
    "    \"7\": \"七\",\n",
    "    \"8\": \"八\",\n",
    "    \"9\": \"九\",\n",
    "    \"10\": \"十\",\n",
    "    \"11\": \"十一\",\n",
    "    \"12\": \"十二\"\n",
    "}\n",
    "def remove_all_punctuation_and_convert_number_to_chinese(text):\n",
    "    # 创建一个翻译表，用于将所有标点符号替换为空格\n",
    "    translator = str.maketrans('', '', string.punctuation + \"。\"+\"、\"+\"'\"+\"“\"+\"”\"+\"：\"+\":\")\n",
    "    # 使用翻译表去除文本中的所有标点符号\n",
    "    text_without_punctuation = text.translate(translator)\n",
    "    \n",
    "    # 将文本中的数字转换为汉字\n",
    "    for digit, chinese_character in num_to_chinese.items():\n",
    "        text_without_punctuation = text_without_punctuation.replace(digit, chinese_character)\n",
    "    \n",
    "    return text_without_punctuation\n",
    "\n",
    "import re\n",
    "def contains_non_chinese_text(input_text):\n",
    "    # 使用正则表达式匹配非中文字符\n",
    "    non_chinese_pattern = re.compile(r'[^\\u4e00-\\u9fff]')  # 匹配不是中文字符的正则表达式\n",
    "    match = non_chinese_pattern.search(input_text)\n",
    "\n",
    "    if match:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "def check_text_input(input_text):\n",
    "    if not isinstance(input_text, str):\n",
    "        return 0\n",
    "    elif contains_non_chinese_text(input_text):\n",
    "        return 2\n",
    "    else:\n",
    "        return 2\n",
    "# # 示例用法\n",
    "# check_text_input('这是一段文本内容')\n",
    "# check_text_input('Hello World')\n",
    "# check_text_input('1234')\n",
    "\n",
    "\n",
    "# 定义一个简单的函数\n",
    "def greet(b):\n",
    "    input_text = remove_all_punctuation_and_convert_number_to_chinese(input_widget.value)\n",
    "    output_num=check_text_input(input_text)  #数字输出 判断输入文本的类型\n",
    "    if input_widget.value == '':\n",
    "        text = '内容为空，请输入文本内容。'\n",
    "    else:\n",
    "        if output_num == 0:\n",
    "            text = '输入不是文本，请输入文本内容。'\n",
    "        elif output_num==1:\n",
    "            text = '输入包含非中文字符，请输入中文文本。'\n",
    "        else:\n",
    "            text = generate_summary(input_widget.value)\n",
    "    \n",
    "    output_widget.clear_output()\n",
    "    with output_widget:\n",
    "        print(f'{text}')\n",
    "\n",
    "# 创建一个输出控件\n",
    "output_widget = Output()\n",
    "\n",
    "# 创建一个大的文本输入控件\n",
    "input_widget = Textarea(\n",
    "    placeholder='输入你的文本...',\n",
    "    description='文本输入:',\n",
    "    layout={'width': '600px', 'height': '300px'},  # 设置宽度和高度\n",
    "    style={'overflow': 'auto'}  # 允许滚动\n",
    ")\n",
    "\n",
    "# 创建一个按钮\n",
    "submit_button = Button(description=\"生成摘要\")\n",
    "\n",
    "# 将按钮的点击事件与 greet 函数关联\n",
    "submit_button.on_click(greet)\n",
    "# 增加间隔\n",
    "# input_widget.layout.margin = '30px 30px 30px 30px'  # 上右下左的间距\n",
    "# submit_button.layout.margin = '0 0 30px 0'  # 上右下左的间距\n",
    "# 布局\n",
    "ui = VBox([input_widget, submit_button, output_widget])\n",
    "\n",
    "# 显示控件\n",
    "ui"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1fe441e8-fdec-40c2-a1e8-6787df6ec76d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}