7.PRETRAIN_METHOD/7.2.1.bert_finetune_NSMC.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "\n",
    "import tensorflow as tf\n",
    "from transformers import *\n",
    "\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 시각화\n",
    "\n",
    "def plot_graphs(history, string):\n",
    "    plt.plot(history.history[string])\n",
    "    plt.plot(history.history['val_'+string], '')\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(string)\n",
    "    plt.legend([string, 'val_'+string])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#random seed 고정\n",
    "tf.random.set_seed(1234)\n",
    "np.random.seed(1234)\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "NUM_EPOCHS = 3\n",
    "VALID_SPLIT = 0.2\n",
    "MAX_LEN = 39 # EDA에서 추출된 Max Length\n",
    "DATA_IN_PATH = 'data_in/KOR'\n",
    "DATA_OUT_PATH = \"data_out/KOR\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-multilingual-cased\", cache_dir='bert_ckpt', do_lower_case=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 토크나이저 테스트"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[101, 9521, 118741, 35506, 24982, 48549, 117, 9321, 118610, 119081, 48345, 119, 102]\n",
      "['[ C L S ]', '안', '# # 녕', '# # 하', '# # 세', '# # 요', ',', '반', '# # 갑', '# # 습', '# # 니 다', '.', '[ S E P ]']\n"
     ]
    }
   ],
   "source": [
    "test_sentence = \"안녕하세요, 반갑습니다.\"\n",
    "\n",
    "encode = tokenizer.encode(test_sentence)\n",
    "token_print = [tokenizer.decode(token) for token in encode]\n",
    "\n",
    "print(encode)\n",
    "print(token_print)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[101, 9521, 118741, 35506, 24982, 48549, 117, 9321, 118610, 119081, 48345, 102]\n",
      "[101, 31178, 11356, 102]\n",
      "[CLS] 안녕하세요, 반갑습니다 [SEP]\n",
      "[CLS] Hello world [SEP]\n"
     ]
    }
   ],
   "source": [
    "kor_encode = tokenizer.encode(\"안녕하세요, 반갑습니다\")\n",
    "eng_encode = tokenizer.encode(\"Hello world\")\n",
    "kor_decode = tokenizer.decode(kor_encode)\n",
    "eng_decode = tokenizer.decode(eng_encode)\n",
    "\n",
    "print(kor_encode)\n",
    "# [101, 9521, 118741, 35506, 24982, 48549, 117, 9321, 118610, 119081, 48345, 102]\n",
    "print(eng_encode)\n",
    "# [101, 31178, 11356, 102]\n",
    "print(kor_decode)\n",
    "# [CLS] 안녕하세요, 반갑습니다 [SEP]\n",
    "print(eng_decode)\n",
    "# [CLS] Hello world [SEP]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Korean Movie Review Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>document</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9976970</td>\n",
       "      <td>아 더빙.. 진짜 짜증나네요 목소리</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3819312</td>\n",
       "      <td>흠...포스터보고 초딩영화줄....오버연기조차 가볍지 않구나</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10265843</td>\n",
       "      <td>너무재밓었다그래서보는것을추천한다</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9045019</td>\n",
       "      <td>교도소 이야기구먼 ..솔직히 재미는 없다..평점 조정</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6483659</td>\n",
       "      <td>사이몬페그의 익살스런 연기가 돋보였던 영화!스파이더맨에서 늙어보이기만 했던 커스틴 ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id                                           document  label\n",
       "0   9976970                                아 더빙.. 진짜 짜증나네요 목소리      0\n",
       "1   3819312                  흠...포스터보고 초딩영화줄....오버연기조차 가볍지 않구나      1\n",
       "2  10265843                                  너무재밓었다그래서보는것을추천한다      0\n",
       "3   9045019                      교도소 이야기구먼 ..솔직히 재미는 없다..평점 조정      0\n",
       "4   6483659  사이몬페그의 익살스런 연기가 돋보였던 영화!스파이더맨에서 늙어보이기만 했던 커스틴 ...      1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 데이터 전처리 준비\n",
    "DATA_TRAIN_PATH = os.path.join(DATA_IN_PATH, \"naver_movie\", \"ratings_train.txt\")\n",
    "DATA_TEST_PATH = os.path.join(DATA_IN_PATH, \"naver_movie\", \"ratings_test.txt\")\n",
    "\n",
    "train_data = pd.read_csv(DATA_TRAIN_PATH, header = 0, delimiter = '\\t', quoting = 3)\n",
    "train_data = train_data.dropna()\n",
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['[SEP]', '[UNK]', '[MASK]', '[PAD]', '[CLS]'] \n",
      " [102, 100, 103, 0, 101]\n",
      "[101, 9521, 118741, 35506, 24982, 48549, 117, 9321, 118610, 119081, 48345, 119, 102]\n",
      "[101, 31178, 11356, 102]\n",
      "[CLS] 안녕하세요, 반갑습니다. [SEP]\n",
      "[CLS] Hello world [SEP]\n"
     ]
    }
   ],
   "source": [
    "# 스페셜 토큰\n",
    "print(tokenizer.all_special_tokens, \"\\n\", tokenizer.all_special_ids)\n",
    "\n",
    "# 토크나이저 테스트하기\n",
    "kor_encode = tokenizer.encode(\"안녕하세요, 반갑습니다. \")\n",
    "eng_encode = tokenizer.encode(\"Hello world\")\n",
    "\n",
    "kor_decode = tokenizer.decode(kor_encode)\n",
    "eng_decode = tokenizer.decode(eng_encode)\n",
    "\n",
    "print(kor_encode)\n",
    "print(eng_encode)\n",
    "print(kor_decode)\n",
    "print(eng_decode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bert Tokenizer\n",
    "\n",
    "# 참조: https://huggingface.co/transformers/main_classes/tokenizer.html?highlight=encode_plus#transformers.PreTrainedTokenizer.encode_plus\n",
    "\n",
    "def bert_tokenizer(sent, MAX_LEN):\n",
    "    \n",
    "    encoded_dict = tokenizer.encode_plus(\n",
    "        text = sent,\n",
    "        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
    "        max_length = MAX_LEN,           # Pad & truncate all sentences.\n",
    "        pad_to_max_length = True,\n",
    "        return_attention_mask = True   # Construct attn. masks.\n",
    "        \n",
    "    )\n",
    "    \n",
    "    input_id = encoded_dict['input_ids']\n",
    "    attention_mask = encoded_dict['attention_mask'] # And its attention mask (simply differentiates padding from non-padding).\n",
    "    token_type_id = encoded_dict['token_type_ids'] # differentiate two sentences\n",
    "    \n",
    "    return input_id, attention_mask, token_type_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 149995/149995 [01:02<00:00, 2399.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# sents: 149995, # labels: 149995\n"
     ]
    }
   ],
   "source": [
    "# train_data = train_data[:1000] # for test\n",
    "\n",
    "input_ids = []\n",
    "attention_masks = []\n",
    "token_type_ids = []\n",
    "train_data_labels = []\n",
    "\n",
    "for train_sent, train_label in tqdm(zip(train_data[\"document\"], train_data[\"label\"]), total=len(train_data)):\n",
    "    try:\n",
    "        input_id, attention_mask, token_type_id = bert_tokenizer(train_sent, MAX_LEN)\n",
    "        \n",
    "        input_ids.append(input_id)\n",
    "        attention_masks.append(attention_mask)\n",
    "        token_type_ids.append(token_type_id)\n",
    "        train_data_labels.append(train_label)\n",
    "\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        print(train_sent)\n",
    "        pass\n",
    "\n",
    "train_movie_input_ids = np.array(input_ids, dtype=int)\n",
    "train_movie_attention_masks = np.array(attention_masks, dtype=int)\n",
    "train_movie_type_ids = np.array(token_type_ids, dtype=int)\n",
    "train_movie_inputs = (train_movie_input_ids, train_movie_attention_masks, train_movie_type_ids)\n",
    "\n",
    "train_data_labels = np.asarray(train_data_labels, dtype=np.int32) #레이블 토크나이징 리스트\n",
    "\n",
    "print(\"# sents: {}, # labels: {}\".format(len(train_movie_input_ids), len(train_data_labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  101  1181 49904 13503   119   119   119  1180 29347 63227 27884 12300\n",
      "  1177 29347 47087 13045 30666 18539 18702 12397   119   119   119   119\n",
      "  1174 29347 40523 25763 13130 20766 23724 20966 97076 46069 17360 12799\n",
      "  1174 25539   102]\n",
      "[1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1]\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0]\n",
      "[CLS] 흠... 포스터보고 초딩영화줄.... 오버연기조차 가볍지 아 [SEP]\n"
     ]
    }
   ],
   "source": [
    "# 최대 길이: 39\n",
    "input_id = train_movie_input_ids[1]\n",
    "attention_mask = train_movie_attention_masks[1]\n",
    "token_type_id = train_movie_type_ids[1]\n",
    "\n",
    "print(input_id)\n",
    "print(attention_mask)\n",
    "print(token_type_id)\n",
    "print(tokenizer.decode(input_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TFBertClassifier(tf.keras.Model):\n",
    "    def __init__(self, model_name, dir_path, num_class):\n",
    "        super(TFBertClassifier, self).__init__()\n",
    "\n",
    "        self.bert = TFBertModel.from_pretrained(model_name, cache_dir=dir_path)\n",
    "        self.dropout = tf.keras.layers.Dropout(self.bert.config.hidden_dropout_prob)\n",
    "        self.classifier = tf.keras.layers.Dense(num_class, \n",
    "                                                kernel_initializer=tf.keras.initializers.TruncatedNormal(self.bert.config.initializer_range), \n",
    "                                                name=\"classifier\")\n",
    "        \n",
    "    def call(self, inputs, attention_mask=None, token_type_ids=None, training=False):\n",
    "        \n",
    "        #outputs 값: # sequence_output, pooled_output, (hidden_states), (attentions)\n",
    "        outputs = self.bert(inputs, attention_mask=attention_mask, token_type_ids=token_type_ids)\n",
    "        pooled_output = outputs[1] \n",
    "        pooled_output = self.dropout(pooled_output, training=training)\n",
    "        logits = self.classifier(pooled_output)\n",
    "\n",
    "        return logits\n",
    "\n",
    "cls_model = TFBertClassifier(model_name='bert-base-multilingual-cased',\n",
    "                                  dir_path='bert_ckpt',\n",
    "                                  num_class=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 학습 준비하기\n",
    "optimizer = tf.keras.optimizers.Adam(3e-5)\n",
    "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')\n",
    "cls_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_out/KOR/tf2_bert_naver_movie -- Folder already exists \n",
      "\n",
      "Train on 119996 samples, validate on 29999 samples\n",
      "Epoch 1/3\n",
      "119808/119996 [============================>.] - ETA: 0s - loss: 0.4312 - accuracy: 0.7908\n",
      "Epoch 00001: val_accuracy improved from -inf to 0.84816, saving model to data_out/KOR/tf2_bert_naver_movie/weights.h5\n",
      "119996/119996 [==============================] - 354s 3ms/sample - loss: 0.4311 - accuracy: 0.7909 - val_loss: 0.3446 - val_accuracy: 0.8482\n",
      "Epoch 2/3\n",
      "119808/119996 [============================>.] - ETA: 0s - loss: 0.3258 - accuracy: 0.8565\n",
      "Epoch 00002: val_accuracy improved from 0.84816 to 0.86220, saving model to data_out/KOR/tf2_bert_naver_movie/weights.h5\n",
      "119996/119996 [==============================] - 334s 3ms/sample - loss: 0.3258 - accuracy: 0.8565 - val_loss: 0.3240 - val_accuracy: 0.8622\n",
      "Epoch 3/3\n",
      "119808/119996 [============================>.] - ETA: 0s - loss: 0.2797 - accuracy: 0.8796\n",
      "Epoch 00003: val_accuracy improved from 0.86220 to 0.86390, saving model to data_out/KOR/tf2_bert_naver_movie/weights.h5\n",
      "119996/119996 [==============================] - 335s 3ms/sample - loss: 0.2797 - accuracy: 0.8796 - val_loss: 0.3293 - val_accuracy: 0.8639\n",
      "{'loss': [0.43106871938013214, 0.32580592471534614, 0.2797445489828251], 'accuracy': [0.7908847, 0.8564786, 0.87957096], 'val_loss': [0.34457365759051106, 0.32403103534414507, 0.32925551826943794], 'val_accuracy': [0.8481616, 0.86219543, 0.8638955]}\n"
     ]
    }
   ],
   "source": [
    "model_name = \"tf2_bert_naver_movie\"\n",
    "\n",
    "# overfitting을 막기 위한 ealrystop 추가\n",
    "earlystop_callback = EarlyStopping(monitor='val_accuracy', min_delta=0.0001,patience=2)\n",
    "# min_delta: the threshold that triggers the termination (acc should at least improve 0.0001)\n",
    "# patience: no improvment epochs (patience = 1, 1번 이상 상승이 없으면 종료)\\\n",
    "\n",
    "checkpoint_path = os.path.join(DATA_OUT_PATH, model_name, 'weights.h5')\n",
    "checkpoint_dir = os.path.dirname(checkpoint_path)\n",
    "\n",
    "# Create path if exists\n",
    "if os.path.exists(checkpoint_dir):\n",
    "    print(\"{} -- Folder already exists \\n\".format(checkpoint_dir))\n",
    "else:\n",
    "    os.makedirs(checkpoint_dir, exist_ok=True)\n",
    "    print(\"{} -- Folder create complete \\n\".format(checkpoint_dir))\n",
    "    \n",
    "cp_callback = ModelCheckpoint(\n",
    "    checkpoint_path, monitor='val_accuracy', verbose=1, save_best_only=True, save_weights_only=True)\n",
    "\n",
    "# 학습과 eval 시작\n",
    "history = cls_model.fit(train_movie_inputs, train_data_labels, epochs=NUM_EPOCHS, batch_size=BATCH_SIZE,\n",
    "                    validation_split = VALID_SPLIT, callbacks=[earlystop_callback, cp_callback])\n",
    "\n",
    "#steps_for_epoch\n",
    "\n",
    "print(history.history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_graphs(history, 'loss')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Korean Movie Review Test 데이터"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>document</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6270596</td>\n",
       "      <td>굳 ㅋ</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9274899</td>\n",
       "      <td>GDNTOPCLASSINTHECLUB</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8544678</td>\n",
       "      <td>뭐야 이 평점들은.... 나쁘진 않지만 10점 짜리는 더더욱 아니잖아</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6825595</td>\n",
       "      <td>지루하지는 않은데 완전 막장임... 돈주고 보기에는....</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6723715</td>\n",
       "      <td>3D만 아니었어도 별 다섯 개 줬을텐데.. 왜 3D로 나와서 제 심기를 불편하게 하죠??</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                           document  label\n",
       "0  6270596                                                굳 ㅋ      1\n",
       "1  9274899                               GDNTOPCLASSINTHECLUB      0\n",
       "2  8544678             뭐야 이 평점들은.... 나쁘진 않지만 10점 짜리는 더더욱 아니잖아      0\n",
       "3  6825595                   지루하지는 않은데 완전 막장임... 돈주고 보기에는....      0\n",
       "4  6723715  3D만 아니었어도 별 다섯 개 줬을텐데.. 왜 3D로 나와서 제 심기를 불편하게 하죠??      0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data = pd.read_csv(DATA_TEST_PATH, header = 0, delimiter = '\\t', quoting = 3)\n",
    "test_data = test_data.dropna()\n",
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "49997it [00:20, 2402.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num sents, labels 49997, 49997\n"
     ]
    }
   ],
   "source": [
    "input_ids = []\n",
    "attention_masks = []\n",
    "token_type_ids = []\n",
    "test_data_labels = []\n",
    "\n",
    "for test_sent, test_label in tqdm(zip(test_data[\"document\"], test_data[\"label\"])):\n",
    "    try:\n",
    "        input_id, attention_mask, token_type_id = bert_tokenizer(test_sent, MAX_LEN)\n",
    "\n",
    "        input_ids.append(input_id)\n",
    "        attention_masks.append(attention_mask)\n",
    "        token_type_ids.append(token_type_id)\n",
    "        test_data_labels.append(test_label)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        print(test_sent)\n",
    "        pass\n",
    "\n",
    "test_movie_input_ids = np.array(input_ids, dtype=int)\n",
    "test_movie_attention_masks = np.array(attention_masks, dtype=int)\n",
    "test_movie_type_ids = np.array(token_type_ids, dtype=int)\n",
    "test_movie_inputs = (test_movie_input_ids, test_movie_attention_masks, test_movie_type_ids)\n",
    "\n",
    "test_data_labels = np.asarray(test_data_labels, dtype=np.int32) #레이블 토크나이징 리스트\n",
    "\n",
    "print(\"num sents, labels {}, {}\".format(len(test_movie_input_ids), len(test_data_labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49997/49997 [==============================] - 41s 823us/sample - loss: 0.3375 - accuracy: 0.8593\n",
      "test loss, test acc:  [0.3374919870977371, 0.8592716]\n"
     ]
    }
   ],
   "source": [
    "results = cls_model.evaluate(test_movie_inputs, test_data_labels, batch_size=1024)\n",
    "print(\"test loss, test acc: \", results)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}