1914 lines
113 KiB
Plaintext
1914 lines
113 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"id": "1cc9cec9",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"'C:\\\\Users\\\\Monoid\\\\anaconda3\\\\envs\\\\nn\\\\python.exe'"
|
|
]
|
|
},
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"import sys\n",
|
|
"sys.executable"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "f9c786f1",
|
|
"metadata": {},
|
|
"source": [
|
|
"파이썬 환경 확인.\n",
|
|
"envs\\\\nn\\\\python.exe 으로 끝나기를 기대합니다"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "4c0a08d7",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"100%|████████████████████████████████████████████████████████████████████████████████████| 5/5 [00:00<00:00, 9.21it/s]\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from time import sleep\n",
|
|
"from tqdm import tqdm, trange\n",
|
|
"\n",
|
|
"lst = [i for i in range(5)]\n",
|
|
"\n",
|
|
"for element in tqdm(lst):\n",
|
|
" sleep(0.1)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "56a66a44",
|
|
"metadata": {},
|
|
"source": [
|
|
"### tqdm 소개\n",
|
|
"tqdm은 다음과 같이 Progress bar 그려주는 라이브러리이에요. 와! 편하다.\n",
|
|
"```\n",
|
|
"from tqdm.auto import tqdm\n",
|
|
"from tqdm.notebook import tqdm\n",
|
|
"```\n",
|
|
"이건 0에서 멈춰이있고 작동하지 않더라고요. 왜인진 몰라요."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"id": "d00b25e6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from preprocessing import readPreporcssedDataAll\n",
|
|
"import torch\n",
|
|
"from torch.utils.data import Dataset, DataLoader\n",
|
|
"from dataset import make_collate_fn, DatasetArray\n",
|
|
"from transformers import BertTokenizer\n",
|
|
"import torch.nn as nn\n",
|
|
"from read_data import TagIdConverter"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "a1d7bb15",
|
|
"metadata": {},
|
|
"source": [
|
|
"대충 필요한 것 임포트"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"id": "433419f4",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"tagIdConverter = TagIdConverter()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"id": "79fb54df",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"cuda available : True\n",
|
|
"available device count : 1\n",
|
|
"device name: NVIDIA GeForce RTX 3070\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"print(\"cuda available :\",torch.cuda.is_available())\n",
|
|
"print(\"available device count :\",torch.cuda.device_count())\n",
|
|
"\n",
|
|
"if torch.cuda.is_available():\n",
|
|
" device_index = torch.cuda.current_device()\n",
|
|
" print(\"device name:\",torch.cuda.get_device_name(device_index))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "dd22dd5e",
|
|
"metadata": {},
|
|
"source": [
|
|
"cuda가 가능한지 먼저 확인해보아요."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"id": "2cd9fe37",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"PRETAINED_MODEL_NAME = 'bert-base-multilingual-cased'\n",
|
|
"tokenizer = BertTokenizer.from_pretrained(PRETAINED_MODEL_NAME)\n",
|
|
"\n",
|
|
"my_collate_fn = make_collate_fn(tokenizer)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "2177c793",
|
|
"metadata": {},
|
|
"source": [
|
|
"데이터 로딩 준비"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"id": "e738062d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from transformers import BertModel"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"id": "70296ee3",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
|
|
"- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
|
|
"- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"PRETAINED_MODEL_NAME = 'bert-base-multilingual-cased'\n",
|
|
"bert = BertModel.from_pretrained(PRETAINED_MODEL_NAME)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"id": "6cbc236d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class MyModel(nn.Module):\n",
|
|
" def __init__(self,output_feat: int,bert):\n",
|
|
" super().__init__()\n",
|
|
" self.bert = bert\n",
|
|
" self.dropout = nn.Dropout(p=0.1)\n",
|
|
" self.lin = nn.Linear(768,output_feat) #[batch_size,word_size,768] -> [batch_size,word_size,output_feat]\n",
|
|
" self.softmax = nn.Softmax(2) #[batch_size,word_size,output_feat] -> [batch_size,word_size,output_feat]\n",
|
|
" #0부터 시작해서 2 번째 차원에 softmax.\n",
|
|
"\n",
|
|
" def forward(self,**kargs):\n",
|
|
" emb = self.bert(**kargs)\n",
|
|
" e = self.dropout(emb['last_hidden_state'])\n",
|
|
" w = self.lin(e)\n",
|
|
" return w"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "7dec337a",
|
|
"metadata": {},
|
|
"source": [
|
|
"모델 선언\n",
|
|
"`nn.CrossEntropy`는 소프트맥스를 겸함."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"id": "e5214de8",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"MyModel(\n",
|
|
" (bert): BertModel(\n",
|
|
" (embeddings): BertEmbeddings(\n",
|
|
" (word_embeddings): Embedding(119547, 768, padding_idx=0)\n",
|
|
" (position_embeddings): Embedding(512, 768)\n",
|
|
" (token_type_embeddings): Embedding(2, 768)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (encoder): BertEncoder(\n",
|
|
" (layer): ModuleList(\n",
|
|
" (0): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (1): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (2): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (3): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (4): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (5): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (6): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (7): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (8): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (9): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (10): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (11): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (pooler): BertPooler(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (activation): Tanh()\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" (lin): Linear(in_features=768, out_features=22, bias=True)\n",
|
|
" (softmax): Softmax(dim=2)\n",
|
|
")\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"model = MyModel(22,bert)\n",
|
|
"model.cuda()\n",
|
|
"bert.cuda()\n",
|
|
"print(model)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "72df7ac3",
|
|
"metadata": {},
|
|
"source": [
|
|
"Tag의 종류가 22가지 입니다. 그래서 22을 넣었어요."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "0bba477c",
|
|
"metadata": {},
|
|
"source": [
|
|
"생성과 동시에 gpu로 옮기자.\n",
|
|
"`cuda` 저거 호출하면 됨."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"id": "5aa129e3",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"bert current device : cuda:0\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"print(\"bert current device :\",bert.device)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"id": "61d356b6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"for param in bert.parameters():\n",
|
|
" param.requires_grad = False"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cf6690b2",
|
|
"metadata": {},
|
|
"source": [
|
|
"bert 는 업데이트 하지 않는다. 메모리를 아낄 수 있다."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"id": "f28a1a61",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"device(type='cuda')"
|
|
]
|
|
},
|
|
"execution_count": 13,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"device = torch.device(\"cuda\")\n",
|
|
"device"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 61,
|
|
"id": "8844beef",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"inputs = {'input_ids': torch.tensor([[ 101, 39671, 8935, 73380, 30842, 9632, 125, 9998, 9251, 9559,\n",
|
|
" 9294, 8932, 28143, 9952, 8872, 127, 9489, 34907, 9952, 9279,\n",
|
|
" 12424, 102]],device=device), \n",
|
|
" 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],device=device), \n",
|
|
" 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]],device=device)}"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "4046945a",
|
|
"metadata": {},
|
|
"source": [
|
|
"적당한 인풋을 정의한다"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 62,
|
|
"id": "c37c3c1b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"emb = model(**inputs)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 63,
|
|
"id": "261d4cc7",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"torch.Size([1, 22, 22])"
|
|
]
|
|
},
|
|
"execution_count": 63,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"emb.size()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "d492e37b",
|
|
"metadata": {},
|
|
"source": [
|
|
"결과가 잘 나왔어요."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 66,
|
|
"id": "c773fdba",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"entity_ids = torch.tensor([21,21,7,17,17,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21]\n",
|
|
" ,dtype=torch.int64,device=device)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "b0b69822",
|
|
"metadata": {},
|
|
"source": [
|
|
"잘 이해하지는 못하겠는데, int64면 실행이 되고 int32이면 실행이 안된다.\n",
|
|
"```\n",
|
|
"RuntimeError: \"nll_loss_forward_reduce_cuda_kernel_2d_index\" not implemented for 'Int'\n",
|
|
"```\n",
|
|
"이런 오류를 내면서 죽음."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 67,
|
|
"id": "0f97b71a",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"torch.Size([22, 22])"
|
|
]
|
|
},
|
|
"execution_count": 67,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"emb.view(-1,emb.size(-1)).size()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 68,
|
|
"id": "d7d0164a",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor(0.4138, device='cuda:0', grad_fn=<NllLossBackward0>)"
|
|
]
|
|
},
|
|
"execution_count": 68,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"nn.CrossEntropyLoss(ignore_index=tagIdConverter.pad_id)(emb.view(-1,emb.size(-1)),entity_ids)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "9fee44d6",
|
|
"metadata": {},
|
|
"source": [
|
|
"크로스 엔트로피를 계산하는 데에 성공.\n",
|
|
"ignore_index는 padding class를 넣어요."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 20,
|
|
"id": "197cfa62",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"del inputs\n",
|
|
"del entity_ids\n",
|
|
"del emb"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "733cb548",
|
|
"metadata": {},
|
|
"source": [
|
|
"본격적으로 학습시켜봅시다."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 21,
|
|
"id": "40a3e52c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"datasetTrain, datasetDev, datasetTest = readPreporcssedDataAll()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "66d41fee",
|
|
"metadata": {},
|
|
"source": [
|
|
"데이터 셋이 적어도 어느정도 성능이 나와하야 할지 생각해봅시다.\n",
|
|
"`O` 토큰으로 범벅이 되있으니 전부 `O`로 찍는 것 보다 좋은 성능이 나와야 하지 않겠어요?\n",
|
|
"한번 시도해봅시다."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 22,
|
|
"id": "80c37a04",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"100%|██████████████████████████████████████████████████████████████████████████| 4250/4250 [00:00<00:00, 265652.17it/s]"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"151572/190488 = 0.7957036663726849\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"total_l = 0\n",
|
|
"total_o = 0\n",
|
|
"\n",
|
|
"for item in tqdm(datasetTrain):\n",
|
|
" entities = item[\"entity\"]\n",
|
|
" l = len(entities)\n",
|
|
" o = sum(map(lambda x: 1 if x == \"O\" else 0,entities))\n",
|
|
" total_l += l\n",
|
|
" total_o += o\n",
|
|
"\n",
|
|
"print(f\"{total_o}/{total_l} = {total_o/total_l}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "c0b67c41",
|
|
"metadata": {},
|
|
"source": [
|
|
"79% 보다 높아야 해요."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 23,
|
|
"id": "619b959f",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"BATCH_SIZE = 32\n",
|
|
"train_loader = DataLoader(\n",
|
|
" DatasetArray(datasetTrain),\n",
|
|
" batch_size=BATCH_SIZE,\n",
|
|
" shuffle=True,\n",
|
|
" collate_fn=my_collate_fn\n",
|
|
")\n",
|
|
"dev_loader = DataLoader(\n",
|
|
" DatasetArray(datasetDev),\n",
|
|
" batch_size=BATCH_SIZE,\n",
|
|
" shuffle=True,\n",
|
|
" collate_fn=my_collate_fn\n",
|
|
")\n",
|
|
"test_loader = DataLoader(\n",
|
|
" DatasetArray(datasetTest),\n",
|
|
" batch_size=BATCH_SIZE,\n",
|
|
" shuffle=True,\n",
|
|
" collate_fn=my_collate_fn\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "7d45dd29",
|
|
"metadata": {},
|
|
"source": [
|
|
"bert paramter를 freeze 안했을땐 batch를 8 정도로 했어요. 그 이상은 메모리가 부족해서 돌아가지 않아요."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 24,
|
|
"id": "efd1837a",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"({'input_ids': tensor([[ 101, 39671, 8935, 73380, 30842, 9632, 125, 9998, 9251, 9559,\n",
|
|
" 9294, 8932, 28143, 9952, 8872, 127, 9489, 34907, 9952, 9279,\n",
|
|
" 12424, 102]]),\n",
|
|
" 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]),\n",
|
|
" 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])},\n",
|
|
" tensor([[21, 21, 7, 17, 17, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,\n",
|
|
" 21, 21, 21, 21]]))"
|
|
]
|
|
},
|
|
"execution_count": 24,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"my_collate_fn(datasetTrain[0:1])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "c4a2e2e1",
|
|
"metadata": {},
|
|
"source": [
|
|
"데이터를 한번 더 확인"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 25,
|
|
"id": "c575cb55",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from torch.optim import AdamW"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "627cb2f8",
|
|
"metadata": {},
|
|
"source": [
|
|
"tqdm 확인"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 26,
|
|
"id": "56844773",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"optimizer = AdamW(model.parameters(), lr=5.0e-5)\n",
|
|
"CELoss = nn.CrossEntropyLoss(ignore_index=tagIdConverter.pad_id)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "eaa08ab2",
|
|
"metadata": {},
|
|
"source": [
|
|
"옵티마이져 준비"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 27,
|
|
"id": "109259b4",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"epoch 0 start:\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 0: 100%|█████████████████████████████████████████| 133/133 [00:26<00:00, 4.98batch/s, accuracy=0.746, loss=1.88]\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"epoch 1 start:\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 1: 100%|█████████████████████████████████████████| 133/133 [00:26<00:00, 5.04batch/s, accuracy=0.814, loss=1.17]\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"epoch 2 start:\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 2: 100%|████████████████████████████████████████| 133/133 [00:26<00:00, 5.10batch/s, accuracy=0.821, loss=0.928]\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"epoch 3 start:\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 3: 100%|████████████████████████████████████████| 133/133 [00:26<00:00, 5.05batch/s, accuracy=0.821, loss=0.795]\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"epoch 4 start:\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 4: 5%|██▏ | 7/133 [00:01<00:30, 4.10batch/s, accuracy=0.853, loss=0.724]\n"
|
|
]
|
|
},
|
|
{
|
|
"ename": "KeyboardInterrupt",
|
|
"evalue": "",
|
|
"output_type": "error",
|
|
"traceback": [
|
|
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
|
"\u001b[1;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
|
|
"\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_28932/1927930699.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 21\u001b[0m \u001b[0mprediction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0moutput\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mview\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutput\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 22\u001b[0m \u001b[1;31m#부정확 할 수 있지만 대충 맞음.[PAD]기호를 예측할 일은 없어야 함.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m \u001b[0mcorrect\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mprediction\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mbatch_labels\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mview\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 24\u001b[0m \u001b[0maccuracy\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcorrect\u001b[0m \u001b[1;33m/\u001b[0m \u001b[0mbatch_inputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"attention_mask\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mview\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 25\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
|
|
"\u001b[1;31mKeyboardInterrupt\u001b[0m: "
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"TRAIN_EPOCH = 5\n",
|
|
"\n",
|
|
"result = []\n",
|
|
"iteration = 0\n",
|
|
"\n",
|
|
"model.zero_grad()\n",
|
|
"\n",
|
|
"for epoch in range(TRAIN_EPOCH):\n",
|
|
" model.train()\n",
|
|
" print(f\"epoch {epoch} start:\")\n",
|
|
" with tqdm(train_loader, unit=\"batch\") as tepoch:\n",
|
|
" for batch_i,batch_l in tepoch:\n",
|
|
" tepoch.set_description(f\"Epoch {epoch}\")\n",
|
|
" \n",
|
|
" batch_inputs = {k: v.cuda(device) for k, v in list(batch_i.items())}\n",
|
|
" batch_labels = batch_l.cuda(device)\n",
|
|
"\n",
|
|
" output = model(**batch_inputs)\n",
|
|
" loss = CELoss(output.view(-1, output.size(-1)), batch_labels.view(-1))\n",
|
|
" \n",
|
|
" prediction = output.view(-1, output.size(-1)).argmax(dim=-1)\n",
|
|
" #부정확 할 수 있지만 대충 맞음.[PAD]기호를 예측할 일은 없어야 함.\n",
|
|
" correct = (prediction == batch_labels.view(-1)).sum().item()\n",
|
|
" accuracy = correct / batch_inputs[\"attention_mask\"].view(-1).sum()\n",
|
|
" \n",
|
|
" optimizer.zero_grad()\n",
|
|
" loss.backward()\n",
|
|
"\n",
|
|
" optimizer.step()\n",
|
|
" \n",
|
|
" result.append({\"iter\":iteration,\"loss\":loss.item(),\"accuracy\":accuracy})\n",
|
|
" tepoch.set_postfix(loss=loss.item(), accuracy= accuracy.item())\n",
|
|
" iteration += 1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 28,
|
|
"id": "f0d9b2d7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%matplotlib inline"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 29,
|
|
"id": "19ca6da1",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"import matplotlib.pyplot as plt"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 30,
|
|
"id": "0bee685c",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAD4CAYAAAAdIcpQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAABZ9klEQVR4nO2dd5gUxdaH39rIBliWJefgigQJiojpmuNVMYtiuHpNKOacs2L8FBMXRb1GxCsqKmZFQAVEQSSDCLLkvDnX90dNTYfpCcvO7uzs1vs888x0T3V39YT61Tl16pSQUmIwGAwGQygSYl0Bg8FgMDR8jFgYDAaDISxGLAwGg8EQFiMWBoPBYAiLEQuDwWAwhCUpVhdOSEiQaWlpsbq8wWAwxCXFxcVSSlnvHf2YiUVaWhpFRUWxurzBYDDEJUKIklhc17ihDAaDwRAWIxYGg8FgCIsRC4PBYDCExYiFwWAwGMJixMJgMBgMYTFiYTAYDIawGLEwGAwGQ1jiUix+Xvszv6z7JdbVaNoUF0N1daxrEZ/MnAl//BHrWhhqyvr1UFkZ61rEjLgUiwNfPZChrwyNdTWaLuXlkJEBN90U65rEJ4ccAgMGxLoWhpqwfj106gQPPhjrmsSMuBOL0spS/+vC8sIY1qQJo2fejx8f23oY4pOFCyE3F7Zti945hYAbboje+dz8+qt6/vnnurtGAyfuxGLR5kX+13PXz41hTeKYd96BNWt2//gSX7YBs8qiYt06WLUq1rWIHx59FFauhC++iM75ysvV8//9X3TO58WKFeo5N7furtHAiTuxWLJ1if/1t6u+xSwLW0OqqmDkSPjHP3b/HMXF6jlexywmT4bt26N3vs6doVevyMpWVETvuvFKs2bqubQ0dLlI2bUrOucJxdKl6jkrq+6v1UCJO7E4b8B5bL5pMwAPzXiI0yedzmvzXmPKsikxrlmcoBv6vLzIypeUwFFHwfz51j7thmoIYvH663DssZGXX7oUTj8drriizqoUkvpo2BoCn30Gr7zi/V60xWLnTvWcVId5UbduVc9lZXV3jQZO3IkFQJuMNv7XHy79kIunXMzwicNjWKM4QotFQoRf/apV8O23MHt24DnKy+HJJ3evHlJGx4110UXw1VeRR6lod0JBQe2vvTvEs1gsWQLXXx9ZJ+HEE+HSS73HJbRYlEQpeaoWi91d8uDSS+Hoo0OX0b+XmoiFlPCf/6jBcTtt28K999asjg2AuBQLgHdPf5fL972c1MTUWFclvtBWQWJiZOV37FDP9j+2FguAm292NggVFfDEE+H/VMcfH1ywsrMD/7ynnaaOsaN7e2DdVzj0WE3XrsHLbNoUHSErL1fnycuDNm1g0SKnWIwfr+oRCwvtq69gyJCaucVOPhmeeQZWrw58r6BADTK/8Yba1g33558Hlo22ZaF/o/q827aputx1V2THv/IKfPMNbN4MEyZ4l9FiYa9zYWHo38nixcqCvfRSa19pKWzZAikpkdWtARG3YjGi/wjGnTiOs/uf7d9XVV0VwxrFCTW1LMKJBTgHd198EW65BZ59NvR5v/wy+Hs7d6o/r50PPwwcEL3nHut1YYSRcX/9pZ7btPF+f9kyaN9e3Ue/frsfHvzii5CaCm+9BR9/rIStf3/Yd1+rzG23wdq18P33u3eN2nDxxSrCZ+NGtb1iRWiBz8tTg9LgLW5r16rnhx9Wz9nZ6ln3+u3oRj1aLh19jWbNlJC1bq22H3+8Zuc55RS45BLLRTt/Plx3HTz3nGVZa7HYsAGaNw/9O//FNxfM/t/RHZxgv78GTNyKheauQ6zew/qC9SFKGoDdFwt7j8otFvn5geWLi2HqVGVyh+r1S7n7PUzd0EHklsW6deo5WI9ai8nHH6ue4VNPRV6fww+3Xs+apZ5/+w3S073LH3OMer7sstoPfP/9N5x/vrJczj9fXTcU+vuvqlLH7LmnqocXFRXQpYu1bW/8qqvVZ6Z72MuXw9dfq/NC4G9FH+M+T6TcfDMMHuzcZxeLTZus/cnJNTv38uXO+h19tBKDa66xyujJqBs2qG1tiRQWOv8HAHPmqOfmza19Wiy0oMURcS8WuTm5fHme6qU+/fPTMa5NPXPddZCZGbrMzp3w++/WdjQsC3fDPHas9b5u9JKSVK98yxb488/g53/zTeWy2J3QU7dLIBL0H1qHW86erVwlF12kBmW1eyBUr3fcOOWT37rVOg/AtGnWa73/mWfgu++8zyOEel61qvbRWXfeqayYxx5Tz2edFbq8/v4rKqzvzstlBFYPWWP//l95BXr2dN77McdYvwMvsdCfze6MGz35pBVssXmz+tx0A9ysmbLmNDUVC/0b0t+91zjYhx/ChRdaZQoKVLlu3QIjpXTHxD5msWWLejaWRWw4sseRHN79cF6a+xIFZTEauKwv7rvPmhj07LPhe9Rjx8JBB1k9v7pwQ02ZApdfrl7rP1hiotW7DGU5TJ6snnUDYHdxLF1qnUPz2GPKVeC2SCK1LLRY6D/7o4+qnuPrrysB0Pvt97jPPpbrBOCFF5SwTJwY2ODp+7eLyFtvedfFfuzuWldbt6pJbtrKmjFDPYfzievvv6TE+oy9GsepU5Ubxo79s17iC2UfPdpZRje8Xt+L/mw+/dQa46gpUkK7dpCTo4QS1D3ZLbRwYrF5s9NVpb97/V1oMXfz1lvW2NOaNeo6XmKvx/K0aICxLGJNYkIiDxz+AGVVZUxeMjnW1ak7pFTpBt59N/JjduxQf1jd2EcqFrNnq16QlxvKbqlo3nwTBg2y/nxJSVYjpM/hhR4IXbRI+dDtjXSfPoF+59tuU6Z/QYGqkzbx3ZZFdTV88onq5b73nrVfN9C6wSoqsvzt9vPY//zz5jktDV3Hq68OHNNwnz8UCxdar0tL1fd79dXqepFQVqZ6qHvvbTXaM2eqZy0WGzbAGWfAggXq/DNnqme7WGjXm/6+fvnF6lz8859KFO3YBaB7d++66fv3siz0Z7l+veqle1FUBA88EPxz9BKhsjJn+a1bnSHfbs4/H269NXB/OLEA+OAD7/32AW/9G9q4UXVS3nwTrr1W7TOWRew4sMuB9G/bnyd+eiLWVak7KitVI6hNWU1Fhdo/fnygH1j/MXWPWv/JwonFsGFqtqrbsli+PHj8vF1EZsyw3E+hXCzan3/PPSo658cfne9r37+b/Hz1p87JUdvuxuPVV1X0ztdfw4gRai5GVVWgG6q42CkE+jxeddbRL3a/+OuvB9bLfv5Q2GfRl5aqhv3555WFY2f6dOVesYsaWAPO+j7spKSoQeuOHVXD9umnatD9kEOUu0l//zNmqH2gfl/TpsHQoSqK5+WXvetdWKjeKy4OHzVWVKTcVBddZO2L5LN54AEVXvrOO97vuwfO27dXlugzzzj3Dx6sBGH16sC62r9HO1osQv1Hgv0H7B2jbdtUkISU6jO/4AL13xXCCgCIIxqNWCSIBM7tfy6LtixixbYV7CqN43h2L6qqrF7vli0w3DavpKRENQaXXw533+08zi0WulFJTFQCM29eYHSL/jPv2mX9ofQfyO46sTdWbj75xHq9fbtyH9kjgTR29w4o946djAzvBsktFvqz+ftv5XqzhyuCChXVPTyweqFucdXn8Yri2bRJfSahXF72ePwDD1QTGiOhtNQ6Ni1NHS+l+m7efFPV9cUXrfLFxc5oMLdY/PIL7L+/tV1drUQHVIOmG0K7FVNZad3/+PHBB7y/+069d/nl4d1nxcXKcrGLqlssXnjBGgwG9VpblPbevf13oAeYNZ07q+f//S+wDo8/Dj16BFoRwYIKIhGLYOh6Sal+9yeeqMYV7Z2gFi0iD11vQDQasQA12A2w5/N7MmBcI8vqOXQotGqlXm/ZosYJNKWlViNo949CaLG44ALlj9chqVLCpEnOeRPaWtCNqv2PHmmKi+3blfvot98ChcndyLnrn57u3RMNZlkceqhl6rvZtMlqkCdNUr117b7RbN4c/D42bgw/891uWWRkQN++zvd79HD2Slu2VM+lpZZA/fmnEtGrr1ZBDLr8ggXWcffea433gPqee/d2Xsvey7UL/3//620BVVYG7/XbG079Gb71Ftx+u3d5TTB3kZ3Ro53CZj9nWZmyLvLyQndUtFiE4gmb1+Gnn1S0mxfLlyurLJgbKpRVoMeO8vPV59mmjSpv9wbEacqQxiUWrawkX3/v+puSiijNEK1vKioCez32UEi3G6qkxPoDVlWp3rkQymcbTCyEUC4asBroiRPh7LPVILpGNzBaLHSvyz2gGQq7S+fzzy2/OgTOaHbPdvXq/YMlFnqgUPeIvSaMaVavDvxc3T3jUMntFi9W9+1uRA46yHptH7NITQ30Tffo4exV2sXCPbYzb55zcFl/DhMnWgPZdvr3D9zXoYNy0cyaZTWwX39t9YDt4lBdHdxqsg/IBnPfeGH/7nfuVKHIwVxL+v732MPal5encpkddZQa19K4xcLdgAcLV/7jD/X92b8zN6NHq1Bi+31ecYWVTiTYOA1Y/yXd6cjJUZ0Ge91PPjn48VFCCHGcEGKZEGKlEOI2j/ezhBCfCCF+F0IsEkJc5HUeO2HFQgjRTAgxx3bS+z3KCCHEWF/FFggh9on8tqJHr1bOnu4Pa34gvyw/SOkGTLt2zrh2N/aZy6AaES0gVVVqUhyoxlGLhW6UtVhUVlrvbd2qrArtQ9cpMezoRlU3WOedF9m9pKU5BxlPPNHykUNgA+m2LHbs8B4k1WKRna3+/Lfdpvzzbv7zH+v1smXh6+sOE7Xzyivwww9OF9fcufDII9b2ccdZLq6UlMBedGKi8/vTA/QlJYHjJD/95Nz+/nvlejrnHGf6FY29d67p2VNdY+bMwM8WAusXLAS5bVvrtX1+SyjatXNeMzs79CRH9+8arMH3NWucbiS7WNgj7zTBBOm110LXORgvvaQafVCCHwwdAq7HTnJz1XF6sauxY+Hpug3xF0IkAi8AxwN9gXOEEC4Tl6uAxVLKgcBhwFNCiJAhdJFYFmXAEb6TDgKOE0IMc5U5Hsj1PS4DXorgvFEnMyWT0fuNpnmK+gMe//bxZI2JQ5Nvx47QvTev3rHuyRQUWJOLEhICLQv9XFJiNcJ33KGirOw5n9yUlKj9etBTjzVceaXVO3bTty8MHOiMwXezfbvq+ebkqJ64u0H79FPr+BtusNJQ5+ere0tLsxpctx8bnAJyxx3B6xEJWkzt4pCbGzim8uOPllicdZbqxWoBr6pSLkWNDu8sLY1srkWoxXcyMgJTXPTsGXrOiHsm/dix3uXsFlKklkWvXoGWoqZlSzVnwU5JifqOZ8xQ9e7QwRKLhATlntMdDfvcnZSUwAlx/fs7J9NpvETWCz1/qU0ba4a97jB16xb8OC0Wv/wCBxygvnstMqB+DzWd/1FzhgIrpZSrpJTlwETAnTxPAs2FEALIBLYDIROshRULqdDdjWTfwz3iOBx4w1d2FtBSCNEh3LnrgudOeI782/Pp0sLqmW8rjuIiK7HAPvhnRw9yl5RYYrFxoxUvX1Zm/cD1n0lH1BQUOEVnwoTQYlFaqhq8jz5S2zrk9YUXgkeGpKaGNtlB+eEPPFD1KocOtcY0HnvMGhMZOVI9H3CAGmcB9ZkUFyvRat8++Plr4h8eODBwn27kwWokMzOt3FXNmweK95FHKsFOSVFhrTNnqsYPlFj84x9WmgjtkopULEKRlqbExO4y6tUr8gmLYKXidmMXC6/BfzcJCeq7d/f4NSUlaqDXzsUXq87AkiXqO23b1hKLsjIlPNp9ZB9batEicL5LSooK9rjqKud+t7UWjMmT1cD4vHlw2GFWHcD6TbvHo0AFI6SnK7Ho1Ent02KRkeG00HafJCHEXNvDHYnQCbCHzuX59tl5HugDrAf+AK6VUoZMUhbRmIUQIlEIMR/YDHwtpXTLcySVQwhxmb7Byjpey/aE3BP8r5duDfIHiAc++8zbvQCWq6qkxGrI7D3z8vJAyyLYokd//22ltvCKcCkpsYQCnFFMwWaRN2umeofh0K4f++zbCy6w6qotB7sVod1LzZoF+qdPO8167W6Q7OdzM2lS+LLJyaqeH32k6ieEEgQv7PejGwwthu3aqWftB4+GWHjlXOrZ0xKL/fbb/XOHm0Sme9u6MWzTJnQW2LKywM9Wr0YH6rebk2NZJlp0dANtd1926xb4G0xJUXV+/vnQ9Q7GkCEqtUgnWzP2zjuqQ6F/U/36eR+rXbV2kQD1XYSauxE5lVLKIbaHe8lKr4u4O/jHAvOBjiiP0fNCCI8/i0VEYiGlrJJSDgI6A0OFEO6RtEgqh5RyvL7BpLrMPQ+8+M8X+eQcFb55zgfnUB1aNBsuofzDWixKS62BXXuDU1YWuVjYcQ+gZ2erXrz92EjEwt5Ytm6tetNeESi6967Pf+WVqmepI750A5uWphpreyPUrFlgiKOOGgNvy0I31G7S01XP/J571GQ0CBQb3cClp1uZa9u1U64ot7vNPotaNxi60dOfn3ZJvPaa071jd11ESjCx0JaiXexrSjixWLhQ/W605dehg3fP204w0QZlAXtFHe2xh/rs3WIxfrzTRRdqBvstt6gxiFB4/W7OOUeNv+m2S2ft1RF57lBj/b/Q32U4Kzt65AH2Qc/OKAvCzkXAZJ83aCXwF7BXqJPWKBpKSrkTmAYctxuVq1cSRAIn5J5ATloOa/PXsnDzwvAHNTTmzw/0xdrRoZKFhUos7I0kBIrFrl3qEW726MaNTr9qx46BAhJMLOwNQGqqtSLf++8rH/JeHr9HHVmlBy31QObxx6sBYx2hoy0Ie0OQmhooFroOxx7r3SCFEou77oL771fx+vPmBTbaoRo4d4SNvZ667losdIOj3VBz5jhnSusGaORI5QbxGrx3o78TPSay777K6tHHBrvvSEhJUb9H7ZJxk5mpBEULefv2KuzXfU37PIhQec2E8G6we/ZUn6XdvTVypLJobr7ZWd9gXHihavhDEWqOxTnnqPDjG25Q36O2no87Ts0n0QPgbssiEis7OvwC5AohevgGrUcA7tXh/gaOBBBCtAN6AyETtEUSDdVGCNHS9zoNOApw+3WmABf4oqKGAbuklB6jjfVLgkjg18uUaXvC2ydQURVnS1oOHhx6Efo991TPy5ercQp3Y3XyydY8gvx8K05//HjViw01ocpufg8aFPi+XSzsDequXdafpVkzlfZ53TqrkQllhr/3nhq/sEeC2YVNN0T2fcnJgQ1DixaqMfn8cyWgF1/sTJIXSizs9zdokKr/oEHWNUM1cG5r2cuy0FaSWyzcaLFITFQDrFdfrbaHDw8e9qm/kylTVP6wuXPVZ/HjjyqLbk0ngvXp43SxDRxojb0EQ1uT3bqpBnf2bJVuRDNkiPW6XTslbO6JpCeeqAbe3YET6elKFOy/t+3brbE7++ftNYj8yCPKoujb19s9GSl6jpL+DvXvKTlZCZHuEOl66s+kNmJdA6SUlcBo4EtgCTBJSrlICHGFEOIKX7EHgQOFEH8A3wK3Sik9wtEsIrEsOgDfCyEWoBTraynlp64LT0Wp0krgZeDKGt5fndGtZTcyUzJZV7COZ2Y9Q2V13Y6V1JqaLLqje7k6hjtU7PiuXdYM3n/8Q/3p7G4iN/ZJTl6hgvY/o26k2rdXYqDPq58j6RUDnHmmatjsgmIXBt2Y2weii4tVao9TTrH2tWihGioh1GPCBOdiSl4uFXu97eTkKAtDzz4PZVmAM6W52wICq0esffvuiXQa/ZnqHq5u6Hv2DP696WOys1W6Fk337pHF9rtdhGlpgalHIrFKwZor0a2bczKcfYA3OVmJyamnOs9x660wYECgZaF9/vp3kJTkdFXZfzdeYnH77dZyukI461UbdICFDv3V35nuWOgxjHrMByWlnCql3FNK2UtK+bBv3zgp5Tjf6/VSymOklHtLKftLKYNku7SIJBpqgZRysJRygO+kD3hcWEopr/JVbG8p5dza3Wp0WX3tagBu+eYWznz/zNhWJhyR5M3R6MZBDwIGG2gFZVl89pn6E9rdVfZ8Tvawy86dVdkjj/Q26e1/zI4d1Z/wq6/Uti4fSowixcuyGDfO2pefrxpc++C0V6/R3qt2pxhJTVUuglBWjxaJcGJx7rnO87qvr8Vi8GDVew62LK3b8tDH6TQtXrjvq6a45/Z4LX0bbuxCj5nZRdD+e/Oqo/v3pcu4LQtt1bh77F5EMpA8alT4MpFwyy3O1R31d6brqQMMwv12GjiNagZ3MHLSc/xzLz5Z9knDti68XENCOBvIl15SrgndeOpQxh491A/WnkNIM3u2ck3oAUjNgAGWeTxggNXz69RJCck334Rv9BMSVJ20WOk/f7DG66+/nHMVQmHvXeoeZevW1qQnHVWXlGT16MK5GHT9hg1T6c5Xrw49GQ+sP3q49UPsriwvt4jdVXXMMcE/I7dYaIHwmoCmqa1YpKbCmDFW79suSrrxDfdbGDNGjR3Yl8C1N5JCqLxh9hnNbrHQ13BbFtodqj/jSDsj8+d7Z4m1H79kiXKDDhpUcxEZMECJpHbdui0LIxbxxeKrFvPwEQ9TJasYPnE4MhprLNcWIZR7QAhrcNcrtUVKivOH/a9/qTGAtDTVqGjTPz1d9e7PPjvwHNpisScg1OhGISPDEiv7mEVN1wt2u6HcdO8eeV4pe+Nsj4K6/HK48UZrUFMIq7EM96e0+5Bfflm5EcK5yiK1LOx1tH9u/fqp8Sd7uvRQjByp3G32yXygxCJY2HkkYrFsWfCUJklJygWkZ1l7BSMcfLB61oLipksXNT/BLoruXv6JJzojpSK1LPRvNxLLws7Agc5wao3d2txrLzVhbt48785WTdBioeunXZi5ud7l44QmIxadW3TmmF7HADB1xVRWbPdIaVGf6MZbZ2edOlU9e4mFju3X6NcJCepPpyco6T9dsMY9I8N7gFILZ2amNbnJ7luuqVhE4oaKNHTaPphpbwybNVMuHLsVoa8XzrLQQuiO8AqF9o2HSy1t95XbP7eEBDWe4SWSXivpdeqkesS6vB4D6NOndpbFnnuqyY1e6Ea9Vy8lKK+8YgmGbugGD1adCy0aEDwPU6SEE4vMTBVNt88+zuvV9HfpJjpzHgLRYqE7YbffrkKLB8R3ctMmIxYAfdtYvZn/LfZIZVyfuGec6gbbyw0lpfOPYf+R6z8QBG+kdfn+/b1DAvWPOjPTqoeOxrGfN1LCuaFg98Qi3J9bXy+YWGzfruLz9RrOkeSK0lx6qZonYg/PDEekn9vhhytry77Ogfs7HDFCDf6PHBloWejPMlI3VGamEg173iw3xx6ryl1wgbquPaJJCOe91eRz9CKYG0r/FgcMcF6/ppZFfeMWi8TE4BP44ogmJRbpyelU31PNXq334t5p9/LXjr9iV5lgYuFlWVRVBf9j2M15XcbdEOtGxCsEFpxioamNWGjzPhqWRbgxAjvhxCI7W/VW9dhKTYIJOnZU80Ts7rlw1KQxy8pS1pz+XLwE/8AD1bPbsnj9deVKi9QnnpCgGnj3+JUX9uva0RZU27aRpQc/4ojgM8iDWRb77aeOs4/XQc3HLOqbQw9Vz/bsuY2AJiUWAEII3j39XSqrK/nyzy/DH1BXuMVCN9helkVVVfAG296Y6jLuP7YWoOPccyld145ELEaNUstRhiKSnp8WlHDWQk1mMkc6ZpGRoQbkQyU4jAY1EdmcHOXv152GUJ+dFov27VWDOnKkSqJY06wItUlop+8t0rG/b78NnuMsmGWRkaGOc0f5NXTLYtQoNQZpn1PSCKjbnBsNlIHtBtK5RWfGzBxDTloOZ/aLQTitWyw09hXmNJWVwRse+4Cq15//xhvVjNOZM4PH2muxsPue7WJh/1NGMvg3YIDyMYdKVaIbtnCrkdWFWEDwAdpoUpOlM594QomAFvRQDaG+v+++U2MYu4t9gPfee2t2bDQzp7rPFU5kozVmUVcIEXkARxzRJMVCCMHzxz/PKe+dwpVTr6S0spRX57/KG6e8QZesEOtIRBN3JlDdQxszJrCslM5BbTt2P7VXL13H8Xstaap5+22VV8feyNpdOTX9U+rEh6GynUYqFjV1Q+n8UQ0B+3hSTcuGEov33lPfmVe00u5iX/AqEvRnHI1BYvdvINw5tQh7WeHnnuu9boeh1jRJsQAYvtdwnj3uWa794lou+Ej5bq//8nr+d1Y9DXy7LYuffgqdjkE32O6edqjMnpFy8smBVof9D1tTsTjqKOXmsQ9KuqkrN1Rt0jhEG7t1Fim60xDqM+/SRS32FEtq6oaKJjpSz73KIigRrQn//W+9zqyOZ5qsWAAMaGeFsl0w8ALe+eMd/t71N12zutb9xd1iMXmytabynXfCww8739c9TXeYYm0nYrm57z61BKedmoqFEOHdPHXhhkpNbRhi8d57wedChCOSMYuGQCytN924e4lFTYlkkN8ANHGxGNJxCP3a9OPxox+nbUZb3vz9TU6ZeAq/XvYroq5isDXBxizAOw5eN65usYiGZWHHy3ddF77hSMWiJsnvhg6tz8yewTnrrN0/NhLLoiEQy/ppyyJURmZD1GnSYpGZksnCK63U5U8e8yQ3fnUjv234jX07hvDxR4NQ/vwOHVSYq33tap1qvK4tCy9iKRY14f6A5eHjj44dVY+5jtd7qTUNwbKoSeizodY0udDZUJzZV0VF/bI+TJ6gaBDKsujQQbmC5s2z9umFU9w9/2CWxemnq4lk0aAuXCLaYohULCKJ5W8MfPWV8qM3BHdaKOpqzMKeHTgYZowhJjTw7kv90j5TpRreVLiJh6Y/xKy8Wbx92ttkNavBOs6R4rWO8RVXqPDJdu1UI2pPuZGV5f3HDGZZ/C+KA/Wxtizy8moWFRXPdO4cH370urIsdObiUOjfY3/3gp2GusSIhY3kxGRapbViweYFTF6iBpt/XPujYz3vqOG13vIZZ6iU4JpIcu7EuxsqkrGhmsyaNtQPsQ5PXr7cWBj1jHFDuWiX0c4vFACLNi8KUboWeImFXvReE8ngdbQHuL2ItWVhaHhEe0zljTe8J6QGIzc3MCutoU4x/1QXOekqNv4f3dTa0bd8cwsLNi2o+Yk+/FAtABMshHLbtsB97sVnImmk49WyqOmYhaFuOO889dhdojVmcf75gavyGRoUxg3lYv7G+QDcMOwGEkQC01ZP4+e1PzvmZETEqFFqnevNm73XSvCyLIJliw1FfYhFQxjgNtQNb765e8fppIzPPhvV6hgaLuaf6uKKfdVkshNyT+CLkV8gEGwo3FDzE+lolh07vN93WxbBGs0xY2DGjODXqY8Qy7qwLLQQ1vV8FkPdkJysftsjR8a6JoZ6woiFi8ePfpyyu8pITkwmNSmVthltWV+wvuYn0ktCbtgA99zjDJWtqHBun3giFBV5n+fWW50LzcSCuhjM1MkLjWVhMMQFxg3lQghBSqLVk+7YvCNrdq2p+Ym0ZfHkk/Dll2rsQq87ra2NNm3Uam0tW9aPO2l3SUhQoasPPBC9c+pQ2GBp0w0GQ4PCiEUYiiqKmPfnPO6bdh+3HnQrldWVFFcU0y6zXegDtWWxwrd8665dasGZ339X1gYoQdmyJTrpjAcOrP05QhFqEuHu0LIlrFplwmINhjhByFhkjQQyMjJkUTDXSwPi7u/u5qEZDzn29W3Tl0VXhgmp/de/1EzcpKTgEVHasnjnHbXmxO6yc6eyTBqydWIwGKKCEKJYSlmDDJvRwTiMw/DA4Q/w879/duxbvGVx+AN1tE+o7KN6jYmarHvgRUN3YxkMhrjHiEUYhBAM7TQ0YH9xRTFTV0wNfmBFRfiTv/aaygHVu3ctamgwGAx1jxGLCEgQCZTe6VyV64S3T+Cf7/yTv3b85X1QuIyYqakqB5ReVc5gMBgaMEYsIiQ1yTkx7Yc1PwDw966/vQ/wsiyeftoKQ23XzswxMBgMcYMRixrw1qlvcf6A8x378vLzvAt7WRZ77GGtudC6dZRrZzAYDHWHEYsaMHLASC4adJFj39r8tYEFr70WPv0UWrVy5ufPyrKSn5mMmQaDIY4IKxZCiC5CiO+FEEuEEIuEENd6lDlMCLFLCDHf97inbqobe1qltfK/btmsJXn5eazasYqq6iqr0Nix6rlvX2d+/qwsK3WGEQuDwRBHRGJZVAI3Sin7AMOAq4QQfT3KzZBSDvI9ojjVt2HRIlXNzN6r9V70zO7Ja/Nfo9fYXrw096XAwu40GVlZ1iLzRiwMBkMcEVYspJQbpJS/+V4XAEuAJjvttlvLbtx20G18du5nnN7ndIorigH4YuUXvPjLizgmObqT/LVsqSbrnXkm3HFHvdXZYDAYakuNZnALIboD04H+Usp82/7DgA+APGA9cJOUMmCKsxDiMuAygJSUlH3LyspqUfXYs614G6dNOo3pa6b79628eiW9cvZQG0ccAd9+a0U9VVZak/UMBoNhN2jwM7iFEJkoQbjOLhQ+fgO6SSkHAs8BH3mdQ0o5Xko5REo5JKk+UmvXMTnpOfzwrx84soe1FOqVU6+0CrjDZ41QGAyGOCUisRBCJKOE4m0p5WT3+1LKfClloe/1VCBZCNFkYkN7Zvf0v/7qT9uAdqhUHwaDwRBHhO3eCyEEMAFYIqV8OkiZ9sAmKaUUQgxFiZDHuqGNk75trPH+ZnZjQlsWM2bAmt1Ic24wGAwNhEh8QQcB5wN/CCHm+/bdAXQFkFKOA84ARgkhKoESYISMVTrbGDCwnZUefOnztje0ZXHwwbFfwMhgMDQZhBDHAc8CicArUsoxHmUOA54BkoGtUspDQ57TpCivPdtLtpPzeA4A8j5r/449OpG9IsgMb4PBYNgNwg1wCyESgeXA0aigo1+Ac6SUi21lWgI/AcdJKf8WQrSVUm4OdV0zgzsKtEprxcsnvczy0csd+7fnh/zsDQaDoS4YCqyUUq6SUpYDE4HhrjLnApOllH8DhBMKMGIRNS7Z5xJye6j1KXb5cg6mYqKfDAZDvdMJsOchyiNwbtyeQLYQYpoQ4lchxAXhTmrEIlrMmeNfenSO72tJkebjNRgMUSdJCDHX9rjM9b5XOmv3eEMSsC/wT+BY4G4hxJ4hL7rb1TVYvPcejBjh39xjrwNg1c8kVFZxyZRLSBAJjD9pfAwraDAYGhGVUsohId7PA7rYtjujJku7y2yVUhYBRUKI6cBA1FiHJ6brGw2mTXNs9uilvseq8jImzJvAy7+9TLWsBqBaVvPp8k/92waDwRBlfgFyhRA9hBApwAhgiqvMx8AhQogkIUQ6sD8qlVNQjFhEA3dUV7t2AMy2eQln580G4H+L/8dJ757E83Oex2AwGKKNlLISGA18iRKASVLKRUKIK4QQV/jKLAG+ABYAc1DhtQtDndeIRTRwi0WHDlxwVz/OPd3adfGUi9lStIUdJTsAmLZ6Wv3Vz2AwNCmklFOllHtKKXtJKR/27RvnmxenyzwhpewrpewvpXwm3DmNWEQDt1gkJzM9p5CiVOjSogufnvMpf+34i+u+vI4/d/wJwOqdq+u/ngaDwbCbmAHuaFBc7NyurOSa/a/hxq9uZNnoZaQlp3HVflfx9CwrW8r6Avd4k8FgMDRcjGURDdyWRWUlNxxwA9X3VJOWnAbAYd0PcxTZVLSJ8iqPdboNBoOhAWLEIhq4xeLUUwEQwgp3Hth+IG42FGyo02oZDAZDtDBiEQ3sYrF4MbQOzM7epYUKez6217FMOHkCAMu3LXcsnGQwGAwNFTNmEQ3sYpGS4llECEHF3RUkiAQWbVaLCN71/V3MWTeH8SeO59J9L62PmhoMBsNuYSyLaFBUBC1aqNdt2wYtlpSQRIJIoGd2T5ISkpizbg4AU1dOrY9aGgwGw25jxGJ3+PhjyPetLFtZCeXlcNNNICU0bx728IyUDIZ0tGbr55e5V6k1GAyGhoURi5ry999wyikwcqRaCW/sWLU/o2brpx/cxVoMaUfJDiMYBoOhQWPEoqZU+3I6/f47vPMO3Hij2q6hWOzdbm//63kb55E1Jou3F7wdrVoaDAZDVDED3DWhqspaKrWwENbbJtYNDAyNDYV93W7NhR9dSHlVOQkigQsGXuAIvTUYDIZYYsSiJmRlWZFPRUWwxJek8cUXYdiwGp2qT+s+ju1hnYdRUVXBxVMuBmCPVntwUNeDal1lg8FgiAbGDVUT7CGy5eWwbBkceSSMGlXjU2WkZPDZuZ9x9dCrARjWaRhPH2ulA9EpzMfMHGPmYhgMhphjLIvasH079Oq124efkHuCP3V589Tm7N9pf/9701ZPY0vxFm7/9nYA5L3uha4MBoOh/jCWRW0oKqrxwLabksoSAJqnNCc1KZX7D7sfgHum3cPpk04PdajBYDDUG0YsakMUxOKmA2/i7H5nc9m+ahndCwaGXTfdYDAY6h0jFrUhCmLRNqMtE8+YSFazLAAyUzI9y5VUlDBw3EBmrJlRq+sZDAbD7mDEojZUVdVaLNw0T/GeAb5k6xIWbFrA6M9HR/V6BoPBEAlGLCJFBhlgTk+P6mVSEr0TEeroqLLKsqhez2AwGCLBiEWk6Ml4bqJsWQSbiKfTgZRVGbEwGAz1jxGLSCkt9d4fZbEIhl4oyVgWBoMhFhixiJSyII10PYnF6p2rASitDBSth6Y/xMdLP66XehgMhqZJWLEQQnQRQnwvhFgihFgkhLjWo4wQQowVQqwUQiwQQuxTN9WNITGwLDbcuIFrhl4DwAdLPgDwXLf7iZ+e4PXfX6+zehgMBkMkM7grgRullL8JIZoDvwohvpZSLraVOR7I9T32B17yPTcegolFlAe4AcYep9Ket89szxVDrmDsnLHM2zgPCByzKCgrIL8sn7z8vKjXw2AwGDRhxUJKuQHY4HtdIIRYAnQC7GIxHHhDSimBWUKIlkKIDr5jGwf16Ia6ev+r/a97ZPdwvFdZXcm4uePo3rI7x+1xHOsK1gEEiMW+4/elTXobvjjvi6jXz2AwND1qlBtKCNEdGAzMdr3VCVhr287z7XOIhRDiMuAygJQga1U3WNyWRVYW7NoFCXU77NMsqVnAvlGfqcSF8l7pF4lNhZsoryr3h97+tuG3Oq2XwWBoWkTc0gkhMoEPgOuklO5l3bziPQMmJkgpx0sph0gphyQlxVkOQ7dYvPceHHII5ObW+aUfOeIRz/2V1ZWsy1eWhUSyvmC9ZzmDwWCoLRGJhRAiGSUUb0spJ3sUyQO62LY7A42r5TrpJOf2wQfD9OnQLLDnH21uP+R25l0+jxuG3eDYv2TLEof7yWvcos8Lffhh9Q91XkeDwdC4iSQaSgATgCVSyqeDFJsCXOCLihoG7GpU4xVVVbBjh3NfcnK9VmFQ+0Gc2udUx741u9b4xyzAWyyWbl3KLd/cUuf1MxgMjZtIfEEHAecDfwgh5vv23QF0BZBSjgOmAicAK4Fi4KKo1zRW7NoFixcH7q9nsQBo2aylY3tDwQby8vPo3rI7q3eu9ouFTg2iCZZvymAwGCIlkmiomXiPSdjLSOCqaFWqQbHffrBiReD+GKyP7RaLGX/PYG3+Wvq07sO24m2s3aViDIorih3lmqcasTAYDLXDzOAOh10oLojtWhNZqVmO7TcXvMn8jfPp3KIznVt0Jq8gj+0l25mybIqj3Hd/fYe4X7CjxOVKMxgMhggxYlETTjstppfXa120SG3h2N+5RWc6tejE+oL1HPHfIxg5eaTjfZ2EcOHmhfVTUYPB0OgwYlETOneO6eWFEHw+8nMWX+kcQ9mr9V60SW/D+oL1/L7p96DH6yVcDQaDoaYYsagJ3brFugYct8dxdGrRiVXXrPLv69umL20z2vL3rr9DHru1eGtdV89gMDRSjFjUhJycWNfAjz0NSG6rXNqktwFUPqlgfPfXd8zOc0++NxgMhvAYsQhHWpp6btYsJhFQoZh24TTGHDmG1KRU2mQoseiV3cv/flpSmmN8Y8K8CQybMIzK6iALORkMBkMQjFiEQ2eVbdcutvXw4NDuh3LrwbcC0CqtFaAGwR847AEAVl+3msHtBwccZ2Z0GwyGmmLEIhwtfD3zzz9Xz3/9Bb81vCR92lpokdqCuw+9m9I7S2mb0Za05DR/Ge2qmr3OckVVVVdRUVUR9vxfrPzCv1qfwWBo2AghjhNCLPOtMXRbiHL7CSGqhBBnhDunEYtwlJXBJZdAnz5qu3t3GBzYW481J+55Imf2PZOnjnkKgNSkVEC5ogDO7Hsmm2/eTPeW3R0htCdPPJmUh0JnAK6qruL4t4/n8P8eXke1NxgM0UIIkQi8gFpnqC9wjhCib5ByjwFfRnJeIxbhKC2tl2SBtSUzJZNJZ06iS1YXx/70ZOVG02MX/dv2d4jF1BVTHeVn5c3if4v/59in52ks37Y86vU2GAxRZyiwUkq5SkpZDkxErTnk5mpUgtjNkZw0zvKEx4A4EYtgpCYqC0PP/s5tlcv3f33Pp8s/pUsLS1iklAghOGDCAWr7XivD/M7SnepcPmvFYDDElCQhxFzb9ngp5Xjbttf6Qo6VS4UQnYBTgSOA/SK66O7VtYkgZdyLxcItyorYr5P6PWQ3y6aoooiT3j2JpATr6z/ktUOYftF0/7Z9ISUtFnrbYDDElEop5ZAQ70eyvtAzwK1SyioRYZSnEYtQVFZCdXVci4VOEXJ0z6MBZzJCewjtj2t/ZEvRFv/2uvx1/rkcfssi0VgWBkMcEMn6QkOAiT6haA2cIISolFJ+FOykRixCUeJLjxHHYvH2aW+zeMtictLVhEJ35lo7m4o2+V+vzV8bIBbGsjAY4oJfgFwhRA9gHTACONdeQErpn9UrhHgd+DSUUIARi0CWL4cfflAZZvVSqnEsFu0z2ztmdYcSi983WnmldLpzMGJhMMQTUspKIcRoVJRTIvCqlHKREOIK3/vjdue8Rizc/N//wbhxUFRkZZmNY7FwE0osLvjISsH+y/pfGDlAZa81A9wGQ3whpZyKWpTOvs9TJKSU/4rknCZ01k2Fb4LaihWwbZt63UjF4pqh1wQt98nyT/yvtVgkisS6qpbBYGjgGLFwo11PmzfD9der161axa4+UcYuFu45GZpOzTuxascqHpr+EFXVVewq2wVAWVVZfVTRYDA0QIwbyk2Zr0HcvBkKC6FrVzj22NjWKYpkNbNW29MRUpoOmR3YULiBNhltWFewjru/v5u0pDQKywuBwOVaDQZD08FYFm7sYpGfDwceCAmN52NqnqLW477toNsY2H4gK65Wy8a2SG3B5yNV/iu7e+q1+a/5xaKkIrLFk1btWMX8jfOjWGuDwRBrjGXhRovFpk2QnAxZWaHLxxlCCMfs7A6ZHQD4Z+4/Gdh+IPJeSWV1JRdPuRiARVsWIX3zeYoritlUuIkqWUXH5h2DXqPXWJUm3X4dL/LL8slMySRBNB4xNhgaK+Zf6kaPWezYAVu3WllnGykZKRksH72cV4e/6t+XlJDEz//+mTmXzCE1MZXFW9QyrmVVZbR/qj2dnu5U6+uWVpbS9f+68s4f79T6XAaDoe4xYuGmzDaIW13d6CwLL3JzcmmW5Iz4GtZ5GPt12o/he3nlH4sMKYNbFoXlhewq28WqHauCljEYDA0HIxZuylwRP43csghH2/S2NSp/0KsH+V8XVRQFLadTjRSUFexexQwGQ71ixMJNWRm0t61j3QQsi1DoNCEZyRkhy5VVlrFm5xp+WvuTf9+Okh1By+sFl/TgucFgaNgYsXBTWqrCZTVN3LLISVNi0Tq9tWN/aaUa25mzbg6/rv+Vs/53Ft2f7e4os71ke9DzVlQrsSgoN5aFwRAPmGgoN2VlSizmzFHbTVws9NreOek5rNm1xr9/Z+lO2me2Z/9X9g92KDtKw1sWRiwMhvjAiIWbsjJoa/PTN6LZ27tD81Q1LyO7WbZj/9d/fs30NdO9DvET0g1VbdxQBkM8YcTCTWkppNoS5g0YELu6NACCrWFhTzoYjJBuKG1ZmAFugyEuCCsWQohXgROBzVLK/h7vHwZ8DPzl2zVZSvlAFOtYv5SVqcSBS5ZATk6jmr29O+j8UYPaD6J5anMqqir4bMVnER0bysVkxiwMhvgiEsvideB54I0QZWZIKU+MSo1iSXW1Wh0vNRX22ivWtWkQ9G3Tl9mXzGZQ+0GkJKawZucapj471T+rOxT5ZflsKtzEkz89yf2H3096crr/PRMNZTDEF2G7zVLK6UBwf0JjQs+xSDXrNtgZ2mmof+Gjbi278ePFP0Z0XEFZAYe+fihP/vwkP/79I3n5eZRXlQNmnoXBEG9Ey8dygBDidyHE50KIfsEKCSEuE0LMFULMraysDFYsduhUH0YsQnJAlwNYOGph2HL5Zfks27YMgPUF6+nyf1246OOLAKcbKtRMb4PB0DCIhlj8BnSTUg4EngM+ClZQSjleSjlESjkkKakBjq1ry6IRLXZUV2SnZQfsm3/5fMe2FgqAPzb/AeDPBaXdUNWy2j9nw2AwNFxqLRZSynwpZaHv9VQgWQjROsxhDRNjWUSM1/KsA9sP5I6D7/Bvz1432//6qZ+f8r9+4IcHuOv7u/zb4Qa5T33vVF765aVa1NZgMNSWWouFEKK9EEL4Xg/1nXNbbc8bE554Qj336RPbesQBaUlp/tfH9DqG4/Y4DsCRbjyYxXDvtHv5bcNv/u1Q4xYbCjbw0dKPuHLqlWHrdOe3d3Le5PPCljMYDDUnktDZd4HDgNZCiDzgXiAZ/AuAnwGMEkJUAiXACBmvTuhvv4WTT4YDDoh1TRo8vv4BAB+e/aE/0sm9NkXLZi39a3gHQ0dErdi2gll5sxi+13BapKqZ81+v+hoIn5sK4JGZjwDw1mlvRXYTBoMhYsKKhZTynDDvP48KrY1vSkpgxQo4++xY1yRumH/5fNpntneExNpFBKBP6z4s37acbSXK2OzSogtr89c6ymg31Jnvn8nvm34H4KOzP+Lk3iezdpcqq9OOGAyG2NC0Z5wBfPUVFBfD4sVqnsXee8e6RnHDwPYDaZfZzrFP4BSLvm36svzq5f7tK/cLdCdpN1RiQqJ/3ynvncKkRZP8Vsmusl0R1yteDVuDoSHTAEOS6pE1a+DYY5U1cZzyuTf19B61RVsWzVOaU1BewPkDzqdVWis23LiBvPw8BrcfTNesroycPNJ/zOx1s8nNyaVrVlfHWMbkpZP5bLmaLZ5flk9pZWnAIk1eFFUUkZmSGeU7MxiaNk3bstjum2v41VewYAGkpUGvXrGtU5yjxyyu2u8qKu+u5NDuhwLQPrM9QzoOITEh0T8Yrrn/h/vJfS6XXaVO62HSokmOBZS2FG2JqA7hxkgMBkPNadpisXWret6xA/74A/r2hcTE0McYQnJWv7MAOH/g+Q63kp1gg9UbCzfSNaur53sAW4qDi4WetwFGLAyGusCIheabb8x4RRTYM2dP5L2Svm36Bi2jU4e4WbJ1SUix0GMbUkqWbFniF4hVO1Y5suBuKtxEtazeneobDIYgNG2x2OLqqZrxinrBHTFlp1tWt6DvHfbfw1i5fSX3TbuPvi/2JeWhFL5Z9Q2jp45m4sKJ/nJHvXkU139xfcDx8zfOZ+Hm8GlKDAZDIE1XLLZuVaGyQkCmbzDUWBYxJ5RlAfDVn1/x/C9WpPb3f33vT05o54VfXgjYN/g/g9n7pb2RUjJm5hhm580OKBNrNhVuYvKSybGuhsEQQNMVizZt4Pnn1Up4jz6q9g0cGNs6NUHaZrR1bLvX+nbz5/Y/HYsqtWzWkrKqsoByaclpAfs0nyz/hNu/vZ2bv765hrWte45/+3hOn3S6ycZraHA0XbHQtG0Lo0erSXlt2sS6Nk2ON099kw6ZHfzbbdKt76BXdi+O6XUM0y6c5t+nExJqMlMyWbFtBQCjhozy79cTBSuqKhD3C16YY1ka2qLo3rI7AC//+jLDXhkWnRuqJat2rAKsrLwGQ0Ohac+zAGuRI5NpNiYc0+sY1t+4HnG/Gsfomd3T/97Ka1YCUFVd5d/3w5ofHMeXVpaypXgLdx5yJw8d8RAvzVUJB7VYbC7aDMDt397uP0ZbItp9ddmnl0X1nqKBPbrLYGgINE2xKLO5LVqZNBINiZ7ZPZl9yWyyUrP8++whuO7xia3FW6mW1f5cUhotFjrctqSyxP9eWaX6/tcXrOecD6xsNlXVVUHDfWvCpsJNSCTtM9vX+Fg9+O/lWjMYYknTdENtsyXFveqq2NXDEED7zPYM7TSU3q17hyx3VM+jANhQuAEgQCwWb1nMr+t/9VsWemU+sARnxt8zHFFUoRroGWtmcMmUSyJKJdL1ma50eKpD2HJe6HQpWtAMhoZC0xaLSZNg8ODY1sUAwJCOQ4DQYbV2PjnnE5qnNGdj4UbAEovzB5xvnfPlIY70IZrSKu/U6aEWYTr+7eOZMG+CX5xCocVoW3HNM/Uby8LQUGmaYjFpknrOyYltPZow5w843+FqmnnRTApvL4z4+GZJzaisruTzlZ8Dlli8fsrrnNH3DH+5CfMmBBwbbIZ3KLHQEwmXb1setIybn/N+jrisxlgWhoZK0xSLhx5Sz0YsYsYbp77Bztt2+rdTk1LJSAm/ZsXQTkN5/8z3Aec4hBaLBJHgGBBfuX1lwDnsobd2QomFTky4bOuyoGXcFJUXhS/koiaWRWV1Za0Hwq/9/Fqu/fzaGh8npeSmr25iwaYFtbq+IX5oemJRYftz5ebGrh6GGvHrZb/yy6W/MPuS2Q7LQWMfs9DrY+jQWDehxKKkQgnQnHVzuPCjC/1pQ3QjHs6ysI9p7E74q7YsvCYauuk1thdZY7LClgvF2DljGTtnbI2P21q8lad+foqj3zy6Vtc31A1CiOOEEMuEECuFELd5vD9SCLHA9/hJCBF2klnTE4udO9Xzc89BenrIooaGwz4d9vGPa3jRPKW5/7We0Hb/YfeTk5bDxNMn8utlvzLpDOV+3FGyw/Mc7/zxDtmPZTN5yWSOfetY3vj9DTYWbmTa6mmsy18HEHbMwm6dRNLgByMSN9Tfu/52WFf1iUSJot2Kqw2vzXuNj5d+HJVzNXWEEInAC8DxQF/gHCGEO1nbX8ChUsoBwIPA+HDnbXqhszotuQmZbVTYLQvdYA9qP4itt1jJIvVgeDDL4uEZDwMwZuYYvwvpf4v/x7VfWG4afY5g5Jfl+1/vjotIp3gP5YaqrK70ryAYK6I9D+TiKRcD8N9T/ssFAy8IU9oQhqHASinlKgAhxERgOLBYF5BS/mQrPwvoHO6kTc+yMGLRKLGLxdunvc1l+1xGvzb9HGV0avRQDXFSQhI7S3f6XUjzN853vL+paFPIethX9HNbFusL1jt64gs2LaCyupI/Nv3ht1z8YxYhLIubv7qZnmN7Bn2/PqiN1RSKCz+6sE7O28hIEkLMtT3cs0o7AfbeRJ5vXzD+DXwe7qJNTyx02KwZ3G40JIpEUpNS/dv92vbjPyf9J2CCXVazQP++Pb0IwNn9znZES9mtkDbpbdhUGCgWUkqenfUs500+jy9WfuHfbx+z2FCwgU5Pd+LeafcCsGTLEgaOG8g939/DgHED6Px/qmPnj4ZyCdqqHavIGpPF0q1LmbpyqvcHUY+Y0N6YUimlHGJ7uF1IXvHnnhOEhBCHo8Ti1nAXbVpiUV4OH/v8osayiHtmXDSDd09/l8p7KsMXBnrn9CZROAXkwC4HOra7tOjC1mLLdWUXiz1a7cG2km1+F4yUku0l25m+ZjrXfXkdb//xtsNlZXfV5OXnAfjFRG/PWTfHX2bI+CH+MRG3ZTF5yWTyy/IZN3dcg1hjvK4si0j5dtW3rNm5JqZ1aMDkAV1s252B9e5CQogBwCvAcCll2ElBTUssbrkFXnlFvTZiEfcc3PVgRvQfEXH5tOS0gEWZjuxxpGM7Oy3bP3gLzgHtPVrtAai5G1XVVYyZOYacx3P4a+dfntezN6h6HEXP16iSyh1lt35+3fCr57EAOWnKEt5Wss1RP42O4qovYjkP5M/tf3LUm0fxr4//FbM6NHB+AXKFED2EECnACGCKvYAQoiswGThfShnR5KGmJRZ//qmeW7eGrNqFHBrikwHt1AJXZ/Y9k3dOe4dL973UMbbRsllLQM2rSBSJjt6rFpZRn43iw6Uf8sjMRwBYunWp4xpXDrkSgfC7oZ6d9Swv//YygN9dpscuhKfHwOnmeW/he35BemvBW6zeudpR9twPziX9kfRa9fZ7PluzMZBYWhbvLXoPgNTE1DAlmyZSykpgNPAlsASYJKVcJIS4Qghxha/YPUAO8KIQYr4QYm648zataKitW+Goo+CLLyChaemkQdEuox2geurn7K2SCH534Xe0e1Lt17PKs5tlk56c7s8tBXBo90P9r+/87k4Ky9WM88d+fMxxjcN7HM4r816hvKqcn9f+zHVfXud/r1mSym6sxSCY77+0spQPFn9Am4w2jPjAaT3Z81wBvLvwXUCFDOekhx6Lu+bza9hSvIW3T3vbsT+YdRQMXe9tJdsorij2J26MBlJKR9qX0spSVu9czV6tVYboXaUqiEALuyEQKeVUYKpr3zjb60uAS2pyzqbVYm7bpqyKxNpnFjXEJ3pxJbsrRzfgYDVALZu15JLBzv9S85TmrLpGrTcRanJe3zZ9SUlMoaKqImD9jdTEVKqqqxg3V/1vg80a/2ntT5zx/hkc+vqhnu97UVxRHLbMc3OeY+LCiSQ+kMg9398T8bnd2C2LmtTRC/dcDfdkxqs+u4o+L/Tx59oqqlBhzZHcbzhGfTqKR2c8WuvzNAWajlgccohaRrV16JXYDI0bLRZ6ljc4xSI5MRlQkVPn7n2u49jmqc3pkd2DTs1VFOL+nfYPOP8BnQ9gr9Z7kZyQTEV1RcCcjtSkVMb/Op6vV30NWGLx2vDXHJMOl23zTivSuUXwcHg9QW/s7LEkPpAYdiDca+nZSLGPWcxdH9aDEfpcLuvKLaAz/p4BWHNctEhEY0LiuF/Hccd3d9T6PE2BpiMWM2eqZyMWTRrtprFPnktOSPa/1kJwep/THQsxgTU43SVLBZp0aN6BFVevcJT5z4n/IUEkkJyYTHlVecBs8Yoqp4DohjEjOcMRxqtXzLOTIBK4eujVQe9NN6LXfXEd1bI67Fre+l414n4R4OKqqKpg3Nxx/t7/8m3L6fFsD1Zsd953bXAPlru3dcjz+gIV0KPvMxqWhSFymo5YaBpA2KEhdmg3k32NayEEzxz7DAuuWEDv1r3Juz6Pa/e/Nug63jrnVPuM9gGCohMOajfUjlKnWHyy/BPHPj1TPCMlw7EeubvRBiUoua2C5zPTEVE6wuqM989gVt6soOUXbVkUsE9PDqyqrlLzR2Y/y6jPRvkH6N/8/U1W71zN0z8/HfS8NSWcZaHHkbRYaDeUVwTYfdPu48rProxa3QwWTUMs7AJRGVlMvqFxomd6291QANcOu5a92+0NQKcWnfwDrK3SVIi1FgGAf3T9B6BW4dPpOTS6XHJCMuXV5QFiUVldydjZVuK+bSXKD5+RnEG3rG6Osm6XU3pyuj981wvd07bPJanpXITVO1ezrXgbSQ8m8fyc59lSpFYa1BZSu0wVCBAsR9amwk08NvOxGuWMckdWucVCf2frCpSQhbIs7v/hfv/SuoboElYshBCvCiE2CyEWBnlfCCHG+rIbLhBC7BP9ataSAl/D0Lw53HxzbOtiiCkdm3cEYJ/2kf1MN9y4gfK7yim43RKX4XsNB/Cc49E8VSU01JaFVx4q+wCubvAyUjLIzXFaDXaBAjVPpFerXkHrWlJZwsUfX+zoqQdbuyMYh/33MHKfU/V4+beXA+Z0hDvfIzMe4bZvb2PSokkRX9PtdrKLxeIti/1ioq2eaI5ZGCInEsvideC4EO8fD+T6HpcBDU/Wt/pm5I4da+ZXNHE6Nu/Ib5f9xnMnPBdR+ZTEFP+gt/0c1fdUe6ZK17H/wcYsgpGRnEGPlj0AtWrg9xd+z8TTJwaUS09O593T3/U8R3FFMa/Nf82xb23+WsoqyzjtvdP4aOlHEdXFbg3pQXIhBNd+fi1TV4RONaItD3dKkmdnPcsTPz4BqOy+v663JiC63VDfrPqGLUVb+H3j7/R7sR+frfgMgM3FKoxZu+6iOWaxtXgr8zbM261jX/zlRT5d/mnU6tJQCTvPQko5XQjRPUSR4cAbUv2qZgkhWgohOkgpw68/WV/ofFBmcNsADO5Q+6V07fMAll61lL1e2MuxPzkhmV1lu1i9czUn7XkSoMYrgpGRkuF3eXVs3pHDuh8WUEbP6xjRfwSjPhsV0Mv3ajwfnvEw+WX5fLj0Q88lZkPhnusQat2LyUsms3L7Sv8kQ3dWXD3X5OaDbmbk5JEAyHuVELktixu+uoEJ8yZw/2H3O/Zrl5jfsqjlrHV7tNh+L+/H6p2r/XWqCVdNvUqdbzeOjSeiMWYRcYZDIcRlOlNiZX2NHcydCyN87gIjFoY6oHfr3qy9fi3T/zXdv08i+e6v79hRuoMjehzBlHOmONxKnVt05p+5//RvZ6Zk0i6zHd9e8C2vD3/dv/+Azgf4X9tzVvXKDnRHBWs8X/lNpbhZs6vmuZS0G0o31ME4fdLp3PrNrX7XUGllKXPXz+WZWc84cmR5jWV4TUxctGVRwLiIniBpH7OoTZ4suztQz4pvCHm3GirREIuIMxxKKcfrTIlJSfUweVxK2G8/WOULQ+zdu+6vaWiSdG7RmUO6HeLf1r3+U/Y6hWv3V8kFtWUA0D6zPW0yVKhsRnIG2c2yATiixxFkp2X7y31x3hfMvTRwHoM7Cgu81+kQiN327QuEf7xgbX5k62dowcovy+ew1w/j+i+vd0xMTHrQ+b/fWLjRc2KiQPDXDues8i3FSrB0NJREcv6H5wddn0Tz0i8vcc4H5wTs98pvFWpp3XDUd36u+iYaYhFRhsOYsMtaW4ChQyE7O3hZgyGK6Hkc5w843+/OOW/AeYCaL9EitQXpSSpFRo/sHg6Xj50WqS3Yt+O+PHvcs7x3xnv+/U8e86TDMgG474f7Ao7XEVXtM9vv1n1ogYtULJ78+UlAreuhQ4/3Hb9v0HN3eKoDR71xVMB7KYkprN612rFvW/E2qqqrHOlF3v7jbR784cGQdbpy6pVMXKjGf/Ly8/yT+7wsmmmrpwWIVCjslojXeu8A3//1PUkPJDksw3gkGmIxBbjAFxU1DNjVYMYr9FjFgw/C11/Hti6GJoWex6HHIQBePflVtt+yndxWubROb+2fr2GfFBiMa/a/hrP6neXf7prVlU/PdQ6qes3N0BaI12xzgOG9hwe95rJty/yLPdnHICLJybSzdGfIOSFguZW8suimJqUGWAwSyaaiTVRWV/pn4gPsLNsJOFPCe7mTpJR0+b8udHiqA4B/7MTOCe+cUKOFpeyCE8zCeernp6iSVfy09ifP9+OFSEJn3wV+BnoLIfKEEP92ZS+cCqwCVgIvAw1nRsxnKoqCQYOgRYuQRQ2GaKL94Tq1OKgIqey0bD446wMeP+pxTuqtBr6jkcFVpyyxr89x4p4nsmfOngC0zWhLUkKg69e+aFR6crrf+gHlktHRT3o+COCP2gpFcUWxo0EPuG5iakCk2Jfnfel4v7C80C9yB3c9GFCz0wHHubUw2wf4y6vKKSgr4KHpD/n3ffXnV/7XUkrHtptIrQD7NbUV9tuG3+j8dGf/OI+2guJ9xnlYsZBSniOl7CClTJZSdpZSTpBSjtMZDKXiKillLynl3lLK2iWKiRa//w7X+haiMQPbhhjhlQW2X9t+dGvZjd45vXnkiEd45/R3an0d7Wsf2nEooFJ5TD5rMnu33dv/vk4V8uHZH/otDr3ULCgL5+ieR4e9lntNkGDoMQYvMlMyHasKglOEUhJTKCwvpGtWV8ruKuPzkZ8zov8I3l/8PqAWqdIUlheSl5/Hrd9Yi72VV5Vz53d3cvf3d/v3Hfe2NQPAK52KnTZPtAn5vkaH8YI1lvLg9AdZV7CO7/76DrDEwl42Hmm8M7i32noGZglVQ4ywWxZuhBDcfsjt/jU2aoPueeuw4LTkNJITk+mRrRrgdQXrePzox5l81mRO7n2y301z3oDz+Ec3NSM9QSTQPKV52GsN7TSUmRfNZN0N63ji6CeClttYuDFgVrpmW8k27vr+Lsc+7ZYD9dkUlReRkZJBSmIKmSmZPHDYA/737ectLC/k5q9v5j+//se/r6yqLKRYTV8zPeh7NcFtWTz505P8vvF3wLIutVi4Z/PHG41XLHbutF4by8IQI+xunrpg1r9n8fJJL/PZuZ8x4eQJHL/H8YC1UNOh3Q5lUPtBPHzEwyQlJHFqn1NJEAn+cYKs1CyeO15NUBRC+FNraO485M6Aa2Y3y+agrgfRsXlH0pK882eBSv2hky5qLh50cUC59898n403biQ5MZnFVy4mKzWLovIiCssLyUy2wo3ta6h3a2mJxZbiLf5oMs2E3yb4B7W9+HRF7SbRVVVXUVVdxRM/WWK5dOtSbv76Zv/aIBsK1NCtTr+yvmA9I/43goWbPZNhNHga7+JHdsvCzNo21DP/zP2nf+ZxXbJ/5/3Zv7Py6188WDXEy0cv9zemzVObM+/ywJnJ1bIaUBaIbswEgr3b7c3QTkNZtWMVheWFPHj4g/TM7sm/p/zbf6w9tDdYFBeodBz2wfBNN22iTXobSipL/As2ARzb61h/mpQ+bfowasgoxvw4BnCmPLELmd2yWLFtBaf0PsVxbbfV4r/ve6rZ7+X9IprN7l6Eyc7eL+1NenK6YyncH9b84Cijc1lp99TXq75m4eaFLNu2zPM7aeg0XrHYYjNBzap4hnpmyjlT/A1yfePOMeWFXyxslkGCSKBtRltmXzLbUfbCgRdSXFHMozMfZX3BekcvPlyuKD3wfnDXg/1Zdd3WljsHlp2MFGtMxb6Mqt1ikUj/mhea5IRkz+gwIQTH9jrW0cgHI+GBBL467yuO7nU05VXl/hT18zfOZ8nWJQHl56yb49jOy88DrIFvvWBWvE78a3yt6MaNUF6uxCI5GTZtinWNDE2QBJHgGX3UUNANVlJCkr8RdGfQ1SQmJDJ66Gj/eIbdHTR66GhePOFFvjn/G8cxevnatKQ0Su8s5fsLv/e/51472917t0eH2YXEXs4uWAkigZ/zfiYjOYMrh6hgTK+JiFq47CHIXhze/XD/65/W/sS8DfNIfSiVp356imdnPcuf2/8MeTyokOkpy6awaPMiv2Wh78srVDgeaFxiUVICHTrAqaeqpIFSQtu24Y8zGJoYZ/c7G1ANv26EQ7mUwFr8yb6yYIvUFozab5TfjaTp17afv2xqUqpDOO1i4R4jAbjrH5YLKZjVYb/eRYMuAtRA8vG5xwetv573ES6gwG61FJYXMm31NABu+vomrvvyupATFE/c80QAPjr7Iyqq1bK67iioWFmctaVxicUPPp/hVF/Gy8G1TxhnMDRGHj/6cbbevJUWqS38EwcfOvyhkMd8cNYH3HnInZ55qdwD3V2zunruB8sNde7e57Lrtl0B72enZfuTL9pDe+3Yo7b6t+0PQJWsCrBaNNcPu57PR34OhBfFlqkt/a8LywsDrJQlWwJdUJqXT3qZLTdv8Ycm55flU1RRxD4drJT4xg3VEPjBOcDE9OiExxkMjY3EhET/HJBmSc2Q90r+vc+/Qx6Tm5PLQ0c85NnY2tO4T//XdNqmt/Wf2422UPQKeF5oy8Fr3AGUFTH/8vksvWqpXzgqqysd4yF2V9UDhz9ApxZWftNgad7BEh9Qa3S/84dzHox7vOLYXsf6X7fLaEfr9NZ+iym/LJ+i8iJHLi/7kr7xROMSi3XrnNvNAn+oBoMh+ujU5Hu33ZtDuh3iT5LoNTtd9/5DDWzrkFn3iob+6wnBwPYD6d26t79hrqyu9AtRwPlc1xrRfwRjjhzjWXZAuwEU3F7gn9DoXn528ZbF/tcZyRl8cZ41uVALaUZKBgJBflk+heWFDgtpc9HmuLQuGpdYbNqkEgY+/DC8/36sa2MwNBlyc3K55cBbmHz2ZMCyGrx60VVSpSkPNUdj9NDRpCenByRL9EKLRUVVhcMNpUOKg3HLQbew7ZZtAfvTktPITMkMOkfGnvpEL/a0f6f9HcvgJogEmqc2Z9m2ZWwt3kpWahY//OsH9u+0P2VVZUFFsCHTcMM1dodNm6BLF7jjjljXxGBoUiSIBB47+jH/tt8NUx4oFjqVt85K68Xe7fam6I7A9BjLRy/3z1/Q2F1W2pJJEAm8d8Z7rN65OuiAshDCkegxJTGF8qpy/4zrYC4wgN45vVm2bZk/6mvWJbMCyrRIbcGkRZNITkjm0n0vpX/b/ly535XMXjebzUWbPQf3GzKNy7LYvBnatYt1LQyGJo/dZ+9GDxiHsiyCkZuTG7CKoL6WRPrncjxx9BO0SG3BgHYDGNR+UETn1hMItSsrVILH2w++HbAsCy90vfbrtJ9/HETXT2fcjScaj2VRUQEbNhixMBgaAPt33p+khCRuOuCmgPd00sNQlkVNsPfQe2T3YPNNm0NmvHUz46IZ7Crdxdbirfzr43/5jw0lFr1bq4XUtGXhhU49f3CXg/37jFjEkrVr4bnnYI1vyUgjFgZDzGmV1oqKuys836uNZeGFO/mhHlyPFJ2EEeDCQRf6X2uxmH3JbC775DJ+36QSBP522W/+6K9QYrFs2zIADup6kH+fEYtYMmIE/GRbVOS002JXF4PBEJYLB17IWwvecixDWxvcEwKjhRaLVmmt/GlHzup3FoM7DPYvBtWheYegx2sLyr7GSDyLRfyOWSxcqGZoz7HlY7nlFujcOfgxBoMh5hzV8yjkvdKRkrw26NnhV+13VVTOp9FikZWa5Q991dZQl6wuvHXqW5y797lBj7/toNvomd3T4RJLSUzhwC4Hhpxj0lCJT8ti/nw1O/uKK6DSFrGw114xq5LBYIgd1fdUh52ZXVM+O/czXvzlRXLSc/zzNHSkFMDIAYHLstp59KhHefSoRwP2/3jxj1GtZ30Rn2Lxxx/qedw45/4DDwwsazAYGj3RFgqAYZ2HMazzMMCKlIrWOEs8Ep9isWKFc3vUKDjgAOjdOzb1MRgMjRqdhDDU3IvGTvyJxdKl8OCDzn1PPgnp6d7lDQaDoZbs0WoPANbsWhPjmsSO+BvgXrNGCcP+vqn86elGKAwGQ52iEwHqtSmaIiJWCa0yMjJkUdFufvDV1SoS6u674cILjfvJYDDUKdWymgd+eIALBl7gyCAbC4QQxVJK79ztdXnduBQLg8FgaKLESizizw1lMBgMhpAIIY4TQiwTQqwUQtzm8b4QQoz1vb9ACLGP13nsGLEwGAyGRoQQIhF4ATge6AucI4To6yp2PJDre1wGvBTuvEYsDAaDoXExFFgppVwlpSwHJgLDXWWGA29IxSygpRAieO4SjFgYDAZDY6MTsNa2nefbV9MyDuJvnoXBYDA0bZKEEHNt2+OllONt217T2d2RTJGUcV40wsoZDAaDoWFQKaUcEuL9PKCLbbszsH43yjgwbiiDwWBoXPwC5AohegghUoARwBRXmSnABb6oqGHALinlhlAnNZaFwWAwNCKklJVCiNHAl0Ai8KqUcpEQ4grf++OAqcAJwEqgGLgo3HljNilPCFENlOzm4UlAY8/oZe4x/mns9weN/x4b4v2lSSnr3SsUM7GoDUKIuWF8dnGPucf4p7HfHzT+e2zs91cTzJiFwWAwGMJixMJgMBgMYYlXsRgfvkjcY+4x/mns9weN/x4b+/1FTFyOWRgMBoOhfolXy8JgMBgM9YgRC4PBYDCEJe7EIlye9nhBCPGqEGKzEGKhbV8rIcTXQogVvuds23u3++55mRDi2NjUOnKEEF2EEN8LIZYIIRYJIa717W8U9yiEaCaEmCOE+N13f/f79jeK+7MjhEgUQswTQnzq225U9yiEWC2E+EMIMV/nXGps9xgVpJRx80DNRvwT6AmkAL8DfWNdr928l38A+wALbfseB27zvb4NeMz3uq/vXlOBHr7PIDHW9xDm/joA+/heNweW++6jUdwjKhFbpu91MjAbGNZY7s91rzcA7wCfNrbfqa/eq4HWrn2N6h6j8Yg3yyKSPO1xgZRyOrDdtXs48F/f6/8Cp9j2T5RSlkkp/0JN0R9aH/XcXaSUG6SUv/leFwBLUCmQG8U9SkWhbzPZ95A0kvvTCCE6A/8EXrHtblT3GISmcI81It7EosY52OOMdtKXzMv33Na3P67vWwjRHRiM6n03mnv0uWfmA5uBr6WUjer+fDwD3AJU2/Y1tnuUwFdCiF+FEJf59jW2e6w18ZZIsMY52BsJcXvfQohM4APgOillvhBet6KKeuxr0PcopawCBgkhWgIfCiH6hyged/cnhDgR2Cyl/FUIcVgkh3jsa9D36OMgKeV6IURb4GshxNIQZeP1HmtNvFkWNc7BHmds0ksb+p43+/bH5X0LIZJRQvG2lHKyb3ejukcAKeVOYBpwHI3r/g4CThZCrEa5fI8QQrxF47pHpJTrfc+bgQ9RbqVGdY/RIN7EIpI87fHMFOBC3+sLgY9t+0cIIVKFED1Qi6zPiUH9IkYoE2ICsERK+bTtrUZxj0KINj6LAiFEGnAUsJRGcn8AUsrbpZSdpZTdUf+176SU59GI7lEIkSGEaK5fA8cAC2lE9xg1Yj3CXtMHKgf7clQUwp2xrk8t7uNdYANQgeqt/BvIAb4FVvieW9nK3+m752XA8bGufwT3dzDKPF8AzPc9Tmgs9wgMAOb57m8hcI9vf6O4P4/7PQwrGqrR3CMqsvJ332ORblMa0z1G62HSfRgMBoMhLPHmhjIYDAZDDDBiYTAYDIawGLEwGAwGQ1iMWBgMBoMhLEYsDAaDwRAWIxYGg8FgCIsRC4PBYDCE5f8B0W1Ocq8ApX0AAAAASUVORK5CYII=\n",
|
|
"text/plain": [
|
|
"<Figure size 432x288 with 2 Axes>"
|
|
]
|
|
},
|
|
"metadata": {
|
|
"needs_background": "light"
|
|
},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"iters = [item[\"iter\"] for item in result]\n",
|
|
"fig, ax1 = plt.subplots()\n",
|
|
"ax1.plot(iters,[item[\"loss\"] for item in result],'g')\n",
|
|
"ax2 = ax1.twinx()\n",
|
|
"ax2.plot(iters,[item[\"accuracy\"] for item in result],'r')\n",
|
|
"plt.show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "2bb67740",
|
|
"metadata": {},
|
|
"source": [
|
|
"학습 그래프입니다."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 31,
|
|
"id": "0defca72",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"torch.cuda.empty_cache()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 32,
|
|
"id": "2f45cae0",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"gpu allocated : 693 MB\n",
|
|
"gpu reserved : 756MB\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"print(f\"gpu allocated : {torch.cuda.memory_allocated() // 1024**2} MB\")\n",
|
|
"print(f\"gpu reserved : {torch.cuda.memory_reserved() // 1024 ** 2}MB\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "68f18f4d",
|
|
"metadata": {},
|
|
"source": [
|
|
"gpu 메모리 사용량을 보는 코드"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 33,
|
|
"id": "73b01630",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"del batch_inputs\n",
|
|
"del batch_labels\n",
|
|
"del loss\n",
|
|
"del optimizer"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "02a3f367",
|
|
"metadata": {},
|
|
"source": [
|
|
"한번 테스트 해보자"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 34,
|
|
"id": "af93a3ec",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"김광현 을 둘러싼 주변 의 우려 는 너무 많이 던진다는 것 .\n",
|
|
"두산 은 주포 인 김동주 와 안경현 이 빠진 상황 이 라 타력 에 적 지 않 은 문제점 을 안 고 있 지만 29 일 잠실 롯데전 이 우천 으로 취소 되 는 바람 에 시간 을 벌 었 고 더구나 삼성 이 4 연패 로 2 위 로 추락 해 어부지리 로 1 위 로 올라서 는 행운 을 잡 았 다 .\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for data in datasetTrain[100:102]:\n",
|
|
" print(tokenizer.convert_tokens_to_string(data[\"tokens\"]))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 35,
|
|
"id": "9b2cd5c4",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"{'input_ids': tensor([[ 101, 8935, 118649, 30842, 9633, 9105, 30873, 119091, 9689,\n",
|
|
" 118985, 9637, 9604, 26737, 9043, 9004, 32537, 47058, 9076,\n",
|
|
" 65096, 11018, 8870, 119, 102, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [ 101, 9102, 21386, 9632, 9689, 55530, 9640, 8935, 18778,\n",
|
|
" 16323, 9590, 9521, 31720, 30842, 9638, 9388, 18623, 9414,\n",
|
|
" 65649, 9638, 9157, 9845, 28143, 9559, 9664, 9706, 9523,\n",
|
|
" 9632, 9297, 17730, 34907, 9633, 9521, 8888, 9647, 9706,\n",
|
|
" 19105, 10386, 9641, 9655, 31503, 9208, 28911, 16617, 9638,\n",
|
|
" 9604, 38631, 29805, 9773, 22333, 9098, 9043, 9318, 61250,\n",
|
|
" 9559, 9485, 18784, 9633, 9339, 9557, 8888, 9074, 17196,\n",
|
|
" 16439, 9410, 17138, 9638, 125, 9568, 119383, 9202, 123,\n",
|
|
" 9619, 9202, 9765, 107693, 9960, 9546, 14646, 12508, 12692,\n",
|
|
" 9202, 122, 9619, 9202, 9583, 17342, 12424, 9043, 9966,\n",
|
|
" 21614, 9633, 9656, 9529, 9056, 119, 102]],\n",
|
|
" device='cuda:0'),\n",
|
|
" 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0],\n",
|
|
" [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
|
|
" 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
|
|
" 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
|
|
" 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
|
|
" 1]], device='cuda:0'),\n",
|
|
" 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0]], device='cuda:0')}"
|
|
]
|
|
},
|
|
"execution_count": 35,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"inputs, labels = my_collate_fn(datasetTrain[100:102])\n",
|
|
"inputs = {k:v.to(device) for k,v in inputs.items()}\n",
|
|
"inputs"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 36,
|
|
"id": "6b31782c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"model.eval()\n",
|
|
"with torch.no_grad():\n",
|
|
" predict_label = model(**inputs)\n",
|
|
" sp = model.softmax(predict_label)\n",
|
|
" p = sp.argmax(dim=-1,keepdim=True).squeeze().cpu()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 37,
|
|
"id": "7f4d43ce",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"['O', 'O', 'I-PS', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
|
|
"['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for data in p.numpy():\n",
|
|
" print(tagIdConverter.convert_ids_to_tokens(data))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 38,
|
|
"id": "5ade3317",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"['O', 'B-PS', 'I-PS', 'I-PS', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
|
|
"['O', 'B-OG', 'I-OG', 'O', 'O', 'O', 'O', 'B-PS', 'I-PS', 'I-PS', 'O', 'B-PS', 'I-PS', 'I-PS', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-DT', 'I-DT', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-OG', 'I-OG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for data in labels.squeeze().numpy():\n",
|
|
" print(tagIdConverter.convert_ids_to_tokens(data))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "b48e22ff",
|
|
"metadata": {},
|
|
"source": [
|
|
"It work!"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 39,
|
|
"id": "383dd24a",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"torch.Size([194, 1])"
|
|
]
|
|
},
|
|
"execution_count": 39,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"sp.cpu().view(-1,sp.size(-1)).argmax(dim=-1,keepdim=True).size()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 40,
|
|
"id": "ff74fced",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor(106)"
|
|
]
|
|
},
|
|
"execution_count": 40,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"correct = (sp.cpu().view(-1,sp.size(-1)).argmax(dim=-1) == labels.view(-1)).sum()\n",
|
|
"correct"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 41,
|
|
"id": "3f6ad5d8",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor(120, device='cuda:0')"
|
|
]
|
|
},
|
|
"execution_count": 41,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"inputs[\"attention_mask\"].view(-1).sum()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 42,
|
|
"id": "986fd52b",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor(0.8833, device='cuda:0')"
|
|
]
|
|
},
|
|
"execution_count": 42,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"accuracy = correct / inputs[\"attention_mask\"].view(-1).sum()\n",
|
|
"accuracy"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "8cc47437",
|
|
"metadata": {},
|
|
"source": [
|
|
"accuracy는 다음과 같이 구해져요."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 43,
|
|
"id": "1f3f8666",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"100%|███████████████████████████████████████████████████████████████████████████████| 16/16 [00:02<00:00, 5.66batch/s]\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"model.eval()\n",
|
|
"collect_list = []\n",
|
|
"with torch.no_grad():\n",
|
|
" with tqdm(test_loader, unit=\"batch\") as tepoch:\n",
|
|
" for batch_i,batch_l in tepoch:\n",
|
|
" batch_inputs = {k: v.cuda(device) for k, v in list(batch_i.items())}\n",
|
|
" batch_labels = batch_l.cuda(device)\n",
|
|
" output = model(**batch_inputs)\n",
|
|
" loss = CELoss(output.view(-1, output.size(-1)), batch_labels.view(-1))\n",
|
|
" \n",
|
|
" prediction = output.view(-1, output.size(-1)).argmax(dim=-1)\n",
|
|
" correct = (prediction == batch_labels.view(-1)).sum().item()\n",
|
|
" accuracy = correct / batch_inputs[\"attention_mask\"].view(-1).sum()\n",
|
|
" \n",
|
|
" collect_list.append({\"loss\":loss.item(),\"accuracy\":accuracy, \"batch_size\":batch_labels.size(0),\n",
|
|
" \"predict\":output.argmax(dim=-1).cpu(),\n",
|
|
" \"actual\":batch_labels.cpu(),\n",
|
|
" \"attention_mask\":batch_inputs[\"attention_mask\"].cpu()})"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 44,
|
|
"id": "b7567f48",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def getConfusionMatrix(predict,actual,attention_mask):\n",
|
|
" ret = torch.zeros((22,22),dtype=torch.long)\n",
|
|
" for i,(p_s,a_s) in enumerate(zip(predict,actual)):\n",
|
|
" for j,(p,a) in enumerate(zip(p_s,a_s)):\n",
|
|
" ret[p,a] += attention_mask[i,j]\n",
|
|
" return ret"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "a898cd34",
|
|
"metadata": {},
|
|
"source": [
|
|
"단순하게 confusion matrix를 계산하는 함수. 클래스 22개 정해져 있음."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 45,
|
|
"id": "15cd73a5",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor([[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])"
|
|
]
|
|
},
|
|
"execution_count": 45,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"getConfusionMatrix(torch.tensor([[0,1]]),torch.tensor([[1,1]]),torch.tensor([[1,1]]))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 46,
|
|
"id": "de9c7932",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"average_loss : 0.7903580265045166, average_accuracy : 0.8021594285964966, size :500\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"total_loss = 0\n",
|
|
"total_accuracy = 0\n",
|
|
"total_size = 0\n",
|
|
"confusion = torch.zeros((22,22),dtype=torch.long)\n",
|
|
"\n",
|
|
"for item in collect_list:\n",
|
|
" batch_size = item[\"batch_size\"]\n",
|
|
" total_loss += batch_size * item[\"loss\"]\n",
|
|
" total_accuracy += batch_size * item[\"accuracy\"]\n",
|
|
" total_size += batch_size\n",
|
|
" confusion += getConfusionMatrix(item[\"predict\"],item[\"actual\"],item[\"attention_mask\"])\n",
|
|
"print(f\"\"\"average_loss : {total_loss/total_size}, average_accuracy : {total_accuracy/total_size}, size :{total_size}\"\"\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "24539b98",
|
|
"metadata": {},
|
|
"source": [
|
|
"test로 보면 결과가 나왔어요. 84% 나와요. F1 스코어는 아직입니다."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 58,
|
|
"id": "f6047991",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor([[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 2, 2, 27, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0],\n",
|
|
" [ 0, 0, 1, 0, 167, 126, 466, 421, 40, 0,\n",
|
|
" 0, 0, 5, 0, 375, 166, 1005, 1154, 101, 0,\n",
|
|
" 0, 16409]])"
|
|
]
|
|
},
|
|
"execution_count": 58,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"confusion"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 57,
|
|
"id": "000d1e68",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAATQAAAEICAYAAADROQhJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAxhElEQVR4nO2deZwVxbXHv2dYlV2GfUdGlmEZZhD0YeIWNzQmGo0SY8xTH2LEbBqfxCQueURjfMl7ionB6DMxRKNxiQtRiSYuMVFBATGKqAEFkUU2kQEHOO+Prhmay517+259q5r68anPvd19un59+vYUVXXOr0tUFQ8PD48koKLcF+Dh4eFRLPgGzcPDIzHwDZqHh0di4Bs0Dw+PxMA3aB4eHomBb9A8PDwSA9+gJRAS4P9EZIOIvFhAPZ8SkSXFvLZyQUT6i8gWEWlR7mvxKB3E56ElDyLyKeAuYKiqflzu6yk1RGQZcL6q/rnc1+JRXvgeWjIxAFi2LzRmUSAiLct9DR7xwDdoZYaI9BOR+0VkrYh8KCIzzf4KEfmeiCwXkTUi8hsR6WSODRQRFZFzRORdEVknIleYY+cBvwIONUOsq0XkqyLyXAqvisgQ832SiPxTRD4SkZUicqnZf4SIrAidM1xE/ioiG0XkNRE5OXTsDhG5WUQeNfW8ICIHNuNz4/X/u4i8Z4bGU0XkYBFZZOqfGbI/UESeMvdnnYjMFpHO5tidQH/gYePvZaH6zxORd4GnQvtaisgBIrJCRD5r6mgvIm+JyFcK/T09ygxV9aVMBWgBLAR+BrQD2gKHmWPnAm8Bg4H2wP3AnebYQECBW4H9gDHAdmC4Of5V4LkQzx7bZp8CQ8z3VcCnzPcuQK35fgSwwnxvZa7nu0Br4CjgI4JhLcAdwHpgPNASmA3c3Yzfjdd/i/H5WGAb8CDQHegDrAEON/ZDgGOANkA34Bngf0L1LQM+k6b+35j7ul9oX0tjcyzwgeG7FfhDuZ8HXwovvodWXowHegPfUdWPVXWbqjb2pM4Cfqqq76jqFmA6cGbK8OlqVa1X1YUEDeOYPK+jARghIh1VdYOqvpzG5hCChvU6Vf1EVZ8CHgEmh2zuV9UXVXUHQYNWk4X3h8bnJ4CPgbtUdY2qrgSeBcYCqOpbqjpXVber6lrgp8DhEfy6ytzX+tQDhvNe4EngROCCCPV5WA7foJUX/YDlpgFIRW9geWh7OUHPp0do3weh71sJGpx88AVgErBcRJ4WkUObuZ73VHVXyjX1KeB6Voe+16fZbg8gIt1F5G4zHN4M/BaozFI3wHtZjs8CRgL/p6ofRqjPw3L4Bq28eA/o38yk9fsEk/uN6A/sYM8/+qj4GNi/cUNEeoYPqupLqvo5guHXg8A9zVxPPxEJPzP9gZV5XE+uuJZguDhaVTsCXwYkdLy5UH2zIXyTvvFLgmHphY3ziR5uwzdo5cWLBPNX14lIOxFpKyITzbG7gG+JyCARaQ/8CPh9M725bFgIVItIjYi0Ba5qPCAirUXkLBHppKoNwGZgZ5o6XiBoGC8TkVYicgTwWeDuPK4nV3QAtgAbRaQP8J2U46sJ5hpzwXfN57nADcBvfI6a+/ANWhmhqjsJGoUhwLvACuAMc/h24E6CCfB/EUyaX5wnz5vANcCfgaXAcykmZwPLzHBuKkEPKLWOT4CTgROAdcDPga+o6hv5XFOOuBqoBTYBjxIESMK4FvieiY5emq0yEakDvk1w/TuBHxP05i4v6lV7xA6fWOvh4ZEY+B6ah4dHYuAbNA8Pj8TA+gZNRI4XkSUmkzvjHEdU22Lbee5kcCfNn1JwW49yZ/ZmKgSZ9G8TRLBaE0TrRhRiW2w7z50M7qT5UwpuF4qVQYHKykodMGAgW7ZsYdWqVVRVVQHwwQerAOjZs9de50S1Lbad504Gd9L8yWa3fPky1q1bJwAmkfoqVT3ObE8HUNVr97rQEFp0HKC6Yy8RRlpo/drHVfX4SMaFoNwtarpSW1un9Q2qs+++V7/67+dpfYNqfYPqbf/3G73gwouatsMlqm2x7Tx3MriT5k82u9raOg310E4DfhXaPhuYmbU3tF93bTv24kgFmBdH21HQHFq2cbcEuNEcXyQitbk2tmnqLMi22HaeOxncpajTFW72VF00VdGc8R5niUQrMSHvBs1kVd9MkGg5ApgsIiNSzE4AqkyZAvwiF44+ffqyYsVuOd7KlSvo3bt3QbbFtvPcyeBOmj+5cBMkdPcLbfclkLplh1REK3Eh364dcCjweGh7OjA9xeaXwOTQ9hKgV9Qh50f1DTpw0CB9/c13dNPH23XUqNE6f8HitF32qLbFtvPcyeBOmj/Z7FKGnC2Bd4BB7A4KVGcdcu7fXduO+1akQkxDzkLe5NmHPd9msAKYEMGmD4F+MStatmzJz/53Jp898Th27tzJOV89lxHV1QXZFtvOcyeDO2n+5MKtqjtEZBrwOEHE83ZVfS2t8R4QqLBL/pp3lFNETgeOU9XzzfbZwHhVvThk8yhwrZp3fInIk8Blqjo/TX1TCIal9Ovfv+7Nt5enmnh4eBQJEyeMY/78eQVNblW076ltRp4TyXbbC9fPV9VxhfBFQSGD2yjj7shjc1WdparjVHVct8puBVyWh4dHPIgYEHAhKAC8BFRJ8Hqb1sCZwEMpNg8BXzHRzkOATaoaabjp4eHhACwLCuTNpMF7uRrH3a8D96jqaxIsdjHVmM0hmGx8i+C97V/LleeJxx9jdPVQqocN4SfXX1cU22Lbee5kcCfNn1y485Y+WdZDK3nUIZ/SGOXcsm2HDho8WP+55O2mSM3LC19LGyWKaltsO8+dDO6k+ZPNLiXKmZf0Sdr10rYTvxep4EJibanx0osvcuCBQxg0eDCtW7fm9DPO5JGH/1iQbbHtPHcyuJPmTy7cBIv1vKXBgjyfELyF+HPNGTdBCKKcUUpMsLpBe//9lfTtuzum0KdPX1auTP8K+6i2xbbz3MngTpo/uXDTfHpVFkhy5tDiQLqUEhulI57bfe5S1OkKN/lKnwAqJFrJAAkW2/6LiLwuwQLW3zD7r5Jgpa8FpkzKdjmFJNaWHK5IRzy3+9xJ8ycW6ZNQrN7XDuASVX1ZRDoA80Vkrjn2M1W9IXJN5Q4AZAoK2Cwd8dzJ4k6aP7FInzr01rZHzYhUyCEoAPwROIZgdbJLc2k7rO6huSId8dzucyfNHwulT5UiMi+0PUtVZ+1Vo8hAYCzBsokTgWki8hVgHkEvbkPGK0o31i436urG6d9emJfd0MPDIy8URfrUsa+2OeQbkWy3zb0sq/RJgvVnnwZmqOr9ItKDYMlEBX5I8GKLczPVYXUPzcPDw2IUMWlWRFoB9wGzVfV+AFVdHTp+K/BItnqsjnKCO5nWntt97qT5E49SoPC0DQnCr7cBr6vqT0P7w+8VPwVYnPV6yh0AyBQUsDnT2nMniztp/sSiFOjYV9se/9NIhQxBAeAwgmHlImCBKZOAO4FXzf6HiPAuRat7aK5kWntu97mT5k8sSoEiJdaq6nOqKqo6WlVrTJmjqmer6iiz/2SN8GILqxs0VzKtPbf73EnzJxalgIXSJ6uDAukisDZmWntu97lLUacr3OStFJBYZU1RYHWD5kqmted2nztp/sS3SEqMrwaKgnIHADIFBWzOtPbcyeJOmj+xKAU69de2J98SqeDAIiklhyuZ1p7bfe6k+ROPUgDremheKeDhsQ+iKEqBLgO1zRHfi2S77cH/sHuRlOZe+ZFic4SIbAq9/uMHhV2uh4eHTZCKikglLhTC1PjKj+HAIcBFaVZOB3g2lFtyTa4krmRae273uZPmT6mVAhKcF6nEhmJNxmFe+ZGy7wjgkXyDAjZnWnvuZHEnzZ84lAIVXQbo/qfdHqng0poCKa/8SMWhIrJQRP4kIulnJYM6pojIPBGZt3bdWsCdTGvP7T530vyJSylgWw+t4AbNvPLjPuCbqro55fDLwABVHQPcBDzYXD2aZqFhVzKtPbf73EnzJ541BewbchbUoKV75UcYqrpZVbeY73OAViJSGbX+dBFYGzOtPbf73KWo0xVuClhToKKiIlKJC4VEOdO+8iPFpqexQ0TGG74Po3K4kmntud3nTpo/8a0pELHEhQKCAM298mMqMNXYTANeI5hk/Afwb7kEBWzOtPbcyeJOmj9xKAVaHDBIO33pzkgF25UCqvocWdpeVZ0JzMyXw5VMa8/tPnfS/IlLKRBrSkYEeKWAh8c+iGIoBVp2HawdJ/1XJNsNvz0rFqWA1VpODw8Pu2FbD803aB4eHvlBQLKsih437Ho7Wxq4Ih3x3O5zJ82f0kuf7EusLXnUIZ/ipU+e20uf7Jc+tew6WLv9++8jFVySPpUKrkhHPLf73EnzJx7pE9bloVndoLkiHfHc7nMnzZ+4FkmxbchpdVAgXUqJjdIRz+0+dynqdIWbAqRPPsqZA1yRjnhu97mT5k8c0idBYtVpRkK5AwCZggI2S0c8d7K4k+ZPHNKnVpUHaq8L7otUsF36FAdckY54bve5k+ZPLNInsW/I6aVPHh77IIohfWrdfYh2P+2GSLYrf3GKlz55eHjYDdt6aJbN6O0NVzKtPbf73Enzp9RKAQikT1FKbCh3ACBTUMDmTGvPnSzupPkTh1Kgdfch2v/ihyIVvFLAnUxrz+0+d9L8iUspYFtirdUNmiuZ1p7bfe6k+eMXSckDIrJMRF6VYFX0vcKSEuBGMy5fJCK1udSfLgJrY6a153afuxR1usJNAUqBYmg5RaSfiPxFRF4XkddE5Btm/wEiMldElprPLtkupxg9tCM1WBU9XUj2BKDKlCnAL3Kp2JVMa8/tPnfS/IllkRSK1kPbAVyiqsOBQ4CLRGQEcDnwpKpWAU+a7cwoZAIOWAZUZjj+S2ByaHsJ0CtqUMDmTGvPnSzupPkTh1KgTY8heuAlcyIVcggKAH8Ejgm3F0AvYEm2cwvNQ1PgCRFR4JeqOivleHNj81WpFYnIFIJeHP369wfcybT23O5zJ82feBZJyWl+rDJlWmpWmvYCERkIjAVeAHqo6ipzjatEpHvWK0o31o4KEemtqu8bornAxar6TOj4o8C1GqwQhYg8CVymqvMz1euVAh4epUUxlAJtex6k/b9yYyTbpT85IatSQETaA08DM1T1fhHZqKqdQ8c3qGrGebSC5tBU9X3zuQZ4gCD8G0beY3MPDw/7Uawop4i0Au4DZqvq/Wb3ahHpZY73AtZkq6eQldPbiUiHxu/AscDiFLOHgK+YaOchwKbGLqSHh4fjEJCIJWM1QYt3G/C6qv40dOgh4Bzz/RyCubWMKKSH1gN4TkQWAi8Cj6rqYyIyVUSmGps5BJONbwG3Al/LlcQV6Yjndp87af6UfpEUqKiQSCULJgJnA0eZFLAFIjIJuA44RkSWEgQJMjsBXvrkomzFc7tRp83cxZA+te1ZpdVXPBGp4KVP7khHPLf73EnzJxbpU5GGnMWE1Q2aK9IRz+0+d9L8iUP6JNgnfbL6fWjpUkpslI54bve5S1GnK9zkLX2KeRHhCLC6QXNFOuK53edOmj/xSZ+iWMWIcgcAMgUFbJaOeO5kcSfNnzikT/v1PkjHXv1kpIJfJMUd6Yjndp87af7EIX1qnEOzCX6RFA+PfRDFkD616zNUh194SyTb+d8/yi+S4uHhYTds66FZnbYB7mRae273uZPmTyyLpFiWh1b2AECmoIDNmdaeO1ncSfMnDqXA/n0O0vE/+mukglcKuJNp7bnd506aP3EoBYRoOs4IWs6iweoGzZVMa8/tPnfS/IlvkRS7hpxWBwXSRWBtzLT23O5zl6JOV7gpYJEU24ICVjdormRae273uZPmTyxKgbgn/KOg3AGATEEBmzOtPXeyuJPmTxxKgfZ9h+phNzwbqeCVAu5kWntu97mT5k88i6TYN+T0SgEPj30QxVAKdOg3TOsuuT2S7dPfmhiLUqCQNQWGhl6Xu0BENovIN1NsjhCRTSGbHxR8xR4eHnYgSS94VNUlGqyYXgPUAVsJVn5KxbONdqp6Ta48rmRae273uZPmT+nXFIj2csdYh6XFmIgjWPHpb2n2HwE8km9QwOZMa8+dLO6k+ROHUqBDv2F61I3PRyo4phQ4E7irmWOHishCEfmTiKSflQREZIqIzBOReWvXrQXcybT23O5zJ82fWNYUACpEIpW4UHCDJiKtgZOBe9McfhkYoKpjgJuAB5urR1Vnqeo4VR3XrbIb4E6mted2nztp/sSypoAUbRm7oqEYPbQTgJdVdXXqAVXdrKpbzPc5QCsRqYxacboIrI2Z1p7bfe5S1OkKNwUoBSokWokLxchDm0wzw00R6QmsVlUVkfEEDeiHUSt2JdPac7vPnTR/4ltTwK48tEKDAfsTNFCdQvumAlPN92nAawSTjP8A/i2XoIDNmdaeO1ncSfMnDqVAx/7DdNItL0QquKAUUNWtQNeUfbeEvs8EZuZbvyuZ1p7bfe6k+RPbmgJpR6vlg1cKeHjsgyiGUqDzgOH66St+E8n24QvG+zUFPDw8LIbEG8GMAt+geXh45AWBWHPMosDqN9aCO9IRz+0+d9L88YukWFK89Mlze+mT/dKnzgOG6xdunx+p4Jj0qSRwRTriud3nTpo/sSySUsS3bYjI7SKyRkQWh/ZdJSIrQ2/rmZStHqsbNFekI57bfe6k+RPXIiktRCKVCLgDOD7N/p/p7rf1zMlWidVBgXQpJTZKRzy3+9ylqNMVbixYJEVVnxGRgYXWY3UPzRXpiOd2nztp/sQhfQqinJG1nJWNb9MxZUq2+g2micgiMyTtktW63AGATEEBm6UjnjtZ3EnzJw7p0wGDRuhZdy6IVIgQFAAGAotD2z0IAhYVwAwCBUPGOqwecroiHfHc7nMnzZ/4FkmJYpUfNPQGHxG5FXgk6/WkG2uXG1765OFRWhRD+tR1cLWe+F/Nvdd1T9x51pis0iczh/aIqo40271UdZX5/i1ggqqemakOq3toHh4e9kKAFkWSPonIXQSv7K8UkRXAlcARIlJDEKBYBlyQrR6rgwLgTqa153afO2n+xKIUiFiyQVUnq2ovVW2lqn1V9TZVPVtVR6nqaFU9ubG3lq0i64pXCnhurxSwXylQOXiEnnf3q5EKXingTqa153afO2n+xLVIim1aTqsbNFcyrT23+9xJ8ycupYBt63JmbdCa0VgdICJzRWSp+Uyb8JbvuLwR6SKwNmZae273uUtRpyvcFKQUcK+Hdgd7a6wuB55U1SrgSbO9B0SkBXAzwapQI4DJIjIil4tzJdPac7vPnTR/YlEKiNCiIlqJDVEm2tg7g3cJ0Mt87wUsSXPOocDjoe3pwPRcggI2Z1p77mRxJ82fOJQC3Q6s1q/d/89IBcsXSemhJoSqqqtEpHsam3Tj8gnNVWi0XVMA+vXvD7iTae253edOmj9xKQVsm4SPpBRIk8G7UVU7h45vUNUuKeecDhynqueb7bOB8ap6cTY+rxTw8CgtiqEU6DFkpJ5xwx8i2d50yvBYFknJt4FdLSK9IJAnAGvS2OS9eKmHh4cbsG3l9HwbtIeAc8z3c4B0CS4vAVUiMkhEWgNnmvNygiuZ1p7bfe6k+VNqpYAI7gUFgLuAVUADQa/rPILFhZ8ElprPA4xtb2BO6NxJwJsEWchXRJ3Y80oBz+2VAvYrBXoMqdZLH34jUsEWpYCm11h9qKpHq2qV+VxvbN9X1Umhc+eo6kGqeqCqzsi1sXUl09pzu8+dNH+8UsBCuJJp7bnd506aP3EoBYRgXc4oJS5Y/fqgdBFYGzOtPbf73KWo0xVuClAK2NYjsrpBcyXT2nO7z500f+JQCkC8w8lIiGOiLtfilQKe2ysF7FcK9Koaqd9/7M1IBcuVArHAlUxrz+0+d9L8iU0pYFkPza8p4OGxD6IYSoE+B43SC25+IJLtlcdWxaIUsLqH5uHhYTdsm0PzDZqHh0d+iFnWFAW2RV33givSEc/tPnfS/IlnkZRo/2JDuSOamaKcNktHPHeyuJPmTxzSpz4HjdQfP/VWpIIt0qdywhXpiOd2nztp/sQnfXJsTYFywhXpiOd2nztp/sQnfbLr9UFWBwXSpZTYKB3x3O5zl6JOV7jJV/oUs/A8Cqxu0FyRjnhu97mT5k9c0qc4heeRUO4AQKaggM3SEc+dLO6k+ROH9Knf0JF643PvRCp46ZM70hHP7T530vyJR/okVMSZkhEBXvrk4bEPohjSpwHDRut/3h7trfoXTRxkxyIpzayc/hMReUNEFonIAyLSuZlzl4nIqyKyQER8C+XhkSREjHDatkjKHey9cvpcYKSqjiZYM2B6hvOPVNWafFtnVzKtPbf73EnzJw6lgG1vrI000UbKyukpx04BZjdzbBlQmW9QwOZMa8+dLO6k+ROHUmDAsFE66x/LIhWyBAWA2wmWw1wc2ncAQedpqfnsku2aipFYey7wp+baS+AJEZlvVkZvFiIyRUTmici8tevWAu5kWntu97mT5k9cSoEiLmN3B3uPBC8HnlTVKoLV5bL2HAtq0ETkCmAHMLsZk4mqWgucAFwkIp9uri5VnaWq41R1XLfKboA7mdae233upPkTm1IgYskGVX0GWJ+y+3PAr833XwOfz1ZP3g2aiJwDnAScpc2ESlX1ffO5BniA4H+CyEhXrY2Z1p7bfe5S1OkKNwUpBSJrOSsbR2CmZByxGfRQ1VXGn1VA92wn5JWHJiLHA/8JHK6qW5uxaQdUqOpH5vuxwDW58LiSae253edOmj+xLZISxSjAujjSNqIEBNKtnP4WQRd1gSm3GNumldMJJhgXmvIaeaycbnOmtedOFnfS/IlDKTBo+Ci9c957kQoRlAKkBB+BJUAv870XsCRbHVl7aKo6Oc3u25qxfR+YZL6/A4zJVn8muJJp7bnd506aP3EtklLihIyHgHOA68xns1GNputpZvqrrPBKAQ+P0qIYSoHBI8bojNlzItl+qbZvRqWAiNwFHAFUAquBK4EHgXuA/sC7wOmqmho42ANWazk9PDzsRWOUsxhoZiQIcHQu9Vj9gkdwJ9Pac7vPnTR/YllTwLI31uaUwR9X8UoBz+2VAvYrBQYPH633vLIyUsGvKeBOprXndp87af7EohTILQ8tFljdoLmSae253edOmj9xKQVaiEQqccHqoEC6CKyNmdae233uUtTpCjf5KgWaObGcsLpBcyXT2nO7z500f2JTCtjWopU7AJApKGBzprXnThZ30vyJQykwZMRofWjRB5EKfk0BdzKtPbf73EnzJzalgGU9NK8U8PDYB1EMpUBVdY3+z++fiGR70qgesawpYHUPzcPDw140Rjltgm/QPDw88oPYN+S0Og8N3JGOeG73uZPmTzzSp2glNpQ7opkpymmzdMRzJ4s7af7EIX2qqh6jT/xzbaSClz65Ix3x3O5zJ82fOKRPgpvrcpYNrkhHPLf73EnzJw7pE9i3Lme+K6dfJSIrzYroC0RkUjPn5jUub0S6lBIbpSOe233uUtTpCjcFSZ+i/YsL+a6cDvAzDVZEr1HVvV5bKSItgJsJlrAbAUwWkRG5XJwr0hHP7T530vyJQ/pk45Az0kQbey9ecBVwaZZzDgUeD21PB6bnEhSwWTriuZPFnTR/4pA+Da2u0WeWrI9UcED6NE1EvgLMAy5R1Q0px9ONyyc0V5lZp28KQL/+/QF3pCOe233upPkTi/TJwjy0SNInERkIPKKqI812D2AdwTj7hwRLTZ2bcs7pwHGqer7ZPhsYr6oXZ+Pz0icPj9KiGNKnYaPG6m33PxXJ9rCDDrBX+qSqqxu/i8itwCNpzPJ+JYmHh4f9sFH6lFfahoj0Cm2eAixOY/YSUCUig0SkNXAmwTp7OcGVTGvP7T53VNsLzj+X/r27U1czMmNd5fYnDqUAErHEhQgBgXQrp98JvAosImikGlc3blo53WxPAt4kyEK+IurEnlcKeG6blQJzn3pan39hvo6ork5bjw3+xKEUGDayRv/x1sZIBVuUAqo6WVV7qWorVe2rqrep6tmqOkpVR6vqyaq6yti+r6qTQufOUdWDVPVAVZ2Ra2PrSqa153afO4rttoadbGvYybhDJtKuYyd26e594WKDP7EskoJ9Wk6vFHA8y9tzx19nVLhyLylAKWDbiNPq1weli8DamGntud3nztU2Cly5lxSgFLBtlRSrGzRXMq09t/vcudpGgSv3knyVAkKsOs1IiGOiLtfilQKe20alwIatO5rKwtff0mEjqvfY11hs8CcOpcDwUTU6/1+bIhViCgqUvfHK1KDVN6g+8NCjOqSqSgcNHqxXXfNfGaNKUW2Lbee5k8GdzbaxwTr19DO0R4+e2rJlS+3du4/e+PNZaRu0cvuTyS7coJlGLeeMhOGjanT+sk2RSlwNml8kxcMjIsIRzExo26pFia+kcBRDKTBidK3OfvjpSLa1AzvaqxTw8PDwAPu0nFanbYA7mdae233u9957j+M+cyQ1o4ZTO6aamTf+b1q7FSve47PHH82EsSM5tG40t9x84x7H0tXx5S+dwYS6GibU1TB0yEAm1NWU3J9SKwWE4uWhicgyEXnVvF8x/+FZuefLMs2h2Zxp7bmTxV3foPrOu+/r8y/M1/oG1TXrN+uQqqo9bBvnyF5/+z39699e1A1bd+i7qzfogUOq9O/zFzUdy1RHfYPq17/5bf3+lVeX9V4WQykwYtRYXfjuR5EKWebQgGVAZaFth9U9NFcyrT2329ybtjawaWsD+3eqZPCwUWza2sCuFm05sGooS99Z3nR8e8MutjfsoktlD4aNrGF7wy5at23HkIOG8u67K5qOHXXB7XQ5eBr9j5rOO6sbmHDqdLocPI0uB0+j87iLuPHmWfz4njfpcvA0K+4lXikQD1zJtPbc7nOn4t3ly1i8aCG148bnbbdr+2Z21a+jYv8eTfv041VIy/2oaNO5pP44qBRQ4AkRmW/ejZgXrA4KpIvA2php7bnd5w7j4y1bOP/sM7jm2hvo0LFjXna68xMalj1Gqz6HIS1aN+3fueFNWnSpKrk/Ofqdn1IgN11TZcrc2CxVnRXanqiq74tId2CuiLyhqs9Ert3A6gbNlUxrz+0+dyMaGho47+wzOPWLkznx5FPyslPdScOyx2jR5SBadD4wtH8XOze9Q5uDvlhyf+JQCgC5LICyLlPahqq+bz7XiMgDBMPgnBu0sgcAMgUFbM609tzJ4f5g0yf6waZPdNXG7XramWfpf1x4cdO+dCWT3aqN27Wiy1BtUTla29ZctEdpNfgklXa999hXrntZDKVA9eix+s+VWyIVMgQFgHZAh9D354Hj82k7rO6hufJOds/tPjfAi/94nj/cPZvh1SM5+rCgMzH9Bz/kM8eeENnuxX88z64NS5C2Xdn+xt3BNfQ+hBYdB7Jzw1tph5vlvpea75oCUCxxeg/gATMkbgn8TlUfy6cirxTw2OexaWtDUesbePi3IttueGlmUbmjohhKgZFjavUPjz0XyXZ473ZeKeDh4WE3bFMKZG3QROR24CRgje5e9en3wFBj0hnYqKo1ac5dBnwE7AR2xNFCe3h4xAfL2rP8Vk5X1TPUrJoO3Afcn+H8I41tXo3ZvirD8dzlWSTlqT8/zsS6ag6pGc5NP72+YLudm5ez/fXZbP/nnexYPT9Wf/wiKc1HIQYSWjk9tF8IEvKqmjlvGXnIGbz0yXPHyd0YoVy5vl4HDBysLyx4Q99du0VHjBylT7+wYK9IZja7xghmmzEXqrTuqK2Hf1nbjJ6q0rarth42OW2U00Xp08gxY3Xp6q2RCrYskpIFnwJWq+rS5tpLImb/isgUEZknIvPWrlsL7JsyHM8dP3en/VvRaf9WvLn4FaqqhjC6eijdOrfjzDMn8/TcOU3Ho9pteGkmG16ayWM3nc3Rhx/CpkV3snH+L7j6e5fwvS+PajoeDgg4K32KWOJCoQ3aZIJl7prDRFWtBU4ALhKRTzdnqKqzVHWcqo7rVtkN2HdlOJ47fu6k+ROX9Mm2Fi3vBk1EWgKnAr9vzkZD2b9AY/ZvZKRLKdkXZDieO37uUtTpCjd5L5Iikf/FhUJ6aJ8B3lDVFekOikg7EenQ+B04lvQrrDeLfVWG47nj506aP7FJnyx720aUgMBeK6eb/XcAU1Nsm1ZOJ5hgXGjKa+Sxcvq+KMPx3PYukpIk7mJIn0aNqdV/rauPVPCLpAQ33tZFJjx38riT5k+pF0kZNaZWl63bFqnE1aB56ZOHxz6IYkifRtfU6SNPPR/JdkDXtl765OHhYTdcVAqUFftq1rrnLo9SIEn+lFwpEDEgYFVQoJxzaPti1rrnLt8iKUnyJw6lwKiaWn1v/fZIBUeUAiXFvpi17rnLw500f+JQCghQIdFKXLC6QXMl09pzu8+dNH9iWyTFsiGn1UGBdBFYGzOtPbf73KWo0xVu8lYK5LSmQCywukFzJdPac7vPnTR/4lIKWNae2R0UsDnT2nMniztp/sShFBhdU5txMZlwwSsFghtva6a1504ed9L8KbVSYMzYWl29+ZNIJa4GzSsFPDz2QRRDKVBTW6dzn3khkm33Dq28UsDDw8Nu2DaFZnXaBriTae253edOmj9xrClgW9pG2efLMs2h2Zxp7bmTxZ00f+JQCowZW6cfbtkRqeCVAu5kWntu97mT5k9cSgHbemhWN2iuZFp7bve5k+bPvqoUyNqgiUg/EfmLiLwuIq+JyDfM/gNEZK6ILDWfXZo5P7/1/nAn09pzu89dijpd4aZApYBrawrsAC5R1eHAIQSrN40ALgeeVNUq4EmzvQdEpAVwM8GqTyOAyebcSHAl09pzu8+dNH9iUQpE7J1ZHRQA/ggcAywBepl9vYAlaWwPBR4PbU8HpkcNCticae25k8WdNH/iUAqMra3TzfU7IxVsVAoQrKD+LtAR2JhybEMa+9OAX4W2zwZmRm3Q6hvszbT23MnjTpo/pVYKjK2t083bdkYqcTVokZUCItIeeBqYoar3i8hGVe0cOr5BVbuknHM6cJyqnm+2zwbGq+rFaeqfAkwB6Ne/f92bby+PdF0eHh65oxhKgdq6cfrM8y9Fsu3QtiIWpUCkKKeItALuA2ar6v1m92oR6WWO9wLWpDk18thc06yc7uHhYTece8GjBKGR24DXVfWnoUMPAeeY7+cQzK2l4iWgSkQGiUhr4ExznoeHRxIgEUu2agrIhggjSg9tIsHc11EissCUScB1wDEispQgSHCdubDeIjIHQFV3ANOAx4HXgXtU9bVcLrDY0pELzj+X/r27U1czMmfu5s79+cybGF09lNox1Xz38su44Pxz6X5AJzq3348JdTVMqKthv1bCsCGDOHjsaL542ils3Lix6fwFr7xC107t6NalA+NqRrFt27a96gTYuXMnY0YOo2e3LtSOqaZvz26MHjF0rzo//PBDjvvMkVR2bs83vz6NnTt3csi4sZz6uZMAWL9+PScefwwjh1dx4vHHsGHDhpLe81zrHDpkIONqRjGhroaJE5ofpYTr/NGMH3LYoeMZXzuG2jHV/PDqKwH48pfOaPoNhg4ZyIS6Gi44/1z69Khk/9YVTccu/tpUfj7zJnp07UT7ti3p23P3KOG5Z5+lR9fOdGzXhsrO7flg1aqmZ2FAnx5UDxvC6OqhXPn9K6gdU83+rSs4dHwdo6uHMqGuhrGjRtC5Q/As9O5+AF067LfHM7R+/Xr69epGp3Zt9vg9Un/HKPcy/IyG7b5wysl7PU+NEJH+IrJFRC4N7asTkVdNA3Oj6dggIgNE5EkRWQTFSdsoNBtiD8QxUZdrKaX0ae5TT+vzL8zXEdXVGSdo09V36+2/3uvcx+Y+pUcedbRu3LJN6xtUl69cvRfHSy8v0h49euhH9Q1a36D67Usv029fepnWNwTRqJ49e+qxxx2vJ0w6UVd8sE4ffWzuXnXWN6j+6LrrtVOnTjrxsE9pfYPq7LvvbbIJ17lu4xb981+e1Rtn/kIvuPAive76/9YvnjlZT5h0otY3qH7rku/oNTOu1foG1WtmXNt0XqnueS511jeo9h8wQN9btTan32fkyFH63N9f1PoG1c1bP9FxB4/Xvz779z3O+fo3v63fv/JqnfvU03rfgw9rmzZt9vodH31srj7/wnw9aOjQpt+nsrJSvzbt61rfoHr5Fd/Xb11yqc596mmdffe92qZNG924ZZu+/uY72qdvX3150Wv6qU8frjVja/W5v7+k9Q2qbyz9V9OzkO75O+nkz+mo0WN0RHX1Hr9H6u8Y5V421j98RHWT3cNzHtd27drrP156pel5Soly3gfcC1wa2vciQZaCAH8CTjD77wXOSQ3eZStkCAqQZzZEQUGBOCEia4HlQDugN7AUqGT320E+SHNa2BagZwbb1kAVkKm32Fx961POHQysBT7KwNGYdd2Yqt3Z1PcG0JVgbvEdoAfwVjN1tjL1NRAkPb6VwtcZ6AL8K7SvK9DBXMuqUP0jCdJuGky9Q4HFWfzOdM83AOsy2OZSJ8Aogh79jmaOZ6uzgsCnd4GPQ+eMJvB7uzl/KPCyORa+5z0J7t1rQCdzbDF7368+BM/lQlNHFcEccV9zDcuBrez9vIW3K4AaggjjAPMZ/j0w19LO+BPlXrYGDjJ+LgWGm+vYHrIboKrdROTzBKOwj4EtqnqDmRP/i6oOAxCRycARqnqBiLxGEOhbISKPEYzOXiE72gLbQtuzVHWWqf804PiU4OEEVZ22dzVZUO7eWKZCKO0DmEeGtA9ySBEhSD9ZHJU7XF/qucAC4GrgBYIo8MGpHASh8JGhcx4G3gmd/yjB/4ibgMvS1Qn8AbgBeIIgAPMycFlKnV9O8eGrBA1YHXAE8IjZvzHFbkOe9/E04FeY/32bs82lTnP8X8a/+cCUHH6fm8292wL8OMX+04R6CcBhwC6CP8anCf7wG+/5FuBtY/dN4BOCaZOXze+zwRz7NfBeqM7bzHX91Vz7q+Z6/pugwWjkOj30bPzMHAs/LxtSrv2rjfcryr1kd3pV49/OVuABgsYs/Iy2A/4OtAeuwvTQgHHAn0P1fSr07PwO+Ib5firBf65dC/w7Pz2NTzflU5fVWk5yk2TkLd8osL6WBD2jQ4DvAPc0zjcAiMgEYKuqLjbbVxD0PNaLyEkED9sIYAbwN+AUgl5BuM6HCRqxDwh6L68Q/EGeIiJHh+qcnXJtY4B6VZ1fAr9zsc31t5moqrUEcyoXicinI9a5S1VrCHpI40UkPNk5GbgrtL2GIBl8LPBtgkagB8E9XwX0M79jS4K3UZyFuefsHilk8ut7qjqKoDEYDXwnxPW/QIWI1ABDCHp+UZHPcy7A/gTDxfAzejXwM1XdkgPHpcDhIvIKcDjBqCNTTzoK8l/TIAW2v+AxF0eLdlNyrG8FcL8G/7W8KCK7CIYhjTgT84ckIucAJwFHA88QdPWHAm2AXxIkLL9JMDQI19kG+DzBcKc9wR/JLGAO8B8EQ5WjjX0YVcAgEVlG0OXvKCK/xaTcqOqqNCk3pbjnOf02qvq++VwjIg8QvA3imah1qupGEfkrcDywWERaEvQm6kL2nwA7jf18EfkIeEVVVUTqCf6AKw3PFqCVqm4VkWeAA00dqwh+k3R+rTV1fyQiswl6PY1c75q6DzXX1A54HqgUkedJnwKV1e8UNITsPiEY/r6vquFndAJwmohcTzBlsUtEthHMqfVNx2F+m1OhKTf1C6q6KcP1RkFTNgRBA3km8KW8aiqkq1jqwp6SjAvJIMkgB/kG0YacaetLPReYClxjvh9E8NYCabQjeAAHE/xx/RPoZmynmM8uBEOZ4wiGnn8mGNqmq7MLQYM3x1zfKwRDi27N+PBVdg9VjmD3sOEnwOXm++XA9Xnex0bb6Zlsc6yzHdAh9P15gvmVbHUuBg4xx/YDngVOMtvHA0+nnF/L7iHeYIJ5wOvN9g0EDULjPV8NfN9wLgX+YOyOAeoJ/kMaZK6nBcGw7mhj04ogVenCENcHBGlQjdfyE+DHxoc9fo80v2PWe8nuZ6/R7neGszr8PKWccxV7BgVeIuitNgYFJpn9lUCF+T4D85wW4W89Z6VC2nqK2QCVouTiaBRbgt7SKvPArgDOi1pfunPNQ/Vb8wC9DBwVsttB0Ns6j2Au6z2COZUFwC0hni8TzBttBq5PV2fIdgbBxPVigj/C5upcRhDA2GKu9Rx2N2hdCV4osNR8HlDKe56j3WCCP9KFBJPmUeu8kaCBX2TuzQ9CdncAU1OegQ0EvbDG3ssp5p5vNL/vjtBvPNXcx+3G9oDQb7zT2H4AXGPO2W7q+Mj48Ij5XGh+k/Upz1BXgumG7am/R5rfcUSme8mez+g6c11vm+djr+cpdN5V7NmgjTP2bxP8B9sYQDzNPDdvEsyftil3GxEuVkY5PTw8PPKB7UEBDw8Pj8iwvkErliTCFojIMpOBvUBEnFurT0RuF5E1IrI4tC/Syz5tRDP+XCUiK1OUMdZDCnwZaxJgdYNWVEmEXThSVWs0hrcPlAB3EEyyh5H1ZZ8W4w729geCdIYaU+bEfE35Iu+XsSYFVjdo5Ll4g0fpoKrPEExSh/E5giRTzOfn47ymQtCMP05CVVep6svm+0cEaos+OPz75ArbG7S8F2+wGAo8ISLzzTvgkoAeqroKgj8qoHuZr6cYmCYii8yQ1LkhmogMBMYSKB+S+Pukhe0NWrGz/21AlCx4j/LiFwTJszUEKRD/XdaryREm4fU+4Juqurnc1xMnbG/Qip39X3ZoKAueQF83vrxXVBREedmnM1DV1aq6U1V3Abfi0G9UwMtYEwHbG7REvSBSRNqJSIfG78Cx7PlWBVcR5WWfzqDxj9/gFBz5jYw+M9+XsSYC1ifWmpD5/xBISm5X1RnlvaL8ISKDCXplEEhYfueaPyJyF4GMqpJAEnQl8CBwD9CfQIp1uqo6MdHejD9HEAw3lSBT/4LGOSibISKHEUi+XiV4kwjAdwnm0Zz8fXKF9Q2ah4eHR1TYPuT08PDwiAzfoHl4eCQGvkHz8PBIDHyD5uHhkRj4Bs3DwyMx8A2ah4dHYuAbNA8Pj8Tg/wHydAVVeo/LvAAAAABJRU5ErkJggg==\n",
|
|
"text/plain": [
|
|
"<Figure size 432x288 with 2 Axes>"
|
|
]
|
|
},
|
|
"metadata": {
|
|
"needs_background": "light"
|
|
},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"import itertools\n",
|
|
"\n",
|
|
"plt.title(\"confusion matrix\")\n",
|
|
"plt.imshow(confusion[:21,:21],cmap='Blues')\n",
|
|
"\n",
|
|
"plt.colorbar()\n",
|
|
"for i,j in itertools.product(range(confusion.shape[0]),range(confusion.shape[1])):\n",
|
|
" plt.text(j,i,\"{:}\".format(confusion[i,j]),horizontalalignment=\"center\",color=\"black\" if i == j else \"black\")\n",
|
|
"plt.show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "543ede4b",
|
|
"metadata": {},
|
|
"source": [
|
|
"혼동행렬을 보면 별로인것 같습니다.\n",
|
|
"왼쪽이 predict이고 밑이 actual입니다."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "9e85493d",
|
|
"metadata": {},
|
|
"source": [
|
|
"$$Precision(class = a) = \\frac{TP(class = a)}{TP(class = a)+FP(class = a)}$$\n",
|
|
"\n",
|
|
"$$Recall(class = a) = \\frac{TP(class = a)}{TP(class = a)+FN(class = a)}$$\n",
|
|
"\n",
|
|
"$$F1Score(class = a) = \\frac{2}{\\frac{1}{Precision(class = a)}+\\frac{1}{Recall(class = a)} }$$"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "200c95b9",
|
|
"metadata": {},
|
|
"source": [
|
|
"F1Score는 다음과 같이 주어집니다."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "db75fb9f",
|
|
"metadata": {},
|
|
"source": [
|
|
"O 클래스에 대해서 계산해보면"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 49,
|
|
"id": "86f318c4",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor(16409)"
|
|
]
|
|
},
|
|
"execution_count": 49,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"TP = confusion[21,21]\n",
|
|
"TP"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 50,
|
|
"id": "60f8af59",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor(4027)"
|
|
]
|
|
},
|
|
"execution_count": 50,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"FP = confusion[21].sum() - TP\n",
|
|
"FP"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 51,
|
|
"id": "0b5d4cd9",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor(0)"
|
|
]
|
|
},
|
|
"execution_count": 51,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"FN = confusion[:,21].sum() - TP\n",
|
|
"FN"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 52,
|
|
"id": "5d88f758",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"precision : 0.8029457926750183\n",
|
|
"recall : 1.0\n",
|
|
"F1Score : 0.8907042741775513\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"precision = TP / (TP + FP)\n",
|
|
"recall = TP / (TP + FN)\n",
|
|
"\n",
|
|
"f1Score = (2*precision*recall)/(precision + recall)\n",
|
|
"print(f\"precision : {precision}\")\n",
|
|
"print(f\"recall : {recall}\")\n",
|
|
"print(f\"F1Score : {f1Score}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "0b23e7d5",
|
|
"metadata": {},
|
|
"source": [
|
|
"다른 클래스에 대해서도 모두 해보자"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 53,
|
|
"id": "38b3eee6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def getF1Score(confusion,c):\n",
|
|
" TP = confusion[c,c]\n",
|
|
" FP = confusion[c].sum() - TP\n",
|
|
" FN = confusion[:,c].sum() - TP\n",
|
|
" precision = TP / (TP + FP)\n",
|
|
" recall = TP / (TP + FN)\n",
|
|
"\n",
|
|
" f1Score = (2*precision*recall)/(precision + recall)\n",
|
|
" return f1Score"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 54,
|
|
"id": "61fe2d6c",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"class 0 f1 score : nan\n",
|
|
"class 1 f1 score : nan\n",
|
|
"class 2 f1 score : nan\n",
|
|
"class 3 f1 score : nan\n",
|
|
"class 4 f1 score : nan\n",
|
|
"class 5 f1 score : nan\n",
|
|
"class 6 f1 score : nan\n",
|
|
"class 7 f1 score : nan\n",
|
|
"class 8 f1 score : nan\n",
|
|
"class 9 f1 score : nan\n",
|
|
"class 10 f1 score : nan\n",
|
|
"class 11 f1 score : nan\n",
|
|
"class 12 f1 score : nan\n",
|
|
"class 13 f1 score : nan\n",
|
|
"class 14 f1 score : nan\n",
|
|
"class 15 f1 score : nan\n",
|
|
"class 16 f1 score : 0.001982160611078143\n",
|
|
"class 17 f1 score : 0.0445544570684433\n",
|
|
"class 18 f1 score : nan\n",
|
|
"class 19 f1 score : nan\n",
|
|
"class 20 f1 score : nan\n",
|
|
"class 21 f1 score : 0.8907042741775513\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for i in range(22):\n",
|
|
" f1 = getF1Score(confusion,i)\n",
|
|
" print(f\"class {i} f1 score : {f1}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0b9b55e7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.7.11"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|