monoid
/
nsmc-study
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
nsmc-study/Training.ipynb

2305 lines
137 KiB
Plaintext
Raw Permalink Blame History

{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "4c31f5ad",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'C:\\\\Users\\\\Monoid\\\\anaconda3\\\\envs\\\\nn\\\\python.exe'"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import sys\n",
"sys.executable"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "2b9e11e7",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"load bert tokenizer...\n"
]
}
],
"source": [
"from transformers import BertTokenizer\n",
"print(\"load bert tokenizer...\")\n",
"PRETAINED_MODEL_NAME = 'bert-base-multilingual-cased'\n",
"tokenizer = BertTokenizer.from_pretrained(PRETAINED_MODEL_NAME)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "82bf44a2",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"cuda available : True\n",
"available device count : 1\n",
"device name: NVIDIA GeForce RTX 3070\n"
]
}
],
"source": [
"import torch\n",
"print(\"cuda available :\",torch.cuda.is_available())\n",
"print(\"available device count :\",torch.cuda.device_count())\n",
"if torch.cuda.is_available():\n",
" device_index = torch.cuda.current_device()\n",
" print(\"device name:\",torch.cuda.get_device_name(device_index))"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "38dcf62d",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"read train set\n",
"100%|██████████████████████████████████████████████████████████████████████| 150000/150000 [00:00<00:00, 208913.67it/s]\n",
"read test set\n",
"100%|████████████████████████████████████████████████████████████████████████| 50000/50000 [00:00<00:00, 259067.57it/s]\n"
]
}
],
"source": [
"from ndataset import readNsmcDataAll, make_collate_fn\n",
"dataTrain, dataDev, dataTest = readNsmcDataAll()\n",
"collate_fn = make_collate_fn(tokenizer)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "650c8a19",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertModel: ['cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias']\n",
"- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
"- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
]
}
],
"source": [
"from transformers import BertModel\n",
"PRETAINED_MODEL_NAME = 'bert-base-multilingual-cased'\n",
"bert = BertModel.from_pretrained(PRETAINED_MODEL_NAME)"
]
},
{
"cell_type": "markdown",
"id": "30d69b45",
"metadata": {},
"source": [
"BERT 로딩"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "7583b0d1",
"metadata": {},
"outputs": [],
"source": [
"import torch.nn as nn\n",
"\n",
"class MyModel(nn.Module):\n",
" def __init__(self,bert):\n",
" super().__init__()\n",
" self.bert = bert\n",
" self.dropout = nn.Dropout(p=0.1)\n",
" self.lin1 = nn.Linear(768,768) #[batch_size,768] -> [batch_size,768]\n",
" self.gelu = nn.GELU()\n",
" self.lin2 = nn.Linear(768,1) #[batch_size,768] -> [batch_size,1]\n",
"\n",
" def forward(self,**kargs):\n",
" emb = self.bert(**kargs)\n",
" e1 = self.dropout(emb['pooler_output'])\n",
" e2 = self.gelu(self.lin1(e1))\n",
" w = self.lin2(e2)\n",
" return w.squeeze() #[batch_size]"
]
},
{
"cell_type": "markdown",
"id": "befe62b0",
"metadata": {},
"source": [
"모델 선언. 비슷하게 감."
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "36585e76",
"metadata": {},
"outputs": [],
"source": [
"model = MyModel(bert)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "0e36d9e2",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\"\\nclass MyModel(nn.Module):\\n def __init__(self,embeddings):\\n super().__init__()\\n self.embeddings = embeddings\\n self.lin0 = nn.Linear(768,1)\\n \\n def forward(self,**kargs):\\n emb = self.embeddings(kargs['input_ids']) #[batch_size,word,768]\\n e0 = emb * kargs['attention_mask'].unsqueeze(dim=-1)#[batch_size,word,768]\\n e1 = self.lin0(e0)#[batch_size,word,1]\\n w = (e1.sum(axis=1))#[batch_size,1]\\n return w.squeeze() #[batch_size]\\nmodel = MyModel(bert.embeddings)\\nfor param in model.embeddings.parameters():\\n param.requires_grad = False\\n\""
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\"\"\"\n",
"class MyModel(nn.Module):\n",
" def __init__(self,embeddings):\n",
" super().__init__()\n",
" self.embeddings = embeddings\n",
" self.lin0 = nn.Linear(768,1)\n",
" \n",
" def forward(self,**kargs):\n",
" emb = self.embeddings(kargs['input_ids']) #[batch_size,word,768]\n",
" e0 = emb * kargs['attention_mask'].unsqueeze(dim=-1)#[batch_size,word,768]\n",
" e1 = self.lin0(e0)#[batch_size,word,1]\n",
" w = (e1.sum(axis=1))#[batch_size,1]\n",
" return w.squeeze() #[batch_size]\n",
"model = MyModel(bert.embeddings)\n",
"for param in model.embeddings.parameters():\n",
" param.requires_grad = False\n",
"\"\"\""
]
},
{
"cell_type": "markdown",
"id": "7969fead",
"metadata": {},
"source": [
"### 학습 과정에서 벌어지는 일"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "8c2a4bc9",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"MyModel(\n",
" (bert): BertModel(\n",
" (embeddings): BertEmbeddings(\n",
" (word_embeddings): Embedding(119547, 768, padding_idx=0)\n",
" (position_embeddings): Embedding(512, 768)\n",
" (token_type_embeddings): Embedding(2, 768)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (encoder): BertEncoder(\n",
" (layer): ModuleList(\n",
" (0): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (1): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (2): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (3): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (4): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (5): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (6): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (7): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (8): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (9): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (10): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (11): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" )\n",
" )\n",
" (pooler): BertPooler(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (activation): Tanh()\n",
" )\n",
" )\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" (lin1): Linear(in_features=768, out_features=768, bias=True)\n",
" (gelu): GELU()\n",
" (lin2): Linear(in_features=768, out_features=1, bias=True)\n",
")"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.cpu()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "def10f6c",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'input_ids': tensor([[ 101, 9405, 62200, 14523, 48549, 119, 102, 0],\n",
" [ 101, 9294, 12424, 69592, 48549, 119, 102, 0],\n",
" [ 101, 9479, 68984, 48549, 119, 102, 0, 0],\n",
" [ 101, 9659, 22458, 119192, 12965, 48549, 119, 102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0],\n",
" [0, 0, 0, 0, 0, 0, 0, 0],\n",
" [0, 0, 0, 0, 0, 0, 0, 0],\n",
" [0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 0],\n",
" [1, 1, 1, 1, 1, 1, 1, 0],\n",
" [1, 1, 1, 1, 1, 1, 0, 0],\n",
" [1, 1, 1, 1, 1, 1, 1, 1]])}"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"inputs = tokenizer([\"사랑해요.\",\"무서워요.\",\"슬퍼요.\",\"재미있어요.\"],\n",
" return_tensors = 'pt', padding='longest')\n",
"inputs"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "e027b926",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([4, 768])"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"hidden = bert(**inputs)['pooler_output']\n",
"hidden.size()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "ae9f8fba",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([-0.0180, -0.0823, -0.0234, -0.0886], grad_fn=<SqueezeBackward0>)"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"w = model.lin2(model.lin1(hidden)).squeeze()\n",
"w"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "5470c3f8",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([0.4955, 0.4794, 0.4942, 0.4779], grad_fn=<SigmoidBackward0>)"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"torch.sigmoid(w)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "b7eb8e67",
"metadata": {},
"outputs": [],
"source": [
"labels = torch.tensor([1,0,0,1])"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "7a324ed7",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor(0.6937, dtype=torch.float64,\n",
" grad_fn=<BinaryCrossEntropyWithLogitsBackward0>)"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"nn.BCEWithLogitsLoss()(w,labels.double())"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "cb54294d",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([False, True, True, False])"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"(w > 0).long() == labels"
]
},
{
"cell_type": "markdown",
"id": "596b89bd",
"metadata": {},
"source": [
"이런 일이 벌어짐. sigmoid 는 나중에"
]
},
{
"cell_type": "markdown",
"id": "9875ce48",
"metadata": {},
"source": [
"### Model Training"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "769c4290",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"MyModel(\n",
" (bert): BertModel(\n",
" (embeddings): BertEmbeddings(\n",
" (word_embeddings): Embedding(119547, 768, padding_idx=0)\n",
" (position_embeddings): Embedding(512, 768)\n",
" (token_type_embeddings): Embedding(2, 768)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (encoder): BertEncoder(\n",
" (layer): ModuleList(\n",
" (0): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (1): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (2): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (3): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (4): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (5): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (6): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (7): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (8): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (9): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (10): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (11): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" )\n",
" )\n",
" (pooler): BertPooler(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (activation): Tanh()\n",
" )\n",
" )\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" (lin1): Linear(in_features=768, out_features=768, bias=True)\n",
" (gelu): GELU()\n",
" (lin2): Linear(in_features=768, out_features=1, bias=True)\n",
")\n"
]
}
],
"source": [
"device = torch.device('cuda')\n",
"model.to(device)\n",
"print(model)"
]
},
{
"cell_type": "code",
"execution_count": 29,
"id": "b9380dcd",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"device(type='cuda', index=0)"
]
},
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"bert.device"
]
},
{
"cell_type": "markdown",
"id": "5e82df0e",
"metadata": {},
"source": [
"모델을 모두 gpu로 보냄"
]
},
{
"cell_type": "code",
"execution_count": 30,
"id": "74c4becc",
"metadata": {},
"outputs": [],
"source": [
"from torch.utils.data import Dataset, DataLoader\n",
"BATCH_SIZE = 16\n",
"train_loader = DataLoader(\n",
" dataTrain,\n",
" batch_size=BATCH_SIZE,\n",
" shuffle=True,\n",
" collate_fn=collate_fn\n",
")\n",
"dev_loader = DataLoader(\n",
" dataDev,\n",
" batch_size=BATCH_SIZE,\n",
" shuffle=True,\n",
" collate_fn=collate_fn\n",
")\n",
"test_loader = DataLoader(\n",
" dataTest,\n",
" batch_size=BATCH_SIZE,\n",
" shuffle=True,\n",
" collate_fn=collate_fn\n",
")"
]
},
{
"cell_type": "markdown",
"id": "4153b2e7",
"metadata": {},
"source": [
"데이터 모델 준비"
]
},
{
"cell_type": "code",
"execution_count": 31,
"id": "3cd5bf7b",
"metadata": {},
"outputs": [],
"source": [
"from torch.optim import AdamW\n",
"from groupby_index import groupby_index\n",
"from tqdm import tqdm"
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "65b5ccde",
"metadata": {},
"outputs": [],
"source": [
"optimizer = AdamW(model.parameters(), lr=1.0e-5)\n",
"BCELoss = nn.BCEWithLogitsLoss()"
]
},
{
"cell_type": "markdown",
"id": "79607e81",
"metadata": {},
"source": [
"학습 준비"
]
},
{
"cell_type": "code",
"execution_count": 33,
"id": "68183cf4",
"metadata": {},
"outputs": [],
"source": [
"def freezeParameter(model):\n",
" for param in model.bert.parameters():\n",
" param.requires_grad = False\n",
" for param in model.bert.embeddings.parameters():\n",
" param.requires_grad = True"
]
},
{
"cell_type": "code",
"execution_count": 34,
"id": "42d1a29f",
"metadata": {},
"outputs": [],
"source": [
"#freezeParameter(model)"
]
},
{
"cell_type": "code",
"execution_count": 35,
"id": "cc8a82a5",
"metadata": {},
"outputs": [],
"source": [
"TRAIN_EPOCH = 5\n",
"\n",
"result = []\n",
"iteration = 0"
]
},
{
"cell_type": "code",
"execution_count": 36,
"id": "4835a0d3",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"epoch 0 start:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Epoch 0: 100%|███████████████████████████████████| 9375/9375 [11:41<00:00, 13.37minibatch/s, accuracy=0.844, loss=5.79]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"epoch 1 start:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Epoch 1: 100%|████████████████████████████████████| 9375/9375 [11:48<00:00, 13.23minibatch/s, accuracy=0.84, loss=5.37]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"epoch 2 start:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Epoch 2: 100%|███████████████████████████████████| 9375/9375 [11:44<00:00, 13.31minibatch/s, accuracy=0.879, loss=4.79]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"epoch 3 start:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Epoch 3: 100%|███████████████████████████████████| 9375/9375 [11:42<00:00, 13.35minibatch/s, accuracy=0.852, loss=5.12]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"epoch 4 start:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Epoch 4: 8%|██▊ | 717/9375 [00:53<10:48, 13.36minibatch/s, accuracy=0.918, loss=3.09]\n"
]
},
{
"ename": "KeyboardInterrupt",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_29640/636965671.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 17\u001b[0m \u001b[0mbatch_labels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmini_l\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 19\u001b[1;33m \u001b[0moutput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mbatch_inputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 20\u001b[0m \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBCELoss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_labels\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdouble\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 21\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_29640/2430988958.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, **kargs)\u001b[0m\n\u001b[0;32m 11\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 12\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m \u001b[0memb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbert\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 14\u001b[0m \u001b[0me1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0memb\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'pooler_output'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 15\u001b[0m \u001b[0me2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgelu\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlin1\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m 1004\u001b[0m \u001b[0moutput_attentions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moutput_attentions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1005\u001b[0m \u001b[0moutput_hidden_states\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moutput_hidden_states\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1006\u001b[1;33m \u001b[0mreturn_dict\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mreturn_dict\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1007\u001b[0m )\n\u001b[0;32m 1008\u001b[0m \u001b[0msequence_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mencoder_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m 590\u001b[0m \u001b[0mencoder_attention_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 591\u001b[0m \u001b[0mpast_key_value\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 592\u001b[1;33m \u001b[0moutput_attentions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 593\u001b[0m )\n\u001b[0;32m 594\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m 475\u001b[0m \u001b[0mhead_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 476\u001b[0m \u001b[0moutput_attentions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moutput_attentions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 477\u001b[1;33m \u001b[0mpast_key_value\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself_attn_past_key_value\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 478\u001b[0m )\n\u001b[0;32m 479\u001b[0m \u001b[0mattention_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself_attention_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m 407\u001b[0m \u001b[0mencoder_attention_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 408\u001b[0m \u001b[0mpast_key_value\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 409\u001b[1;33m \u001b[0moutput_attentions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 410\u001b[0m )\n\u001b[0;32m 411\u001b[0m \u001b[0mattention_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutput\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhidden_states\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m 304\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 305\u001b[0m \u001b[1;31m# Take the dot product between \"query\" and \"key\" to get the raw attention scores.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 306\u001b[1;33m \u001b[0mattention_scores\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquery_layer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey_layer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 307\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 308\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mposition_embedding_type\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"relative_key\"\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mposition_embedding_type\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"relative_key_query\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;31mKeyboardInterrupt\u001b[0m: "
]
}
],
"source": [
"model.zero_grad()\n",
"\n",
"for epoch in range(TRAIN_EPOCH):\n",
" model.train()\n",
" print(f\"epoch {epoch} start:\")\n",
" with tqdm(train_loader, unit=\"minibatch\") as tepoch:\n",
" tepoch.set_description(f\"Epoch {epoch}\")\n",
" \n",
" for batch in groupby_index(tepoch,16):\n",
" corrects = 0\n",
" totals = 0\n",
" losses = 0\n",
" \n",
" optimizer.zero_grad()\n",
" for mini_i,mini_l in batch:\n",
" batch_inputs = {k: v.to(device) for k, v in list(mini_i.items())}\n",
" batch_labels = mini_l.to(device)\n",
" \n",
" output = model(**batch_inputs)\n",
" loss = BCELoss(output, batch_labels.double())\n",
" \n",
" prediction = (output > 0).to(device,dtype=torch.int64)\n",
" corrects += (prediction == batch_labels).sum().item()\n",
" totals += prediction.size()[0]\n",
" losses += loss.item()\n",
" loss.backward()\n",
"\n",
" optimizer.step()\n",
" accuracy = corrects / totals\n",
" result.append({\"iter\":iteration,\"loss\":losses,\"accuracy\":accuracy})\n",
" tepoch.set_postfix(loss=losses, accuracy= accuracy)\n",
" iteration += 1"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "81b69931",
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt"
]
},
{
"cell_type": "code",
"execution_count": 37,
"id": "c3a73c68",
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD4CAYAAAAHHSreAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAABPbUlEQVR4nO2dd5gUxdPHv3UBOA7JOQfJICgZRJIkEURFRFSCoIKKGQVUggnDKxhAkR8iCKiABBGRHEWiBEmSM0eQDEe6u37/6J3d2cmzOxtmrz/Pc8/O9vT09NzudnVXVVcRYwwCgUAgEMiJi3QHBAKBQBB9COEgEAgEAhVCOAgEAoFAhRAOAoFAIFAhhINAIBAIVCSE82ZxcXEsKSkpnLcUCAQC15OamsoYY2GdzIdVOCQlJeHq1avhvKVAIBC4HiK6Fu57CrWSQCAQCFQI4SAQCAQCFUI4CAQCgUCFEA4CgUAgUCGEg0AgEAhUCOEgEAgEAhVCOAgEAoFAhSuEw+mrpzF9x/RId0OQWTh+HJg7N9K9EAgiiiuEw8PTHkbnXzqj64yuuHTjUqS7I4h1GjQA2rePdC8EgohC4Uz2k5yczALZIU3DyO89GyISFAlCCHm+byIRliBKIKJUxlhyOO/pipXDO/e8E+kuCDIjQjgIMjGuEA7vNnvX771IbSoIC+J7JsjEuEI4AMDsR2d7j99d8a5+RYHAKYRwEGRiXGFzkJDbHoTdQRAyJJvDzZtAYmJk+yIQQNgcTMmWkC3SXRBkJsTKQZCJcZVwmNZpWqS7IMhMZGREugeCWOD6dWC6+/ZpuUo4tK/o8z0/eP5gBHsiyBSIlYPACd54A+jcGVi5MtI9sYWrhIOcwxcPR7oLglhHrByco2dP4P/+z/51aWlA3brAokX6ddatA+64A0hNNW9vyhSgbVv7/QiGo0f56/nz4b1vkLhWODSb2CzSXRDEOpl95XD1KnDypLr8xAmuKrHDhAlA//7AtWtASopx3YwM4NAhfpySAmzYADz1lH79V18Ftm0DNm/WPn/jhm+AfuIJYP584NgxXm7EgQPG583qp6fz55AcHFw22XCtcBAIQo7LfsyO06ABUKSIurxYMeChhwJrs3VroGhR4zoffACUKQPs3esrMxLUZkK8Rw+gZEnufSZRogTQrZv+NYsWAeXKAT/+aNy2xB9/8PrTZHbRwYP5cxw5wt+77PskhINAoEcgK4dRo4Dt253vixzGgPff57PfULJtm+941ixg4ULf+z/+UNefNw+YPdu4zVWrzO87eDB/PXAAeOstfnz8uP/grsVbb2mrlqQ+paVpl0vMnMlXFQDwzz/89e+/zfurrH/kCFeFffghLztzhr8K4RBaNj2zKdJdEMQy1675jgP5MffrB9So4Vx/tNixA3jnHW7kDBcPPcRn/Ua0awc8+KBz95w4EZg0yff+hx+060lCfMUK4OOP1eelz9FM2D/8sM8eIV0TF8AQ2b49V4VJCLVSeCiXt1ykuyCIZSpV8h0HanNwehBo04arQSSkGbA0Sx40yDcAKTl1ip/7+Wft8x9/zM/fumWtL/L/iZVVgB7//stfP/lE//4//eT/Xlnn22/5tWvX+spSU/mKg8i3CkhP5685cvhfr/f5EnEPI+n46FH+OmeOuu6AAer//blz/u8PHza+X5TiOuGQGOfbsSpiLAkcR9IPA8EN8leu8NcNG3zG0EBZsEBbhSR9/4cP1792507++u232uffe4+/WjUwy39zwfjuf/65//0PHeKD/IkT+tcoZ/Gffaauc/KkT1hIKw29z/HWLe69tGOH/j1PnQJGj+bH48apzytXKidO6Kv75BMPF+A64ZAlPov3eP3x9RHsiSDmCWby0aMHf61bFyhd2one+NBbJWhh9gzSeattOrUqkoSV1F6DBvzPaABV9lGrL5Mn+4SIFXXSE08A1arpn//hB58AkFYgRkyerH+ufHnz66MI1wmH+Lh473H97+pHsCcCx/nyS99yPhqQDz7Vq/PBycgoKh+EZswALl70b2fECJ+BFeBqDyI+i27cGLh8Wd3mhQu+48aNgV279O//wgvqsvff56/LlwObNOx1kjunVcOrPNaU3qArDaJDhwKffmrcntTG2bP8Vet/IHHtGrB4Mf8/EAH792vX+/pr/jp1KvDSS8b3l7j7bvM6mzcDTZrw1Y5kNJewMpGIjzevE00wxsL2lz17duYE3Wd1ZxgKhqFwpD2BA6SnM5aSElwb/Cdm/7rjxxnLyAju3so+AIwdPcrYqVP+5UuXMpaaqr7u6lXGzp71v37iRP9nUj6fvC7A2PTp6nbff9+/TocOjG3ezI/vuEPdjpyLF/3P1avH2H//MXb9uroPFSvq/y+0+gow9sILjJ04wf9P8jrXrzN24YLv/blz2tczxljWrNrntP5mzGAsKcl6/VD/yZ954EDz+rdu6X7tzABwlYVpnJb+XLdyAID7K9wf6S4IlLz/PveJl+vsw8HWrdzvfswY59t++mmgUCH/1ULz5kDt2uq6NWsC+fL5lzGT2aRSTaKcWZ4+Dbz9tv41ZqqgXLnU7efPzw3cTpCayvcslCjBPYskMjKA3Ll97/Pm1b5+2TJrqhqJW7eiK0qu5L4KGNt9JEK4ciCiNkS0m4j2EdEAjfN5iGgWEf1DROuJyECXxnGlcMiRJYd5pcxOejrfTGS0THeSefP46/HjobsHY8BHH/l7g+zezV+XLQuszf37gW++0T4nebssX+5fLhl55cg3bEmYCQflebnBddEibddNIm3DqBWk9pXPY4ae4JX+94Dv8wes7/No3ly998CIdeuAS1GUQ/4dmxkq7diKbDVL8QBGA2gLoAqAx4ioiqLaIABbGGN3AOgG4Auzdl0pHO4uaUE/mNmZMYPPOt98M7z3DdEPAACwdCkwcCDQp4/6fmYDsR6NGwPPPWfsrWPm36+HvE/K/mkNinLh0KoVDzehVUfynrFLoDPXvn21y1ev9h3LP/e6dQO7jxkjR4am3UDRcm2NDHUB7GOMHWCM3QTwM4AHFHWqAFgCAIyxfwGUJqJCRo26UjjIVw4s0EHBrfToAZQqZV5PMjTqzbS6duU/6Oef56/Sbs5ACcfnIKl35M+k9Eyxi7QKsdv/nj3N6xgJBy11SocOfO+AkYA1E7579/I6WvXkwoHIf5dzsJ/f1KnBXS8wI4GINsr+npGdKwZA7i99zFMmZyuAhwCAiOoCKAWguNENXSkc5Hy+9vNIdyG8TJxoTa9vNqOWNhhJnh1yL5pAkO6zYQMfcA8dMvasAYA1a3wePXIOHvQd79jh2yegNeAZCYcjR3wqoOXLtVcHUpsLFtgTMBMm8Gf88kt+rRby/71c3bZ4sb7XU4cOxvc1C1onD3GhRLlPQK4u2rMHePJJvneBMf/QGYJoII0xVlv2N1Z2TmvGoPzhfwQgDxFtAdAPwGYAhjo9U+FAROOJ6DQRbZeV5SWiRUS01/Oax6ydUDFl25RI3Tq6kQaCcK2spPu8+CLXJZcpA1RRqj1l3LgBNGyoPRiWLes7rlaNB03Tw0gIlioFVK3KhVSzZrxvetc/+KBvY5ZVypThrpJ6Bl55n+T/i5YtuZunFnLXVS3++st3bFeFZ6ZWmjyZh+SYOZOHwBa4hWMAZFvoURyA325CxtglxlhPxlhNcJtDAQAHYYCVlcMEAMpv/wAASxhj5cH1WCrreKipX5zvcaBQ6rjdSloa8Pjj/Pinn7Rn53bYuJGroazOrLduNa8jhUKQ/OulnbJGLFnCXxcs4MJl7lxfLJ9ff/XVe+89f+8ZKY6+1mxY/v05aPhbsY98li/tmJaw8j9yGuXKQS/8xb59oe+LwEk2AChPRGWIKAuALgD8DCJElNtzDgB6A1jJGDO07psKB8bYSgCKYCF4AID065sIoKNp9x1GCqOx8cRGrDi0Ity3jzznz/s2Dsm5dk098Fg1YEpqlwsXuGpIMpp26MCFTEoKH5SVAx1gHBv/2jX/gHaAT9Bcvcpn2MpNRVrIk8WsXcsDnGkxeLBvhzLgv4q6dcvnwZWW5h/F87//zPtgByOBZxSyIRi0PKkklCsHveQzixc71x9ByGGMpQF4AcACALsATGOM7SCiPkQkeW9UBrCDiP4F92p6yazdQG0OhRhjKZ6OpQAoqFeRiJ6RjChpdtzWTEiM9/k7T9/pvvysQZM3L/dZV5I/v9oP36o9oXNnPojnycN99iUvFelzS0jgbd92m/paIx11cjJvU45c5RJqLxS5XaJ9eyBnTv5eLkAA/eB0gWL0fddKouMEkg1JC6sRRoVwcB2MsXmMsQqMsXKMsQ88ZWMYY2M8x2sYY+UZY5UYYw8xxnRmBj4SwtDpsQDGAkBycrJjCnB5AL44cr1d3TmMUiVeugR89RWQLZv2+d9+898XMXky8L//+VRACQmB5SpgTL2ykHvrvPaaeRvrFXG0XnlFu56R6ujoUf9QylNCbK8Ktb3n33/teQlFj+ulwAUEKhxOEVERxlgKERUBcNrJTlkhIc7XdSEcLMAYT6f43XfG9eRhkaVB1Wo4Zy30vHLs7IwFgHr1/N/rpYTUMqRKM+ZQzdYjxfXrQJcuke6FIEYJdFSdA6C757g7gF8N6oYEuVopnqIwoNXq1XxwDXW2LjmjRumfy5bNXDAA/oLg2jVu9JXK5DNhIm2bhxK5EfvFF/mARgQU1NVEBo5eWGrhtBBdOB2l1g0ULhzpHtjGiivrTwDWAKhIRMeIqBe4z2xLItoLoKXnfViJerWS5EMeaFgHqygHXz3MUixKKFcJH37oK1Oqdtau5Z5MVvv31VfWVTn799u3Rch3TsvREg7BJKoRBEdmVG+tWxfpHtjGVK3EGHtM51QLh/tiC/nKISqFg0So9c63bgFZs/LjhITgVECA2oialuZ7hnbt/M/16cNXRkarI6X6qHdva/24/XZr9aygZYi95x7n2hfYQxmgMDNgtFcnSoniUdUY+cpBnuMhaghGlfHXXzz1oxVSU7nXTUqKM1EflYO53ICrRBIKvxpoFe3aFgTuQr4pzyp2vxOhDOYoEWz4mBgk5N5KoUIuHFJvGXjoOEVqKpCUFNigf/UqkD27/7XXr/Pww1oDeqNG/NXKF3baNL7h68YNvnIIFqM0jXo8/7z+uWhIqh4NfYhV9DzfjChuGNJHTTjCdCtdrQXuXTkkJSZ5j79YZxp9NjguXuS++nohD7SQBMHevTyx+fjx/ueTkoAHlIETA0BS+TDmjHAIVi2lJBpWDpktOGM4CWS1qjfBekxHg211f4Zyo6UdKlcO/NoYxbXCITkxOXw3k3bOTppk/RrpByDtC9Aywv3+u71+bNmiLpMGvitXzOPyWMFqonmrRINwkIIMCuxTpozx+aQk4/NWefZZ9QRKwupq3YqgSkkBvtCYTDZpwgNBCry4Vjg0LNEw0l2whqTScMKd8s471WWScLAraPQIZvalRTSodORhNwT26NzZ+HyyQ5O0d9/VVlF99pn1lYNSOFSs6Dvu2ZPvji9cmMcJ06K+yEkvx7XCoc3tDqU6NGL/fv7FNAs9bYS0Yvj1V18wPDmHDnEdrBSD/3//850j4qEM4uP9UxLKMdL3B8KTTzrb3jPPmNcRRC9SqBE9kpICT4YkwZj+vpdXX9WfWJ054zv+5ht1PXmK1fHjfb/F/PnFhMECrhUOIY3GevMmD143dSr/4k6ezMsZ4wHTVq70r3/0KN99e/w4P960SfsL/eOP6rJZs/y9MZSD6WOP8dm31lLYDTi1ohFEBrPQJjlzGqtbp03T3lOijGsVLL17q39zGRnAihXq3yvA9wSNGGGt7a5d9e0hMYxrhYNyb8Oaow7qC196iSeMP3TIczPPva5d4/kFmjThSVEkSpYEihThK4CSJYFatayHZLZqANbTxwoEoUTaQ6NHfLx2AEiJ5s2BuzXS+jZpYr0Peg4Fksoyf35tZ4yMDL6fpXFj9bnERF98rurV9e9dtSrfuPnEE9b7GyO4VjiQIvnRt39rhE749Vd7ydhXrQI++cTnuy2lkJSEg3yXsVFoZAA4rRNuasYMoGhR33uzHM9Oh5EWWGfQIO3Q2nLdtpYdKLMgBWk0WsXnzu3/XgoMaceDTG63GjbMdyy1oXd/K/auq1fNd/kDwH33Wd97FCO4VzgovhAZTOOL0LEj8PTT1hpMT+ezjDffVH+pJOEg97wxm/HruZV26mSe6lEQHSQlAVmyqMvlA1u5cs7es2pVZ9uzQ6dO9urnyGF8/vXXfYL0u+94SlXJu8nO/og8ebi6df16/5VMgQJcnTRvnq9Mvov/0UfN286eXfszlpCPM1qh6tu25a9SwMeWLYGlS33nhw7lE0IX4tpNcEo0hYNVTp8GChWSNaYjHOSJ7c2Ew+HDgfdHEB2ULu0bHMqWBQ4c4Md16/K4UgBXSfzyi7X2WrfWzzctsX175AIFTp/u7L0//dR3/NRT/ucqVNC/LmdO/98akS+oonzgjYvzd+AAuKagSBHuZRWIJ1WfPv65teVorUTy5zdeBQ0ZYr8PUYJrVw5KpmybAmZnqXr+vC8WvmRbkFC6nyrPA9YD2QncyZIl2t5lADB7Ng+ktmQJ94iRBIURc+ZYFyLbtjkbinvsWB4GZcIE/l6u1owUtWppl//zD7Bnj/n1enm7CxfmauFAbXSff84N2FLQR7mwlGsOBg4MrH0XETPCAQD2n99vvXLXrvwHuF/jGulLIH0xVmikIXUwq50gyqhdmxtSifgsFPBPY5o7N189NG/OZ6/KXBNatG9vroaRqFbNp6YINLS5XFVSqxZ/pgYNAmvLCs89Z75hTkm5cupIutWr+1bxH3+svqZDB/76wQf67TZoEPj+i6xZuQG7eXP1OSlfef36fNMeoK+2fu89c2N+lBNTwsHWymH+fP5686Z6WSitHIzaGzWK60LF7tvYQ/6Dz56dfw+6d/eVWQ0ZUbOmtXp6ubABro4pW9ZaO3Ik92vAZ/+SBqu8ee23Z8bo0T61m1X27eP7E7RgDHjjDXV55cr83F132e+jHaR4TvKYS9Wq8XuvWQOUKsWPtTyhAL6idDraQJhxtXCY0dnf0GMpdPd///lnBIuLUwsBaTVhFvrhwgX93ZbRhtzLI1rQ+vFHA2aTDCPhIFdDzJ1r7X4ZGcDw4Xz2bUZSEvfPb9lSv84ff/i/l1w1S5XikxqjKLpaKJ932TJ928nSpdxQPHasvXtEG5UqcRXTtGmR7knEcLVwaFDcf5lsaWNcgQI+VQG/SL9uNMQFcgqzMAiRwK2ugVrfGUkHLo/bX6yYseFVIj0dGDCAz76VMOZ/v65duX++kSdOmzZ8cAO4cVV+/fPPAyVKmPdJzq5d/oNk06ZAq1badZs14w4eVr0EoxUivt9J7qiSyXC1cFAKg4CS/nz7LbB8ufa5WMpY5UTEVqdxyjNGLwNcOJH2q5Qq5V++ZQuP6muE1iRE63+zdq3Pk0bu/aMVQqV6deDUKe3wJfIQ2EZJj6S9OuXLA488ol9PEJNE4YgRZqxuoXc7RsKhZUuuj7aqBnEKp4SDU8HfgqF+fT5j//RTPsuWchYkJWlHLv3gA+Ctt/ix1RVq3ry+z9HM/x4wNmZ/+CG3p7Rt6x+gTo5yA9uCBT6PJ0HM42rhoNwlnZ7Bf2TX065j0JJByCTDvjWMhEPVqtx1L5DBukYN66FC5GjFwgmkzUceUc/WI0G2bD5df7Vq5vUHDfIJB7OdvNWrczuYnhAMJF+FFVdM5efTqpW+OkkQc7harVQoh78+cMnBJQCA8ZvHY+Ram8npYx0j4WDV+0Zuq5EIJpGOnnCwasxcvJj7szsdmTbcmK0cJk3ihl69/QmhSmYUqc14gqjA1cIBAMrm8bn5PTuX+x6nZYg9CCqMhIPVePlNm6rLAh2YlIZWgKs5AOMEMnIVSosWfO+A1f4rkdKxhptXX/V/byTcGOPP2KyZfp2HH+avderoq5gCQQiHTI2r1UoAkBDn/wgfrvoQObLobDa6cSMMPYpSjISD1UFAa1OP2cauli2BhQut3UMa5JV1GzUCVq/mx5cuGbelJXT06skJ10CoJUytxAAyom7d0KwehHDI1Lh+5aAUDsMWvoUcpy/4V5KSgmTmeEdWVg7yuDVKypVTe6x07eofulwLaYBZt46rR6RZstYgLqm3iPzzQHz5Jd+d+vXXxvcyY9Gi4K6PBJEaoEeNEsIhk+N64aA0Sk+eCTzVYQji5WpcyWvDqXy3bsTKyqFZM6BKFe06w4ap1TdTpnBffiOka+rW5THx5SGu9YQDwEMkS23nywfMnAn07es7rzRCS2EV9ChfHrj3Xn6cK5dx3UhGRrWLk2okOW634wiCxvXC4Wa6fwC8B/7lr3Faq+zPPw95f0JK+fKBX6slHHr35q9WdPZE/qoLpd4c4CszpdFUb/aptXJQqpWk+FVyv3yArwSVeRamTTMOhb55M389e5Zn61Ny9ixXWR0+rJ2cJlIYqYvOnvXPIigQOIjrbQ5K4SChOSS5fU9DtWrA3r3a56pW1U5MI6HlkSTtlLVqD5APVFrpI0uWVHveGAke5X2V7/WEg1bmsaxZeUROOV99BfTrx1cKkhuoXlwhqTxUM3Etvv6a5ygIFKdiJL3/vi+xlUDgwfUrhy/aaOdW1lw5uB2jQdwo1aHetdJgb3XlIG2Keu45fbdKSThI7qhGfdYTDtKr1Jbd3d2S+lAv5Ha00Lcv8P332ufCqe9/6y3gs8/Cdz+BK3C9cHig0gN+75nnN6USDsuWhadDocRowBg3juv1jVi6lKtehg/nBmJl3go5XyiELhHQsCEwa5Z6INm7l+cLAHxtSgLHzspBWS6tHKzuw5DYs4enfBUGVUEmgYjaENFuItpHRAM0zuciot+IaCsR7SCinmZtul44AEDR29SzWJVw0IrPHg1o7R3QwyhqZ3IycP/9vvd586rbbtaMq14GDOCCRDmQA1wNAwDduvlfW6cOf+3YUZ3i8fbbeb4AQJ0LQykcjAZsaZ+DVGfAAP9yqxQvHhq7wf3320tvGQzSTmSjcN4CAQAiigcwGkBbAFUAPEZESs+S5wHsZIzVANAUwGdEZJAfNUaEw75++7zHkkxwjVrpxx+t123enG/80kNuEzh71ny1JOny5fsX+vTh7cjj6jBmPZGLcoBXCgPpnomJ/ucYU9sW3nqLl0dL0MDffgOuXQvPve68kz97NBnHBdFKXQD7GGMHGGM3AfwM4AFFHQbgNuLRSnMAOAfAcLdwlPzqgiMpUe2iWuVMBDoycyaPhCl3uTSDSO0JZMQPP/hcPEeP5slI8uWz31eAG5UvXwZefFH7/J9/8nDNdli6lKfRlAy7ypVDly48DeagQeZqJacIVXgJgSB8JBDRRtn7sYwxKc5MMQByF7xjAJTpCUcBmAPgBIDbADzKGDMM6hUTKwct1nzncINWUjw++KD18NGSyicpifv/W6VoUZ/6qHhx4LHHfCoIu4NgcjKPIqq3/6NRI5+7q1UqVeLqID17RmIi8MknfGUinZOex+lBXNgcBLFDGmOstuxPHoBM64uu/DG1BrAFQFEANQGMIqKcRjeMiZWDHBaq8cDpgeabb4Bbt7ibpTIqZ/78PGOdHHn2ulD1yUmkgd7MW+nQId8mRSvXCAQCJccAyDM4FQdfIcjpCeAjxnMp7yOigwAqAdD1pY7ZlYPjOD2rTU72uZ8qhUPp0ur68oxUeoNoT48Dgh21Vqiw6iZbqpR65eKUcMiRg4f/njjRmfYEguhkA4DyRFTGY2TuAq5CknMEQAsAIKJCACoCMEz6HXMrB1ciFzydO5snatcTDiVLRo9+XcsTyowyZXjeAqc8guLjeSY2gSCGYYylEdELABYAiAcwnjG2g4j6eM6PAfAegAlEtA1cDfUmY+w/3UYRg8IhJEPjlCk8XMSVK861KR/Y5SsHK4OpHfXLrl3ccyncBCIcpk3jKVulLGoCgcASjLF5AOYpysbIjk8AsJWpSaiVzOjVi0cfNcvWZVcVoiccrHgu2REOlSpFJm+BtO+hUyfr1+TJw436gshhlFNakKkISjgQ0Sue3XbbiegnIgrTDiF9HDdId+9uWuVmuTL6woMx4J13jBuQxyOyIxyimUqVeD87dox0TwR2WLHCHd8vQcgJWDgQUTEALwKozRirBq7r6uJUx6IOyaPmhx9Up45fVkTGtJLmMlxqJYFAIAiAYNVKCQCSiCgBQHao3afcjzIInJQ8XhbvhyktHU8/bd6uPCSEQq2UWvcu42uFcBAIBCEmYOHAGDsO4P/AXaRSAFxkjC1U1iOiZ4hoIxFtTEsz3K0dNGXPAcm3gmjg6695TH850kCsFz7aCspl+s6d/uEpFCuHh+setNaeEA4CgSBEBKNWygMev6MM+K67ZCJSbfVljI2VdvUlhDhGzlKL7uyn9eK4lSnDwz7UU+48B88FUaSILwNZMANz5cr+7wcN8h3HxeH0rfNYWRI43r8P8NFHQP36/vUHD+ab58yisAoEAkGABKNWuhfAQcbYGcbYLQAzATR0plv2yXcVKHXRvF6PB4BCbwCj6xhUWrvW5+EjCYH27YETJ0ITlbNhQ+B///O7X5OngJSXewFvvgmsWeNfv3Fj4MIF/9WHQJMxG8fg0IVDke6GQOA6ghEORwDUJ6Lsnkh/LQDYjNLmHC1N9o1JZGjlvFHmIwbseWyYVbXSlp3EOwJLXL5xGX1/74tmE5tFuisCgesIxuawDsAvADYB2OZpy4KbTmj4aYa1etVO89c/bveVsYoVVPV0h3PJEO0XYM8B1z9ZoDrSTnIqsEmGJ+jk2dQIbAIUCFxOUNNUxtgQxlglxlg1xtiTjLEbTnUsVORP5a+/V/SVHbx0hB/IZvgnLnPHq9NXT/s3EBcH3LjhlynNVDTYWYWIlYNAIIgCMt1IpLVJrkmD3fi2FoB77/WWSbPNk1dOqS/IksX5QbxbN+Cpp3iyd4FAIIgwmU44aNkcjucC+rSHn5vq5ZuXPUcOqIwqVjSvk5QEfPcdD9ctiEpOXjmJ3f/tjnQ3BIKwEHOB98zQEg4SP237CQWTC6JFWYNUnIHw5JNA+fL8LxJB8ASOUOSzIgAANkSElxDEPkI4yOg6sysA/uN//x7gjylAaoWygd2ocWOgalV+TAQ0aMCPbawMmIhxIxAIIoT7hcPGjUAdo00L/lgdbueXB2gogJltwaoHMEivXGn7kovXLyJXtlz27yUwhMROcoHANu63OXz5pa3q8pVD0+5Ag16B37p7R+nIuvBIy0jDf6nqHBvrj69H7o9z45edvwTeIYEmYgUmENjH/cLBLM+CArm30ooywNoS6jqz/52tKjt37Zxq4P7Nsz1iTo0kVf1PVn+CH7f9qCp/Zf4rKPBpAVy+cdlbtillE+qN4yE7lhxY4p3pqgL6CQQCQZhwv3CQ50KwgJXh9sGp6oQznad3xiPTH8GRi0dAwwgf//kxzmcHbhsI/F+rHKr6by5+E4/PfFxV/ssuLmB83lDAkOVDZP0TAkEgEEQedwuH9HS+Ic0GgSYDOniBR0o9evEoAODtZW8DAK5kBTIs/heXHFiCk1dOqsoT43wutOFUgTDGsO7YOqF2EQgEKtwtHBISgFmzTKvd19V3bOStZMSB8zx4U+/fegPwH8SlMA1m3DvJt8lOag8AEuJ8fgEMzDR8BmMMt9KDiU3OmfzPZNT/rj6m75wedFsCgSC2cLdwMGFqVaBZd+APWeikYOfIRy4eCbIFTrsf23mP5cLBCuM3j0eW97Pg8IXDQfVh91m+oWvP2T1BtSMQCGKPmBYOaXHA8jL+ZdOqBtlmBk/6I7cNBKKWuZHmU4dZFQ6MMXy36TuM3zIegG9wDxahVhIIBErcv8/BgHQN7czmIsG1KQkHOYEYkeXX+KmVFAP1qsOrsPXUVrxQ9wUsP7Tcq9bSqmsXSX0Vq0bwWH0ugSAcxLRw0LIvBGJzkAsEyb4gH5i19i3YIZ7081HfM+EeAMALdV/AlZtX/M4FO/jFEV84ipWDQCBQEtNqJaeEQ93/WU/HeerKKdAwezfR28FrNmgHvXLw3NeqQd1tCKEnEAROphAO/Rv295YFMlxsPrnZct2JW60lspYPXHLvJKMBTSlEgl05RJtaaeaumcj1US5cT7se6a4IBJmeTCEc6hevjxbdgJ+qBb7PwSpabqgXr19Er1/943TIB2T5oH/q6ilsOLHB0r2cWjlEywy7/6L+uHTjEo5fOu5Ie9Ei9AQCN5IpbA7xFI+lZYGlAQZY1UJv4FHO7g+cP4ARa0Z4PYy81+sMyL/v/d17/Pm6z037kJaRZujtxBgDA/PaF/z6mknSkQohIRDYJ7ZWDr/9Bkydij871gLgEw6NSjbCyNYjUTp36ZB3QTnglvuyHEZvGK2qZ0XP//P2nw3bfmfZO0h8LxGpt1J122g0vhHi343XPQ84N3jK3XODwan+RMuKSCBwI7ElHMqVAzp3RoNmTwLwCQcC4eX6L2PWo+a7qYPh5JWTljeU+amVApzBbzm5BQD8gvgpWXNsje45J9VKSw8uRbYPsmHV4VUBt5FZVjICgRuILbWSJ69zfDYeJfVqFl6cznhwPrnLaCgoPqK4915Os/30dsfb9LqyOjBTX3JgCQBgxeEVaFyqcVBtOTXjF+okgSBw3Lty0Aq4J+n7e/QABg/Gu03424LJBQGEdrCYtHWSbcHw3O/PWa5b/ZvqjietkWbqTriyOtE3kZRHIIge3CscevZUl3lWDsiSBRg2DKlZwtedbrO72b7mm43fOJKwPtBBNRTeStGk54+mvggEoYSI2hDRbiLaR0QDNM73J6Itnr/tRJRORHmN2nSvcJg7V11mMkhG42CRlpGGn7b/5Fh7Z1PPWq7r5D4HLXvB4gOLkZ5hX80m1EECgXWIKB7AaABtAVQB8BgRVZHXYYx9yhiryRirCWAggBWMsXNG7bpXOGTRWBbEue9x4igOF29cdKStRfsXIf+n+TF/33xL9ZUrjlaTWqHU56WC6oM0sC85sAQtJ7XEB6s+sHyt0wZpIWQEmYS6APYxxg4wxm4C+BnAAwb1HwNgOiN132gqoSUczFYOUThYaO0/0MNs8Fx7bC0AYPWR1bp1ft/zO9YcXYM1R31eTNKKatGBRQGHJFcKmhOXTwAA9p7ba3rttB3TUPrz0iEz5gsEMUACEW2U/T0jO1cMwFHZ+2OeMhVElB1AGwAzzG7oXuGQmKguc6FB0xFDrkdomOWePnThEO7/6X40HN8QDcc3tO2tlHorFc0nNseuM7t069hV3U3+ZzIe/eVRHL542NAlNxCiUY0Y7Xyx9gv0ntPbvKIg3KQxxmrL/sbKzmkNInpf/vYAVpuplAA3CwctQhyoLhTYWTlcunHJ8LzXhsAYes/pjfdXvu93XrlZzq630rKDy7Ds0DK8tvA13XvbgTGGJ2c96Xvv+T4rP6e0jDTHQmoIjHl5wcv4bvN3ke6GwB7HAJSQvS8O4IRO3S6woFIC3CwcKlZUl2W4L7qoHeHQZUYXzXJpUJXaymAZ+G7zd3hn2TuG7dn1VjJbmcjP6Q30WnW973Xqvjz/ZRQfWRznrplOdvyYu4c7LVy5eQWnr562da1A4CI2AChPRGWIKAu4AJijrEREuQA0AfCrlUbdKxy0MBnkKuSrYHg+Euw7ty/oNqRB1crgLUfPW2nwssHYd24fWvzQwm9QNVodSPdec2wNcg7Pie6zu1t/AA9KwSIhxZu6cP2CrfbkiZHGbx5vUDMytJrUynZ4d4FACWMsDcALABYA2AVgGmNsBxH1IaI+sqoPAljIGLtqpV33CgctXb2JcEjOkhyizgRO68mtg25DGkylwfvj1R9r1lMO7q8ufJVfr/i/vbfyPZT/qjyWHlyK0evVcaGMVgML9y/E5Zs+24GRTUXZjp56S64uCxQ7K7RwsejAIkv1jl06hhFrRoS4N9HBxC0Tsf74+kh3w3UwxuYxxiowxsoxxj7wlI1hjI2R1ZnAGNNWP2jg3vAZAQiHWEUaNEM9ABoN9E64oSoH/7OpZ7EpZZPtFZEW0SgcrNLx5474O+VvdKzUEWXzOBhaOArp8WsPAAAbkjl/y9GEe38xWigGl9H3jcaguwf5la3oscIbTiNWYGDIYBl4Y/EbhvV0M84ZDLpa1zgWNVVpc1C8b/djO7Sa3Mqb/MeplcObi97Eg1MfDLitcCOp0wLZUCgQBIp7Vw5aA0XWrH5vn6ujjl10T6l70Ob2Nvhh6w+h6lnYYYxZGjj0ZvfBenGtPrIas/61H/FWT60kle/6j7vM3kq/xcuDEEryZ//kr08AADvP7ESVAlX0LokavGpDF7pqC9yLO4XD2rXAhQv+ZTNmAKWC293rVj768yPsOWctVLgWVl1Z9YTL3d/fHfC9jfohJTGSNsfZEWLHLh3ze6+lVmo+sTlOvn7SbjcjhghpLggn7lMr3boFNGgArFcYrR56yHITVn9k+bPnt9OziDFqwygs3L/QtN7XG77WLDeakQ9bMUxdXzZIHzh/wEIPtVHeVykcpBDrequiJQeWYFPKJs1zJUaW8HsfH6cO134rg69Ijl065qjK5nradWw7tc2x9qJxf44g9nGfcNDay/Dkk+oyB6iYT2MvhYv5cv2XmuWMMUsDkKTWyGAZeG3Bazh84TDKfVnO0r1PXz2N1FupuHzjMtr/1B5HLx5V1VEO0NLKwatuAsOJyyew4/QOAMC9k+5FrbG1LN1fb0Jw7NIxlBhZAoOXDbbUDuALDaJHj9k9cMeYO3D+2nnLbVrBTK108PxB/HnkT0fvKWfo8qHC9TYT4U61kpLcuW1VL5RcyFI9N3u42GHM32PwUv2XDOtksAw8NuMxADwD3ZKDS/DnUWsD0aELh1DmizIAgMr5K2PXf7uQP3t+jGk3xq+eXAgAvtm+XK1UbAQPGfNaA/UubQmtzXJ6ObRPXuFqpfn75+ODFtaCBBYbUQz7+u1DubzagnHl4ZUA+I70PEl5LLVphFVbS9kvuSdTqDx9tFaRgtglqNGPiHIT0S9E9C8R7SKiBk51zBbVqtmq/m6zdy3ViyUDoJld4fO1nxueP5t61jvoJsbzuFanrpyydO/DFw57jyUjM4FUg7gkBKQwIV6bg2dF8eGfH3rrfrbmM817nbl6Bvk+yacq1xP0ge6hMApQmHIlhbcdouRMEsVHFEfrya2RnpGuyjcuEARLsFPjLwDMZ4xVAlADfHdeaFH+iOPigKefttVE1oSs5pUQWwZAM536lZtXLLeVJZ5HxD188bBJTR5YT2vm+/2W71F0RFHNa/ov6g+ArzgAHltJasuIKzevYOTakZrnUq6koOhnRbH3rC9KLBH5hRxxGqe+P3LBdeH6BdAwwm+7f8Pxy8excP9CfLX+K++qTiBwioCFAxHlBHAPgO8AgDF2kzF2waF+6aO0OdSsGVA01r+e+gtTO03VPJc1nguP7InZbbcbrUgDrB7SLF0P+SzYrrrtr6N/2ar/55E/AzLC9vujH4b/OVzz3C87f0HKlRS/kBoAQiocrHL1pjqaAWMMC/cv9Ps/EBF2ntkJAH7PaWYDETgDYwyrDq/KNA4CwawcygI4A+B7ItpMROOISBWfgoiekWKQp6UZD1CWUH4wAX5QDUo0QOeqnTXP5U3i2fN632UeurhDxQ4B3T/cmOVKkDaaadFtVje/89KM3ipvLX3LVn0A2H56u636O8/sxIQtE3TPS6owyR4A8Jm9Mmz5icsnQMPIG7RPD7mw/GLtFzh4/qBhHSO0wp38vP1ntJ7cGmM2jvFbeUlCLJZUntHE9bTrWHpwKRhjGLxssJ/6cMq2Kbhnwj2Y9M+kCPYwfAQjHBIA3AXgG8bYnQCuAlDlLmWMjZVikCckOGD/DoPUlgaSGoVqmNaVBEm0Y6ZWmrpDexUFAJP+mYQ5u1VBHkPKHWPusFX/rm/vMjy/5eQWzXK5BxYA/H3ibwDAt39/a9iepDI6f+08Xl7wMlr80MJOd/3Q8mqSVHZyQUwgzVAp4Z7JxvLM+cU/XkSLH1pg2o5peG/le3hk+iPec/vP7fd7jXWCEQ7HABxjjK3zvP8FXFiEFuUXMwQzKMm/PpYINstaJNUuVriRfiOg66wYpG+k3dANFy79X7VSvVq1OVy5pW/vISK/vimDLALA0kNLDdtfeXgl/kv9z1JfJM5fO297hRgLSGo76f+ltaKOxoySoSBg4cAYOwngKBFJmwFaANjpSK+Mb+xoc9UKVsMbDf1jEkkulNE+INoh2E1ekfxfhOrHKDdIG93jvh/v0/SA8mtLQxBYVf1oCSZNgaAQFBJ6GwElmkxoguYTm+uen7hlIr5a95VfWbERxbzux6q+xfDgqAzyqLT5ZCaC1fP0AzDFk2DiAICewXfJBIeFw7a+fCfruM3jvLND787cGMppHOyzXLyunhmHi1AKJqVBWmvgW3pQf2buhIrFLNqtN18HSJXYyQzp2m2n9XdsS5FQ+9Xr5y27lnbNUvuxhtEmwlhWp2kRlCsrY2yLx55wB2OsI2PM2S2h2jf1f1+njiPNyndDSysHMw8fNxHsymHwcus7iMON1m5rK8gN0noCyGr2OadnlVpCiogsG6QzWAY++vMjTXVX0H3LRINkLK+SzHDfFmDlF/OLLxxp9rfHfvMeSysHKzPW7jW0M54Z7eCNBLvP7o50F0LGoKWDzCvpoDRISzpnaWKgFwLDaBOckyiFgNXcHb/v+R0DlwxEno+1d2inXE6xtbdFzrW0a6p85LGGlgDMbGol9wuHrNY2tJmRL3s+3FPqHgDAAxUfAAAUyF7Ar06urLlU1zUt3VSzvXbl2znSL6cIxpsm2vn3v38Duo6IUP6r8gD4YDBr1ywMXDIQgM+VVm/m2G12N9xI8xnBb6bfVO3nsJybW8NeoWVzAGSurCbGbiPXZAAoOqKoqYeXHoX+rxCSP4y+rIqhIDOtkpS4Tzi8/nrIml74xEKcfeMshjUbhpOvnUSR24r4nbez+SszL0fDzcYTG4Nu4/jl43homvXIvoC/2vHSjUtoNL6Rd+UB8O8AYww0jDBw8UDddow8reRCgDHm/V7J7xMoe8/tNa+kQayvGgDx+wXcKBy+/z5kTWdNyIq8SXkRR3EolEMdnE+5rHy53ssAgDK51V4dTs043BI23I3IP6Ob6TftXw+mGkTOXD3j17400/9o9UcAgEFLBmHU+lFYcWiFt96P237UbFurn1J7UvymQJBvMFR6KQk4Rr/fzLKacJ9wiBC97uyFxU8u9isb0ZonfT/w0gEMaTLE75xTHjaZ5YsYCQzTo1rY/3Ar/ZbK0C9v8+KNi6p7DP9zOPr90Q9NJzbVbXfKP1Ow4cQG3g/ZhERaiQTLh6t8AQxfnP9i0O0Fy59H/vSGYTdj55mdoGHk3awoCB1COFhkXIdxuLPInX7hkOU/3KFNh/rVD1Q4FL3NPxidXlhoQfCcvnpa95wV42PeT/Ki5rc1/crkg3fl0ZWx79w+S30Z+/dYbD25FQDwxKwn/Hak+60iglB3SHaIaFILbTm5BY2/b4xq31iLrPzbbu44orej/8L1C1i0f5Fj/dPzGssMCOEQACt6rMCKHisM6wT6Iy6du7Tf+7a3t0X94vVNr0uMSwzofgJtpJWD2UCgFDDKz73ZxGaW7vfs3GdVggYA/k7525tzwnJSJp2EPA9OfdBSX4LhbOpZQ6GrZMjyIeaVZHg3qen8HzpN64RWk1vhbOpZ07bWHF2j66r8/RZ99XVmWc0L4WDCv8//i+19/YPA3VPqHq9nkx7ZErIFdD/l3orOVTtjVc9VuKOQcawhKeWlwBkCnR0qV4xyG0QgzN833+99MCuH+fvmY+/Zvdh8cnNQfTIi/6f5Uej/tJNp/XnkT/yx94+g2jfz0pIM9WbeWgDQcHxDtJzUUvPc1lN8Fee3Q9pz76u3rmYKASGEgwkV81dE1YJVbV/XpFQT7/GJV62HVJbrsNkQhioFqiAhLgENikcmj1JmJdAkQMp8EvKd6cEOKAwsaFtWhVEVbO3ROHnlpGPpTht/3xj3/XifI20xMNxKv4VSn5fCjJ0zvOXK8Be613s+C7PQI1qMXDsSn6z+xPZ1bkMIBweZ0Zl/SRuVaAQiwuGXDyPltRSVS6ySduXbIWfWnAACD3MhXS9whoMXDqLv3L62r5u3d57uOaMQFlYJ94y1yGdFUPizwravM9uRf+3WNbyz9B3b0X7lK7pz187hyMUjeG7ec77zFoV6sELWKIpxrCCEg4NIm+akL3DJXCVROIf6hzWvq28AYUMY5nadi90v7Mbfz/yt+6My+rKv770er9Z/NZiuCzQY8/cYywZlKxgFv7OCfJ9DOAnEzTfhPeOwbZ+v/Rzvr3rfVpuNxjfyZgn02ySoERzPdOWgcV4vXM6o9aNAw0gVQsaqDcitCOHgIEZfSGlPBAC0Ld9Wdb5wjsK4q8hduiuH5Cz6O1LrFKuTaTwowo1dNYhRRj0rdiGzeF5uixQ8f998LNy/UFX+6+5fDa+bsGWCSjDLd6DvOLMDxy4dA+D/u1OuHCZsmYCJWyaq2tf6Py7Yt0BVdurqKfT7o5+qHODRbuuOq6t5bv3x9fhg5Qea59yCEA4hQMtoNrIN10VrheCQ81mrzzTL3232bvAdE4Qco1wgl25cMr3eaFPai/NfDDqAYqD0X9jf9jWzds1C2ylt0Xpya5WdY93xdTpXcXr+2hO1x9YGwPNpKD2wFuxfgNr/q626Trly6PlrT2/UWTlrj61VlWlNzIwCL646sspvd/63G7/1eknVG1cPby97GwC3a7hxhSGEQwjQW0FMenAS/n7Gt3lHK8Vom9vbaF6bI0sO43u68MsXiwSadEji1YX66sG5e+Z6Yz+Fm/9b83+2r5GHIyn1eSnb10sRZc0CBGp5FBn9HjalbEKTCU1U5VaTMwHq3/jWk1vR5/c+6Da7m1/5jJ0zUGtsLUzZNsVy29GCA3k7I8hC9XI1mnnijie8xzfevuFoxjkRCyZzsP+8tRSV4f4+dPy5o6mqKBzIVw795mmrg1Iua4cesfM/U6adlVxn5+2dh6HLh3rLpaCQu87sstx2tODulUNLbR/lSGNlBpIlPos3b4RVRrQaoXtOrBwEcgJRP03dHrgHTigFgx17mvTbG71+NEZtGKVZRykE5u6ZG3jnwO0a8nhVw1YMC6q9aMHdwiFG2fLsFk3bwysNXtG9RqwcBHICCWPeZUaXEPQk9Jy9dhbjN48H4IucPGKt/kRKSfuf2gd1/56/9kTv33oH1UawEFEbItpNRPuIaIBOnaZEtIWIdhCRcYgHuF2tFGU4NXuvUbgGahSuEZF7C2KDd1eGz4HBifDhQHARYnvN6YVsCdksq920CMVvKByTNiKKBzAaQEsAxwBsIKI5jLGdsjq5AXwNoA1j7AgRFTRr110rhxWmwi6iSJvd7i55d8jv5aS9QiAIhqpf248goIVehFirhuLHZz5uWmdzymbNlYLLdzzXBbCPMXaAMXYTwM8AHlDU6QpgJmPsCAAwxkwDYLlLODz3nHmdCFIhXwXsfG4n3mv2XsjvpYzdFEm10oanN0Ts3oLIcezSMeT6yNg1O9oYt2mcZvmbi98Myf3seEAFQTEA8kTqxzxlcioAyENEy4nobyLqBhPcpVZywUavygUqh+U+ermFI4GdDHmC2KHEyBIhv0fqrVRHJj4L9i3Q3NsgJxQTLAfbTCAiecrDsYyxsZ5jrYFReeMEALUAtACQBGANEa1ljO3Ru6G7ftXyAfEVfeNsZmDSg5P83jv1JZzTxV6sG8BYxXXq9VPBdMcR+tTqE+kuCAKk3Y/tHJn4tJnSBkNXDMXlm5cd6JUx5fL4crC8s+wdp5pNY4zVlv2NlZ07BkAuqYsDUEb7PAZgPmPsKmPsPwArARgaNt0lHOTkzh3pHkSUjpU6+r13auWgJWTMwoUbueTK80wUStYO5axHwxINbdXXo189bX93QWTQSouqx/JDy9FwvDPfA8A4TtRHf37kyD2CMYoHyAYA5YmoDBFlAdAFgHKW9yuAxkSUQETZAdQDYLj5wl3CQb5yKBH6Ja2bCGbl8NPDPxme/73r74bnjdRKcvXXkVesh4oGgEerPmqrvm4fwqP3FVhECp5nlT1ndTUftjGKpmoW0iNaYYylAXgBwALwAX8aY2wHEfUhoj6eOrsAzAfwD4D1AMYxxrbrtQm4yeZw6RKwTRbyuEePiHUlWljSbQmu3rwadDsPVX7I8Lw8HtTj1R9XhQLQEw4nXj3hdy5LfBZb/XLKlmEWzE4QXk5ctp7fRGANxtg8APMUZWMU7z8F8KnVNt2zcuja1f+9C4zToaZ5meZoX5G75d2W5baA25HPrLXUU7dlvc2b27pFmRaq83o2h4S4hKBm7b3u7BXwtXKU0VAr5w+P04BA4GbcIxx2ydRjOYyD0MUyJXOV1Cx/veHr+LSl5UmBH1bCEyQlJAHwV191r9EdD1R8wCs4lDCwoGb/SYlJAV8rR7lyKJFLqCQFAjPcIxziZF1Ny7xqgp3P7cR//f9TlWdNyIrXG74eUJtWBnBJgMjj4Her0Q2zu8w2zGFgNS5O/eL1/d4fePGApeusIDeKA8L1ViCwgnt+JfJB5rp58vBYJTlLMvJlz+dom35qJR3DtlRHLhyqFKgCgAsmPQIdiMvkKRPQdVrULFwT37T7xtsXIRwEAnPc8ysRNoaAuDLwCtgQ5k1hqoWV2X2FfBUA+Gwb91e4XzMFqhw77rXyuq/Ud3YPCxGhT+0+yJMtD38vvJcEAlPcIxzi3NPVaKFivore9KInXjuB9hXUMWX2vODvJqgc0I++wnflT3loCuZ0mYMu1brgg+YfYHyH8Yb37t+wPwomm8b20mREa3VEzc3Pbg6oLS3EykEgMMc9vxIhHIIiIS5B0zZQPl953WuSEpJQPGdxAECubLnQvmJ7EBEGNR6EAsn6KxEA+KTlJyAilb7/x4esb4CSU7NwzYCu08JuHg2BIDPinhFXqJUskT97fu+xnrqodlF17l2JpqWbIne23ACcCcmhHIiVO7sl7N6reM7iWN59eUB9GtpkKDpV6YRtfbeZV3aIfEnO2okEglDjHuEgsMS/z/+LX7toZ+WSBuCBdw/UHazyJOXB8VeP276vXIW08emNuvWSEpPAhgQvdAomF0ST0uo8wAD36DLqQ+EchTH9kemoVrAaqhWsFnRftGhVrpXfe2UUXUHmYtWRVZHugm3cIxwyMszrCJAvez6Uz6uvKgK4QfbASwdw+nXtkO5WkrQrkdRPAFCraC3L10m8WFc7lr8eWeP1PaQqF6iMu4rchZfrvYyl3ZaqzstXVHLvKyeJozg/lZqeIBNkDoRwCBVHj/pvghMETc6sOVV2gza3twHgn6TdKoF6ACUl8JXE43doJ2rZ8uwWHHzpoKrcaG8FwJ9hZJuRaFammV+Zsq/SCmprn622+25EHMXhuTq+/CNf3/e1o+07wbCmsZHrWBAa3CEchg+PdA9iguEthqNhiYa4t+y9qnOnXz+N2Y/OBhDYQP/kHU8G1KdxHbSTr0jUKFwDpXOXVpUr7Smj2monk5ejtRKa2mkq3rnnnYDUS0ZJnZQeUbmyWUuK06lKJ1t9+KhF4JFEBzcZHPC1gtjHHcJBaVjt2DEi3XALt+e9HU1KNcG49v4Db6X8lbD6qdW4Las6DlOB5ALezWyJ8Ykomauk6nojXqj7gu653Nlyq/ZZLH5yMVY/tRpdq3fVucoefev0BWA/uF+R24rg3WbvBuTe+vY9b+uuOAJdSWVPzG65bp2idfDm3f4ZzL5o80VA95Uz/ZHpQbchcD9BR2X1JLfeCOA4Y+z+4LukeRP/90WKhOQ2sUJifCKW91ge8PVxFIfDLx+2dY00k29Wupnq3Jn+Z1SDZYuy6gB+ZkztNBWP/qIdxjuO4vDv8/8iT1Ie0z46iV6uiziKC/lmOy21n5GQy5MtD85fP2/arl6sLEHmwomVw0swSRoRNMoftXBrjTriKA7b+m7DnMfUmeQS4hIc2VsgBf/To2L+igFvvAO4F5dTJMQloFTuUn5lu1/Ybenatre3Dfi+j1R5xHv8RsM3/M61KtcKKa+l4NBLh/DXU39ZbtOp6LgCdxGUcCCi4gDaAbCuf3CCK1fCejuBNaoVrIYcWdwbMffDFh861lZSYhJerMc9sKSQI1Jo87J5yhpeO6PzDM3yj+/92PS+hXIYZ9srnKMwSuUuhQYlGpi2JWFmFxLEJsGuHD4H8AYAXX9AInqGiDYS0ca0QKOpKlcKbQOfWQncSyjUQqEiKSEJcRSH7X23Y2+/vQCse3/phSpXCl4zV+NBjQf5vbfqthtIylkRksScG2k3It0FWwT8iRLR/QBOM8b+NqrHGBsrJcVOSAjQxKEcFLp0CawdgauR6/CfvuvpCPYEmNl5puF5SRVTtWBVzdl87zt7276nXRuG0kPq4o2Ltu8pIQVe1EMIB3NCtacmVATziTYC0IGIDgH4GUBzIprsSK+UuGjGKAg995W/D0/c8YTt60a2Honsidm94UGC4cHKD6rK+tXt5z2uV7ye4fVft9Pe9/B4de39HhJreq2x0Ds1pXKVCsqTySyPsxAO5jgRjiacBPyJMsYGMsaKM8ZKA+gCYCljzP4v1gpCOAh0eLney5Z9/Z+44wlcHXQVifGJ5pVlrOttLfH8l22/tFTPaAWgDLvhdx2RX1IkK4PN2TfOInVQKg69fAiV8ley1D95u0VyWPMM1EsVK/CRmVYO4UMIB4EOI9uMVPn6O03dYnXxZRtrA78Rcl1+Ynyi7R3Kytm5FdtA3qS8hulWpTYvDbikavfuknfjxGsnLPVNHj4lMxCIR1kgtpxI4ohwYIwtD9keB0AIB0HE6Vevn9dO8FTNp/zONSkVWNykwU0G48rAKyiUbOxhJNGtRjfN8mD2JUirmGCN/Uu6LQnq+kjSt3Zf29fMe3webr5909Y1YuUgEISYcM3AJj04CUObDPW+l8Kh3573dr96y3sstxRpVtr9LFfvJGdJRs+aPQEYq2a61eimiuwqqX/W915vem89tOJN1SxcE4VzFMb7zd633I5y5WBFYElpZgPFyspLL0S8nHrFjO1DethVT2Yam4NAEG7C4cr6actPvcftyrfDkKZDVHWMfuSf3PsJNj2zSfNcsZzFMP/x+fjxYf+ER9KKQCvxkpRxTz54K9svlrOYbn/M0LJ/3Jb1NqS8lmIrkqzys6lbrK7pNf/0+UdVNrXTVMv3fOeed0zrWJlIhGvQdtvKIejwGQKBUyzrvgznrp0zrRfKH/PrDV9H/0X9AQTmgdO/UX/D861vb60qk1xdH6v2mOqc9Kx2+rKm1xpcvnHZUl3vyoEIi59c7E0rG2oKJRfS3DXfuWpn3RApeZPy+n0/rEwWrHxXrK5E8yXlw9lrZy3V1UIIh1Bw7VqkeyAIA01LNzU8H+pYRUqUA7I0Q7fqwWOVvEl5cWXgFc2gezmz5gQAVeBCQH9Qk3s0meG1OYBsx7vqdWcvfLf5O1vXSEgDJYEsC/thTYfhraVv4dKNS+aVPaRnpJvWsXp/o3qty7XGgv0LjK/PjAbpkDNpUqR7IIgCahSuASCwDWSBoBQOz9V5DtMfmY4eNXs4fq/kLMmaM+GHKz+Mb+//FsOa+fTrTqrXrObu0BKI8n0dqnZNBLkUIt1OzK06RevYHmDNZutvNjL2dCuTuwzuLHwnAOPB3YpTgttWDu4QDi6TuILQUPS2omBDGB6u8nBY7qcUDnEUh05VOoU1jAcR4Zlaz2imGXVCvWY169/6p/2N3nEUhxqFayB1UCquv3UdgH+4cbO0qDMf5TvM7eyPqFe8nu0B1ux/9NG9xntkutfojsXdFpu25ZSKK5pwh3AQrqyCCBCtsZyqF6yOXnf2ws8P/xx0W5IANBu4iucsjoYlGgLgdgEpj0VSYpI3D8jp108jOZHbLIY2Heq9tnHJxqr2pF3q8v9x9YLVA3sIA5TCRB680IpgTMtIM1wFyVPB2u1LtCOEg0CgQ7SGhIiPi8e4DuNQuUDloNuSkjRZSZL0Vduv0LBEQ0x4YIJm5rzkLMne4ICSrQQA5nadq9umfOXwT1+195ISpRA7+dpJ7Ou3z/u+fYX2WNd7HX7v+ju2PLvFOyBLn2WpXKW82fYkwfRQ5YdQp2gdr2CTk5aR5lO9aQiRf/r+YzkplhAOAkGMEK3CQQstg7UVhrcYjvTB6ZaEw11F7sLqp1Yb7rjWWoHIBYUSu/9jaYCVQpoUylEI5fKWw2etPgPA96DULVYX95W/DzUK1/DWl69UJj/IQ8BJK4I8SXmw/un1KJe3nOp+aRlpXnXZ83WeV52vlL8Set1lLd+FMEiHArFyEEQANwmHPf324OgrR21fR0SOPqc0AFr1LDMSNEbtKzPwKVcIEtLGRS1PMGVd5UZAAKhcoDKyxGdB+uB0vN9cf1Ngm9vbAACmdZqm++xi5SAQxAjhdp0NhtzZckdVfCOlvWZ4i+GaQQVX9Fhhq906xeoAUBuy5a6xcr69/1t8fd/XmgELVbGqZKuezc9uxqZnNnl3r8dRnKENqmbhmmBDGB6p+ohuHSEcQoFYOQgiQCQN0idfO4mU11Iidv9AaVSyEQC1t9KAuwfg0arqzW1WI8VKzH1sLtb2WqsKXSGtKJQDfu5sudG3Tl80KM4z35XNU9ZrRJbvhtfiziJ3BvUdyJXVP59GKL2ViKgNEe0mon1ENEDjfFMiukhEWzx/g83adMcmOCEcBGGkXrF6WHfcWpjuUGGW7jNamfLQFOw5u0fTzmCWMEhJx0odMfvf2X5lubLl0syV4V056IwVr9R/Be3Kt0PF/BUBwFIsrEDRW5mEauVARPEARgNoCeAYgA1ENIcxtlNRdZWdAKnuWDnkcG9eYoH7WPDEAmx5dkuku+FKsidm9+rrldxd8m7seG4HMgb7D5Jf3/e11zUWAH555BcAQM1CvJ0n7ngCZ98wDlvxbO1n0a58O7za4FXN80TkFQx6WLG9rOyxEt+0+8a0HqAOzBdCg3RdAPsYYwcYYzfBk689EGyj7hAORZwNVyAQGJErWy7vbmyBs1QpUEU1u+9bp6+fgfnhKg+DDWEokMw9sHIk5kDepLyG7eZNyou5XeeiYHLBgPs2/ZHppnUal2qMPrX7WGpPmXEwmDStABKIaKPs7xnZuWIA5N4IxzxlShoQ0VYi+oOIqpreMJjeho0M2UyjdOmIdUMgEFhnWNNhmnsHrCIZsJ+s8aRTXTJEGYo9UDpX7YxpO6YhIc5/eL2Zbi//g4I0xlhtnXNaujTlMmUTgFKMsStEdB+A2QDUYYDljYbT9zY5OZldvXrV/oW1agGbPGGKXeYrLBAI3AMN82x4C8ImkZ6RjhvpN1BvXD1sP70dANC1eldMeWhK4P0iSmWMaUpaImoAYChjrLXn/UAAYIwNN2jvEIDajLH/9Oq4Q60kBIJAIAgDc7rMQYeKHYJqIz4uHtkTs3vdat9r9h7G3j/Wie7psQFAeSIqQ0RZAHQBMEdegYgKk0efR0R1wcd+Q0OO+9RKAoFAECLaV2yP9hXbO9KWZFu5v8L9Ic2TwRhLI6IXACwAEA9gPGNsBxH18ZwfA6ATgL5ElAbgGoAuzERt5A7hkG4ek10gEAiiCW9QwzBoPhhj8wDMU5SNkR2PAjDKTpvuUCulpUW6BwKBQGALSa3ktp3REkI4CAQCQQiwmkgpWnGXcFhgnIZPIBAIooVwqpVCgbuEQ+Xg49cLBAJBOJCEg1ArhZJsniBeidazLgkEAkEkkTYARmtGQTPc4a30xx/A9OlAIXcGIxMIBJmPyQ9NxpiNY1CnaJ1IdyUg3LFDWiAQCDIxRjukQ4U71EoCgUAgCCtCOAgEAoFAhRAOAoFAIFAhhINAIBAIVAjhIBAIBAIVQjgIBAKBQIUQDgKBQCBQIYSDQCAQCFSEdRMcEWWAJ5oIhAQAmTk8q3j+zPv8mfnZAfH8CQASGWNhncyHVTgEAxFtNEiwHfOI58+8z5+Znx0Qzx+p5xdqJYFAIBCoEMJBIBAIBCrcJBzGRroDEUY8f+YlMz87IJ4/Is/vGpuDQCAQCMKHm1YOAoFAIAgTQjgIBAKBQIUrhAMRtSGi3US0j4gGRLo/oYCIDhHRNiLaQkQbPWV5iWgREe31vOaR1R/o+X/sJqLWket5YBDReCI6TUTbZWW2n5eIann+b/uI6EtySU5GnecfSkTHPd+BLUR0n+xczDw/EZUgomVEtIuIdhDRS57yTPH5Gzx/dH3+jLGo/gMQD2A/gLIAsgDYCqBKpPsVguc8BCC/ouwTAAM8xwMAfOw5ruL5P2QFUMbz/4mP9DPYfN57ANwFYHswzwtgPYAGAAjAHwDaRvrZgnj+oQBe16gbU88PoAiAuzzHtwHY43nGTPH5Gzx/VH3+blg51AWwjzF2gDF2E8DPAB6IcJ/CxQMAJnqOJwLoKCv/mTF2gzF2EMA+8P+Ta2CMrQRwTlFs63mJqAiAnIyxNYz/Un6QXRPV6Dy/HjH1/IyxFMbYJs/xZQC7ABRDJvn8DZ5fj4g8vxuEQzEAR2Xvj8H4H+lWGICFRPQ3ET3jKSvEGEsB+BcKQEFPeaz+T+w+bzHPsbLczbxARP941E6SWiVmn5+ISgO4E8A6ZMLPX/H8QBR9/m4QDlo6tFj0v23EGLsLQFsAzxPRPQZ1M8v/RELveWPt//ANgHIAagJIAfCZpzwmn5+IcgCYAeBlxtglo6oaZbH4/FH1+btBOBwDUEL2vjiAExHqS8hgjJ3wvJ4GMAtcTXTKs3SE5/W0p3qs/k/sPu8xz7Gy3JUwxk4xxtIZYxkA/gefqjDmnp+IEsEHximMsZme4kzz+Ws9f7R9/m4QDhsAlCeiMkSUBUAXAHMi3CdHIaJkIrpNOgbQCsB28Ofs7qnWHcCvnuM5ALoQUVYiKgOgPLhhyu3Yel6P6uEyEdX3eGl0k13jOqSB0cOD4N8BIMae39PX7wDsYoyNkJ3KFJ+/3vNH3ecfacu9Rev+feAW/f0A3op0f0LwfGXBvRG2AtghPSOAfACWANjrec0ru+Ytz/9jN1zgoaHxzD+BL51vgc+AegXyvABqe35E+wGMgmfXf7T/6Tz/JADbAPwDPiAUicXnB3A3uPrjHwBbPH/3ZZbP3+D5o+rzF+EzBAKBQKDCDWolgUAgEIQZIRwEAoFAoEIIB4FAIBCoEMJBIBAIBCqEcBAIBAKBCiEcBAKBQKBCCAeBQCAQqPh/vK0nliTtrE0AAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 2 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"iters = [item[\"iter\"] for item in result]\n",
"fig, ax1 = plt.subplots()\n",
"ax1.plot(iters,[item[\"loss\"] for item in result],'g')\n",
"ax2 = ax1.twinx()\n",
"ax2.plot(iters,[item[\"accuracy\"] for item in result],'r')\n",
"plt.xlabel(\"iter\")\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 45,
"id": "cab7889f",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"gpu allocated : 2778 MB\n",
"gpu reserved : 3272 MB\n"
]
}
],
"source": [
"torch.cuda.empty_cache()\n",
"print(f\"gpu allocated : {torch.cuda.memory_allocated() // 1024**2} MB\")\n",
"print(f\"gpu reserved : {torch.cuda.memory_reserved() // 1024 ** 2} MB\")"
]
},
{
"cell_type": "code",
"execution_count": 46,
"id": "29ffab84",
"metadata": {},
"outputs": [],
"source": [
"torch.save(model.state_dict(), \"model.zip\")"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "4b9b9579",
"metadata": {},
"outputs": [],
"source": [
"del batch_inputs\n",
"del batch_labels\n",
"del loss\n",
"del optimizer"
]
},
{
"cell_type": "code",
"execution_count": 88,
"id": "80d0ee50",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<All keys matched successfully>"
]
},
"execution_count": 88,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.load_state_dict(torch.load(\"model.zip\"))"
]
},
{
"cell_type": "code",
"execution_count": 89,
"id": "5c9b570c",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"MyModel(\n",
" (bert): BertModel(\n",
" (embeddings): BertEmbeddings(\n",
" (word_embeddings): Embedding(119547, 768, padding_idx=0)\n",
" (position_embeddings): Embedding(512, 768)\n",
" (token_type_embeddings): Embedding(2, 768)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (encoder): BertEncoder(\n",
" (layer): ModuleList(\n",
" (0): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (1): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (2): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (3): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (4): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (5): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (6): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (7): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (8): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (9): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (10): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (11): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" )\n",
" )\n",
" (pooler): BertPooler(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (activation): Tanh()\n",
" )\n",
" )\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" (lin1): Linear(in_features=768, out_features=768, bias=True)\n",
" (gelu): GELU()\n",
" (lin2): Linear(in_features=768, out_features=1, bias=True)\n",
")"
]
},
"execution_count": 89,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"device = torch.device('cuda')\n",
"model.to(device)"
]
},
{
"cell_type": "code",
"execution_count": 38,
"id": "fff7a7d0",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████| 1250/1250 [00:34<00:00, 36.39batch/s]\n"
]
}
],
"source": [
"model.eval()\n",
"collect_list = []\n",
"with torch.no_grad():\n",
" with tqdm(dev_loader, unit=\"batch\") as tepoch:\n",
" for batch_i,batch_l in tepoch:\n",
" batch_inputs = {k: v.cuda(device) for k, v in list(batch_i.items())}\n",
" batch_labels = batch_l.cuda(device)\n",
" output = model(**batch_inputs)\n",
" loss = BCELoss(output, batch_labels.double())\n",
" \n",
" prediction = (output > 0).to(device,dtype=torch.int64)\n",
" correct = (prediction == batch_labels).sum().item()\n",
" accuracy = correct / prediction.size()[0]\n",
" \n",
" collect_list.append({\"loss\":loss.item(),\"accuracy\":accuracy, \"batch_size\":batch_labels.size(0),\n",
" \"predict\":prediction.cpu(),\n",
" \"actual\":batch_labels.cpu()})"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "4e9a90b5",
"metadata": {},
"outputs": [],
"source": [
"def getConfusionMatrix(predict,actual):\n",
" ret = torch.zeros((2,2),dtype=torch.long)\n",
" for p_s,a_s in zip(predict,actual):\n",
" ret[p_s,a_s] += 1\n",
" return ret"
]
},
{
"cell_type": "code",
"execution_count": 39,
"id": "b7a513c9",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"average_loss : 0.33939967472716237, average_accuracy : 0.86565, size :20000\n"
]
}
],
"source": [
"total_loss = 0\n",
"total_accuracy = 0\n",
"total_size = 0\n",
"confusion = torch.zeros((2,2),dtype=torch.long)\n",
"\n",
"for item in collect_list:\n",
" batch_size = item[\"batch_size\"]\n",
" total_loss += batch_size * item[\"loss\"]\n",
" total_accuracy += batch_size * item[\"accuracy\"]\n",
" total_size += batch_size\n",
" confusion += getConfusionMatrix(item[\"predict\"],item[\"actual\"])\n",
"print(f\"\"\"average_loss : {total_loss/total_size}, average_accuracy : {total_accuracy/total_size}, size :{total_size}\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 38,
"id": "1ac327de",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[8716, 1638],\n",
" [1194, 8452]])"
]
},
"execution_count": 38,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"confusion"
]
},
{
"cell_type": "code",
"execution_count": 122,
"id": "3e71d4d2",
"metadata": {},
"outputs": [],
"source": [
"def getF1Score(confusion,c):\n",
" TP = confusion[c,c]\n",
" FP = confusion[c].sum() - TP\n",
" FN = confusion[:,c].sum() - TP\n",
" precision = TP / (TP + FP)\n",
" recall = TP / (TP + FN)\n",
"\n",
" f1Score = (2*precision*recall)/(precision + recall)\n",
" return f1Score"
]
},
{
"cell_type": "code",
"execution_count": 242,
"id": "6756408c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"f1 score : 0.7763830423355103\n"
]
}
],
"source": [
"print(f\"f1 score : {getF1Score(confusion,1)}\")"
]
},
{
"cell_type": "code",
"execution_count": 86,
"id": "f28f64e9",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"MyModel(\n",
" (bert): BertModel(\n",
" (embeddings): BertEmbeddings(\n",
" (word_embeddings): Embedding(119547, 768, padding_idx=0)\n",
" (position_embeddings): Embedding(512, 768)\n",
" (token_type_embeddings): Embedding(2, 768)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (encoder): BertEncoder(\n",
" (layer): ModuleList(\n",
" (0): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (1): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (2): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (3): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (4): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (5): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (6): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (7): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (8): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (9): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (10): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (11): BertLayer(\n",
" (attention): BertAttention(\n",
" (self): BertSelfAttention(\n",
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (output): BertSelfOutput(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" (intermediate): BertIntermediate(\n",
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
" )\n",
" (output): BertOutput(\n",
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" )\n",
" )\n",
" (pooler): BertPooler(\n",
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
" (activation): Tanh()\n",
" )\n",
" )\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" (lin1): Linear(in_features=768, out_features=768, bias=True)\n",
" (gelu): GELU()\n",
" (lin2): Linear(in_features=768, out_features=1, bias=True)\n",
")"
]
},
"execution_count": 86,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.eval()"
]
},
{
"cell_type": "markdown",
"id": "2da5789b",
"metadata": {},
"source": [
"한번 테스트해보기"
]
},
{
"cell_type": "code",
"execution_count": 51,
"id": "cc727fd9",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"웹 gpu\n",
"긍정적 output : 54.185086488723755 %\n",
"부정적 output : 45.814913511276245 %\n"
]
}
],
"source": [
"sen = input()\n",
"inputs = tokenizer(sen, return_tensors = 'pt', padding='longest')\n",
"output = model(**{k: v.to(device) for k,v in inputs.items() })\n",
"prob = torch.sigmoid(output).item()\n",
"print(\"긍정적 output :\",prob * 100,\"%\")\n",
"print(\"부정적 output :\", (1-prob) * 100,\"%\")"
]
},
{
"cell_type": "markdown",
"id": "2faa8141",
"metadata": {},
"source": [
"```\n",
"5471412\t맘에 들어요~ 0\n",
"```\n",
"라벨이 잘못 붙어있는 것들이 있다. 별점가지고만 긍정, 부정을 매긴 것 같다."
]
},
{
"cell_type": "code",
"execution_count": 40,
"id": "b40f071c",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████| 1875/1875 [00:51<00:00, 36.45batch/s]\n"
]
}
],
"source": [
"model.eval()\n",
"collect_list = []\n",
"with torch.no_grad():\n",
" with tqdm(test_loader, unit=\"batch\") as tepoch:\n",
" for batch_i,batch_l in tepoch:\n",
" batch_inputs = {k: v.cuda(device) for k, v in list(batch_i.items())}\n",
" batch_labels = batch_l.cuda(device)\n",
" output = model(**batch_inputs)\n",
" loss = BCELoss(output, batch_labels.double())\n",
" \n",
" prediction = (output > 0).to(device,dtype=torch.int64)\n",
" correct = (prediction == batch_labels).sum().item()\n",
" accuracy = correct / prediction.size()[0]\n",
" \n",
" collect_list.append({\"loss\":loss.item(),\"accuracy\":accuracy, \"batch_size\":batch_labels.size(0),\n",
" \"predict\":prediction.cpu(),\n",
" \"actual\":batch_labels.cpu()})"
]
},
{
"cell_type": "code",
"execution_count": 52,
"id": "871b72d4",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"average_loss : 0.3305340794156926, average_accuracy : 0.8672333333333333, size :30000\n"
]
}
],
"source": [
"total_loss = 0\n",
"total_accuracy = 0\n",
"total_size = 0\n",
"confusion = torch.zeros((2,2),dtype=torch.long)\n",
"\n",
"for item in collect_list:\n",
" batch_size = item[\"batch_size\"]\n",
" total_loss += batch_size * item[\"loss\"]\n",
" total_accuracy += batch_size * item[\"accuracy\"]\n",
" total_size += batch_size\n",
" confusion += getConfusionMatrix(item[\"predict\"],item[\"actual\"])\n",
"print(f\"\"\"average_loss : {total_loss/total_size}, average_accuracy : {total_accuracy/total_size}, size :{total_size}\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "62333944",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue View Git Blame Copy Permalink