{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "4c31f5ad", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'C:\\\\Users\\\\Monoid\\\\anaconda3\\\\envs\\\\nn\\\\python.exe'" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import sys\n", "sys.executable" ] }, { "cell_type": "code", "execution_count": 2, "id": "2b9e11e7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "load bert tokenizer...\n" ] } ], "source": [ "from transformers import BertTokenizer\n", "print(\"load bert tokenizer...\")\n", "PRETAINED_MODEL_NAME = 'bert-base-multilingual-cased'\n", "tokenizer = BertTokenizer.from_pretrained(PRETAINED_MODEL_NAME)" ] }, { "cell_type": "code", "execution_count": 3, "id": "82bf44a2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "cuda available : True\n", "available device count : 1\n", "device name: NVIDIA GeForce RTX 3070\n" ] } ], "source": [ "import torch\n", "print(\"cuda available :\",torch.cuda.is_available())\n", "print(\"available device count :\",torch.cuda.device_count())\n", "if torch.cuda.is_available():\n", " device_index = torch.cuda.current_device()\n", " print(\"device name:\",torch.cuda.get_device_name(device_index))" ] }, { "cell_type": "code", "execution_count": 4, "id": "38dcf62d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "read train set\n", "100%|██████████████████████████████████████████████████████████████████████| 150000/150000 [00:00<00:00, 208913.67it/s]\n", "read test set\n", "100%|████████████████████████████████████████████████████████████████████████| 50000/50000 [00:00<00:00, 259067.57it/s]\n" ] } ], "source": [ "from ndataset import readNsmcDataAll, make_collate_fn\n", "dataTrain, dataDev, dataTest = readNsmcDataAll()\n", "collate_fn = make_collate_fn(tokenizer)" ] }, { "cell_type": "code", "execution_count": 25, "id": "650c8a19", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertModel: ['cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias']\n", "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" ] } ], "source": [ "from transformers import BertModel\n", "PRETAINED_MODEL_NAME = 'bert-base-multilingual-cased'\n", "bert = BertModel.from_pretrained(PRETAINED_MODEL_NAME)" ] }, { "cell_type": "markdown", "id": "30d69b45", "metadata": {}, "source": [ "BERT 로딩" ] }, { "cell_type": "code", "execution_count": 26, "id": "7583b0d1", "metadata": {}, "outputs": [], "source": [ "import torch.nn as nn\n", "\n", "class MyModel(nn.Module):\n", " def __init__(self,bert):\n", " super().__init__()\n", " self.bert = bert\n", " self.dropout = nn.Dropout(p=0.1)\n", " self.lin1 = nn.Linear(768,768) #[batch_size,768] -> [batch_size,768]\n", " self.gelu = nn.GELU()\n", " self.lin2 = nn.Linear(768,1) #[batch_size,768] -> [batch_size,1]\n", "\n", " def forward(self,**kargs):\n", " emb = self.bert(**kargs)\n", " e1 = self.dropout(emb['pooler_output'])\n", " e2 = self.gelu(self.lin1(e1))\n", " w = self.lin2(e2)\n", " return w.squeeze() #[batch_size]" ] }, { "cell_type": "markdown", "id": "befe62b0", "metadata": {}, "source": [ "모델 선언. 비슷하게 감." ] }, { "cell_type": "code", "execution_count": 27, "id": "36585e76", "metadata": {}, "outputs": [], "source": [ "model = MyModel(bert)" ] }, { "cell_type": "code", "execution_count": 8, "id": "0e36d9e2", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\"\\nclass MyModel(nn.Module):\\n def __init__(self,embeddings):\\n super().__init__()\\n self.embeddings = embeddings\\n self.lin0 = nn.Linear(768,1)\\n \\n def forward(self,**kargs):\\n emb = self.embeddings(kargs['input_ids']) #[batch_size,word,768]\\n e0 = emb * kargs['attention_mask'].unsqueeze(dim=-1)#[batch_size,word,768]\\n e1 = self.lin0(e0)#[batch_size,word,1]\\n w = (e1.sum(axis=1))#[batch_size,1]\\n return w.squeeze() #[batch_size]\\nmodel = MyModel(bert.embeddings)\\nfor param in model.embeddings.parameters():\\n param.requires_grad = False\\n\"" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\"\"\"\n", "class MyModel(nn.Module):\n", " def __init__(self,embeddings):\n", " super().__init__()\n", " self.embeddings = embeddings\n", " self.lin0 = nn.Linear(768,1)\n", " \n", " def forward(self,**kargs):\n", " emb = self.embeddings(kargs['input_ids']) #[batch_size,word,768]\n", " e0 = emb * kargs['attention_mask'].unsqueeze(dim=-1)#[batch_size,word,768]\n", " e1 = self.lin0(e0)#[batch_size,word,1]\n", " w = (e1.sum(axis=1))#[batch_size,1]\n", " return w.squeeze() #[batch_size]\n", "model = MyModel(bert.embeddings)\n", "for param in model.embeddings.parameters():\n", " param.requires_grad = False\n", "\"\"\"" ] }, { "cell_type": "markdown", "id": "7969fead", "metadata": {}, "source": [ "### 학습 과정에서 벌어지는 일" ] }, { "cell_type": "code", "execution_count": 9, "id": "8c2a4bc9", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "MyModel(\n", " (bert): BertModel(\n", " (embeddings): BertEmbeddings(\n", " (word_embeddings): Embedding(119547, 768, padding_idx=0)\n", " (position_embeddings): Embedding(512, 768)\n", " (token_type_embeddings): Embedding(2, 768)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (encoder): BertEncoder(\n", " (layer): ModuleList(\n", " (0): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (1): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (2): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (3): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (4): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (5): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (6): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (7): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (8): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (9): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (10): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (11): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " )\n", " (pooler): BertPooler(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (activation): Tanh()\n", " )\n", " )\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (lin1): Linear(in_features=768, out_features=768, bias=True)\n", " (gelu): GELU()\n", " (lin2): Linear(in_features=768, out_features=1, bias=True)\n", ")" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model.cpu()" ] }, { "cell_type": "code", "execution_count": 10, "id": "def10f6c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'input_ids': tensor([[ 101, 9405, 62200, 14523, 48549, 119, 102, 0],\n", " [ 101, 9294, 12424, 69592, 48549, 119, 102, 0],\n", " [ 101, 9479, 68984, 48549, 119, 102, 0, 0],\n", " [ 101, 9659, 22458, 119192, 12965, 48549, 119, 102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 0],\n", " [1, 1, 1, 1, 1, 1, 1, 0],\n", " [1, 1, 1, 1, 1, 1, 0, 0],\n", " [1, 1, 1, 1, 1, 1, 1, 1]])}" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "inputs = tokenizer([\"사랑해요.\",\"무서워요.\",\"슬퍼요.\",\"재미있어요.\"],\n", " return_tensors = 'pt', padding='longest')\n", "inputs" ] }, { "cell_type": "code", "execution_count": 11, "id": "e027b926", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "torch.Size([4, 768])" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "hidden = bert(**inputs)['pooler_output']\n", "hidden.size()" ] }, { "cell_type": "code", "execution_count": 12, "id": "ae9f8fba", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([-0.0180, -0.0823, -0.0234, -0.0886], grad_fn=)" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "w = model.lin2(model.lin1(hidden)).squeeze()\n", "w" ] }, { "cell_type": "code", "execution_count": 13, "id": "5470c3f8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([0.4955, 0.4794, 0.4942, 0.4779], grad_fn=)" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "torch.sigmoid(w)" ] }, { "cell_type": "code", "execution_count": 14, "id": "b7eb8e67", "metadata": {}, "outputs": [], "source": [ "labels = torch.tensor([1,0,0,1])" ] }, { "cell_type": "code", "execution_count": 15, "id": "7a324ed7", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor(0.6937, dtype=torch.float64,\n", " grad_fn=)" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "nn.BCEWithLogitsLoss()(w,labels.double())" ] }, { "cell_type": "code", "execution_count": 16, "id": "cb54294d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([False, True, True, False])" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "(w > 0).long() == labels" ] }, { "cell_type": "markdown", "id": "596b89bd", "metadata": {}, "source": [ "이런 일이 벌어짐. sigmoid 는 나중에" ] }, { "cell_type": "markdown", "id": "9875ce48", "metadata": {}, "source": [ "### Model Training" ] }, { "cell_type": "code", "execution_count": 28, "id": "769c4290", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "MyModel(\n", " (bert): BertModel(\n", " (embeddings): BertEmbeddings(\n", " (word_embeddings): Embedding(119547, 768, padding_idx=0)\n", " (position_embeddings): Embedding(512, 768)\n", " (token_type_embeddings): Embedding(2, 768)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (encoder): BertEncoder(\n", " (layer): ModuleList(\n", " (0): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (1): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (2): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (3): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (4): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (5): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (6): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (7): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (8): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (9): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (10): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (11): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " )\n", " (pooler): BertPooler(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (activation): Tanh()\n", " )\n", " )\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (lin1): Linear(in_features=768, out_features=768, bias=True)\n", " (gelu): GELU()\n", " (lin2): Linear(in_features=768, out_features=1, bias=True)\n", ")\n" ] } ], "source": [ "device = torch.device('cuda')\n", "model.to(device)\n", "print(model)" ] }, { "cell_type": "code", "execution_count": 29, "id": "b9380dcd", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "device(type='cuda', index=0)" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "bert.device" ] }, { "cell_type": "markdown", "id": "5e82df0e", "metadata": {}, "source": [ "모델을 모두 gpu로 보냄" ] }, { "cell_type": "code", "execution_count": 30, "id": "74c4becc", "metadata": {}, "outputs": [], "source": [ "from torch.utils.data import Dataset, DataLoader\n", "BATCH_SIZE = 16\n", "train_loader = DataLoader(\n", " dataTrain,\n", " batch_size=BATCH_SIZE,\n", " shuffle=True,\n", " collate_fn=collate_fn\n", ")\n", "dev_loader = DataLoader(\n", " dataDev,\n", " batch_size=BATCH_SIZE,\n", " shuffle=True,\n", " collate_fn=collate_fn\n", ")\n", "test_loader = DataLoader(\n", " dataTest,\n", " batch_size=BATCH_SIZE,\n", " shuffle=True,\n", " collate_fn=collate_fn\n", ")" ] }, { "cell_type": "markdown", "id": "4153b2e7", "metadata": {}, "source": [ "데이터 모델 준비" ] }, { "cell_type": "code", "execution_count": 31, "id": "3cd5bf7b", "metadata": {}, "outputs": [], "source": [ "from torch.optim import AdamW\n", "from groupby_index import groupby_index\n", "from tqdm import tqdm" ] }, { "cell_type": "code", "execution_count": 32, "id": "65b5ccde", "metadata": {}, "outputs": [], "source": [ "optimizer = AdamW(model.parameters(), lr=1.0e-5)\n", "BCELoss = nn.BCEWithLogitsLoss()" ] }, { "cell_type": "markdown", "id": "79607e81", "metadata": {}, "source": [ "학습 준비" ] }, { "cell_type": "code", "execution_count": 33, "id": "68183cf4", "metadata": {}, "outputs": [], "source": [ "def freezeParameter(model):\n", " for param in model.bert.parameters():\n", " param.requires_grad = False\n", " for param in model.bert.embeddings.parameters():\n", " param.requires_grad = True" ] }, { "cell_type": "code", "execution_count": 34, "id": "42d1a29f", "metadata": {}, "outputs": [], "source": [ "#freezeParameter(model)" ] }, { "cell_type": "code", "execution_count": 35, "id": "cc8a82a5", "metadata": {}, "outputs": [], "source": [ "TRAIN_EPOCH = 5\n", "\n", "result = []\n", "iteration = 0" ] }, { "cell_type": "code", "execution_count": 36, "id": "4835a0d3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "epoch 0 start:\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Epoch 0: 100%|███████████████████████████████████| 9375/9375 [11:41<00:00, 13.37minibatch/s, accuracy=0.844, loss=5.79]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "epoch 1 start:\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Epoch 1: 100%|████████████████████████████████████| 9375/9375 [11:48<00:00, 13.23minibatch/s, accuracy=0.84, loss=5.37]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "epoch 2 start:\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Epoch 2: 100%|███████████████████████████████████| 9375/9375 [11:44<00:00, 13.31minibatch/s, accuracy=0.879, loss=4.79]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "epoch 3 start:\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Epoch 3: 100%|███████████████████████████████████| 9375/9375 [11:42<00:00, 13.35minibatch/s, accuracy=0.852, loss=5.12]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "epoch 4 start:\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Epoch 4: 8%|██▊ | 717/9375 [00:53<10:48, 13.36minibatch/s, accuracy=0.918, loss=3.09]\n" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_29640/636965671.py\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 17\u001b[0m \u001b[0mbatch_labels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmini_l\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 19\u001b[1;33m \u001b[0moutput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mbatch_inputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 20\u001b[0m \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBCELoss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_labels\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdouble\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 21\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_29640/2430988958.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, **kargs)\u001b[0m\n\u001b[0;32m 11\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 12\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m \u001b[0memb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbert\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 14\u001b[0m \u001b[0me1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0memb\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'pooler_output'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 15\u001b[0m \u001b[0me2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgelu\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlin1\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m 1004\u001b[0m \u001b[0moutput_attentions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moutput_attentions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1005\u001b[0m \u001b[0moutput_hidden_states\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moutput_hidden_states\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1006\u001b[1;33m \u001b[0mreturn_dict\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mreturn_dict\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1007\u001b[0m )\n\u001b[0;32m 1008\u001b[0m \u001b[0msequence_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mencoder_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m 590\u001b[0m \u001b[0mencoder_attention_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 591\u001b[0m \u001b[0mpast_key_value\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 592\u001b[1;33m \u001b[0moutput_attentions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 593\u001b[0m )\n\u001b[0;32m 594\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m 475\u001b[0m \u001b[0mhead_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 476\u001b[0m \u001b[0moutput_attentions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moutput_attentions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 477\u001b[1;33m \u001b[0mpast_key_value\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself_attn_past_key_value\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 478\u001b[0m )\n\u001b[0;32m 479\u001b[0m \u001b[0mattention_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself_attention_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m 407\u001b[0m \u001b[0mencoder_attention_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 408\u001b[0m \u001b[0mpast_key_value\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 409\u001b[1;33m \u001b[0moutput_attentions\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 410\u001b[0m )\n\u001b[0;32m 411\u001b[0m \u001b[0mattention_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutput\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhidden_states\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1102\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1103\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32m~\\anaconda3\\envs\\nn\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m 304\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 305\u001b[0m \u001b[1;31m# Take the dot product between \"query\" and \"key\" to get the raw attention scores.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 306\u001b[1;33m \u001b[0mattention_scores\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquery_layer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey_layer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 307\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 308\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mposition_embedding_type\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"relative_key\"\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mposition_embedding_type\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"relative_key_query\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "model.zero_grad()\n", "\n", "for epoch in range(TRAIN_EPOCH):\n", " model.train()\n", " print(f\"epoch {epoch} start:\")\n", " with tqdm(train_loader, unit=\"minibatch\") as tepoch:\n", " tepoch.set_description(f\"Epoch {epoch}\")\n", " \n", " for batch in groupby_index(tepoch,16):\n", " corrects = 0\n", " totals = 0\n", " losses = 0\n", " \n", " optimizer.zero_grad()\n", " for mini_i,mini_l in batch:\n", " batch_inputs = {k: v.to(device) for k, v in list(mini_i.items())}\n", " batch_labels = mini_l.to(device)\n", " \n", " output = model(**batch_inputs)\n", " loss = BCELoss(output, batch_labels.double())\n", " \n", " prediction = (output > 0).to(device,dtype=torch.int64)\n", " corrects += (prediction == batch_labels).sum().item()\n", " totals += prediction.size()[0]\n", " losses += loss.item()\n", " loss.backward()\n", "\n", " optimizer.step()\n", " accuracy = corrects / totals\n", " result.append({\"iter\":iteration,\"loss\":losses,\"accuracy\":accuracy})\n", " tepoch.set_postfix(loss=losses, accuracy= accuracy)\n", " iteration += 1" ] }, { "cell_type": "code", "execution_count": 18, "id": "81b69931", "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", "import numpy as np\n", "import matplotlib.pyplot as plt" ] }, { "cell_type": "code", "execution_count": 37, "id": "c3a73c68", "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD4CAYAAAAHHSreAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAABPbUlEQVR4nO2dd5gUxdPHv3UBOA7JOQfJICgZRJIkEURFRFSCoIKKGQVUggnDKxhAkR8iCKiABBGRHEWiBEmSM0eQDEe6u37/6J3d2cmzOxtmrz/Pc8/O9vT09NzudnVXVVcRYwwCgUAgEMiJi3QHBAKBQBB9COEgEAgEAhVCOAgEAoFAhRAOAoFAIFAhhINAIBAIVCSE82ZxcXEsKSkpnLcUCAQC15OamsoYY2GdzIdVOCQlJeHq1avhvKVAIBC4HiK6Fu57CrWSQCAQCFQI4SAQCAQCFUI4CAQCgUCFEA4CgUAgUCGEg0AgEAhUCOEgEAgEAhVCOAgEAoFAhSuEw+mrpzF9x/RId0OQWTh+HJg7N9K9EAgiiiuEw8PTHkbnXzqj64yuuHTjUqS7I4h1GjQA2rePdC8EgohC4Uz2k5yczALZIU3DyO89GyISFAlCCHm+byIRliBKIKJUxlhyOO/pipXDO/e8E+kuCDIjQjgIMjGuEA7vNnvX771IbSoIC+J7JsjEuEI4AMDsR2d7j99d8a5+RYHAKYRwEGRiXGFzkJDbHoTdQRAyJJvDzZtAYmJk+yIQQNgcTMmWkC3SXRBkJsTKQZCJcZVwmNZpWqS7IMhMZGREugeCWOD6dWC6+/ZpuUo4tK/o8z0/eP5gBHsiyBSIlYPACd54A+jcGVi5MtI9sYWrhIOcwxcPR7oLglhHrByco2dP4P/+z/51aWlA3brAokX6ddatA+64A0hNNW9vyhSgbVv7/QiGo0f56/nz4b1vkLhWODSb2CzSXRDEOpl95XD1KnDypLr8xAmuKrHDhAlA//7AtWtASopx3YwM4NAhfpySAmzYADz1lH79V18Ftm0DNm/WPn/jhm+AfuIJYP584NgxXm7EgQPG583qp6fz55AcHFw22XCtcBAIQo7LfsyO06ABUKSIurxYMeChhwJrs3VroGhR4zoffACUKQPs3esrMxLUZkK8Rw+gZEnufSZRogTQrZv+NYsWAeXKAT/+aNy2xB9/8PrTZHbRwYP5cxw5wt+77PskhINAoEcgK4dRo4Dt253vixzGgPff57PfULJtm+941ixg4ULf+z/+UNefNw+YPdu4zVWrzO87eDB/PXAAeOstfnz8uP/grsVbb2mrlqQ+paVpl0vMnMlXFQDwzz/89e+/zfurrH/kCFeFffghLztzhr8K4RBaNj2zKdJdEMQy1675jgP5MffrB9So4Vx/tNixA3jnHW7kDBcPPcRn/Ua0awc8+KBz95w4EZg0yff+hx+060lCfMUK4OOP1eelz9FM2D/8sM8eIV0TF8AQ2b49V4VJCLVSeCiXt1ykuyCIZSpV8h0HanNwehBo04arQSSkGbA0Sx40yDcAKTl1ip/7+Wft8x9/zM/fumWtL/L/iZVVgB7//stfP/lE//4//eT/Xlnn22/5tWvX+spSU/mKg8i3CkhP5685cvhfr/f5EnEPI+n46FH+OmeOuu6AAer//blz/u8PHza+X5TiOuGQGOfbsSpiLAkcR9IPA8EN8leu8NcNG3zG0EBZsEBbhSR9/4cP1792507++u232uffe4+/WjUwy39zwfjuf/65//0PHeKD/IkT+tcoZ/Gffaauc/KkT1hIKw29z/HWLe69tGOH/j1PnQJGj+bH48apzytXKidO6Kv75BMPF+A64ZAlPov3eP3x9RHsiSDmCWby0aMHf61bFyhd2one+NBbJWhh9gzSeattOrUqkoSV1F6DBvzPaABV9lGrL5Mn+4SIFXXSE08A1arpn//hB58AkFYgRkyerH+ufHnz66MI1wmH+Lh473H97+pHsCcCx/nyS99yPhqQDz7Vq/PBycgoKh+EZswALl70b2fECJ+BFeBqDyI+i27cGLh8Wd3mhQu+48aNgV279O//wgvqsvff56/LlwObNOx1kjunVcOrPNaU3qArDaJDhwKffmrcntTG2bP8Vet/IHHtGrB4Mf8/EAH792vX+/pr/jp1KvDSS8b3l7j7bvM6mzcDTZrw1Y5kNJewMpGIjzevE00wxsL2lz17duYE3Wd1ZxgKhqFwpD2BA6SnM5aSElwb/Cdm/7rjxxnLyAju3so+AIwdPcrYqVP+5UuXMpaaqr7u6lXGzp71v37iRP9nUj6fvC7A2PTp6nbff9+/TocOjG3ezI/vuEPdjpyLF/3P1avH2H//MXb9uroPFSvq/y+0+gow9sILjJ04wf9P8jrXrzN24YLv/blz2tczxljWrNrntP5mzGAsKcl6/VD/yZ954EDz+rdu6X7tzABwlYVpnJb+XLdyAID7K9wf6S4IlLz/PveJl+vsw8HWrdzvfswY59t++mmgUCH/1ULz5kDt2uq6NWsC+fL5lzGT2aRSTaKcWZ4+Dbz9tv41ZqqgXLnU7efPzw3cTpCayvcslCjBPYskMjKA3Ll97/Pm1b5+2TJrqhqJW7eiK0qu5L4KGNt9JEK4ciCiNkS0m4j2EdEAjfN5iGgWEf1DROuJyECXxnGlcMiRJYd5pcxOejrfTGS0THeSefP46/HjobsHY8BHH/l7g+zezV+XLQuszf37gW++0T4nebssX+5fLhl55cg3bEmYCQflebnBddEibddNIm3DqBWk9pXPY4ae4JX+94Dv8wes7/No3ly998CIdeuAS1GUQ/4dmxkq7diKbDVL8QBGA2gLoAqAx4ioiqLaIABbGGN3AOgG4Auzdl0pHO4uaUE/mNmZMYPPOt98M7z3DdEPAACwdCkwcCDQp4/6fmYDsR6NGwPPPWfsrWPm36+HvE/K/mkNinLh0KoVDzehVUfynrFLoDPXvn21y1ev9h3LP/e6dQO7jxkjR4am3UDRcm2NDHUB7GOMHWCM3QTwM4AHFHWqAFgCAIyxfwGUJqJCRo26UjjIVw4s0EHBrfToAZQqZV5PMjTqzbS6duU/6Oef56/Sbs5ACcfnIKl35M+k9Eyxi7QKsdv/nj3N6xgJBy11SocOfO+AkYA1E7579/I6WvXkwoHIf5dzsJ/f1KnBXS8wI4GINsr+npGdKwZA7i99zFMmZyuAhwCAiOoCKAWguNENXSkc5Hy+9vNIdyG8TJxoTa9vNqOWNhhJnh1yL5pAkO6zYQMfcA8dMvasAYA1a3wePXIOHvQd79jh2yegNeAZCYcjR3wqoOXLtVcHUpsLFtgTMBMm8Gf88kt+rRby/71c3bZ4sb7XU4cOxvc1C1onD3GhRLlPQK4u2rMHePJJvneBMf/QGYJoII0xVlv2N1Z2TmvGoPzhfwQgDxFtAdAPwGYAhjo9U+FAROOJ6DQRbZeV5SWiRUS01/Oax6ydUDFl25RI3Tq6kQaCcK2spPu8+CLXJZcpA1RRqj1l3LgBNGyoPRiWLes7rlaNB03Tw0gIlioFVK3KhVSzZrxvetc/+KBvY5ZVypThrpJ6Bl55n+T/i5YtuZunFnLXVS3++st3bFeFZ6ZWmjyZh+SYOZOHwBa4hWMAZFvoURyA325CxtglxlhPxlhNcJtDAQAHYYCVlcMEAMpv/wAASxhj5cH1WCrreKipX5zvcaBQ6rjdSloa8Pjj/Pinn7Rn53bYuJGroazOrLduNa8jhUKQ/OulnbJGLFnCXxcs4MJl7lxfLJ9ff/XVe+89f+8ZKY6+1mxY/v05aPhbsY98li/tmJaw8j9yGuXKQS/8xb59oe+LwEk2AChPRGWIKAuALgD8DCJElNtzDgB6A1jJGDO07psKB8bYSgCKYCF4AID065sIoKNp9x1GCqOx8cRGrDi0Ity3jzznz/s2Dsm5dk098Fg1YEpqlwsXuGpIMpp26MCFTEoKH5SVAx1gHBv/2jX/gHaAT9Bcvcpn2MpNRVrIk8WsXcsDnGkxeLBvhzLgv4q6dcvnwZWW5h/F87//zPtgByOBZxSyIRi0PKkklCsHveQzixc71x9ByGGMpQF4AcACALsATGOM7SCiPkQkeW9UBrCDiP4F92p6yazdQG0OhRhjKZ6OpQAoqFeRiJ6RjChpdtzWTEiM9/k7T9/pvvysQZM3L/dZV5I/v9oP36o9oXNnPojnycN99iUvFelzS0jgbd92m/paIx11cjJvU45c5RJqLxS5XaJ9eyBnTv5eLkAA/eB0gWL0fddKouMEkg1JC6sRRoVwcB2MsXmMsQqMsXKMsQ88ZWMYY2M8x2sYY+UZY5UYYw8xxnRmBj4SwtDpsQDGAkBycrJjCnB5AL44cr1d3TmMUiVeugR89RWQLZv2+d9+898XMXky8L//+VRACQmB5SpgTL2ykHvrvPaaeRvrFXG0XnlFu56R6ujoUf9QylNCbK8Ktb3n33/teQlFj+ulwAUEKhxOEVERxlgKERUBcNrJTlkhIc7XdSEcLMAYT6f43XfG9eRhkaVB1Wo4Zy30vHLs7IwFgHr1/N/rpYTUMqRKM+ZQzdYjxfXrQJcuke6FIEYJdFSdA6C757g7gF8N6oYEuVopnqIwoNXq1XxwDXW2LjmjRumfy5bNXDAA/oLg2jVu9JXK5DNhIm2bhxK5EfvFF/mARgQU1NVEBo5eWGrhtBBdOB2l1g0ULhzpHtjGiivrTwDWAKhIRMeIqBe4z2xLItoLoKXnfViJerWS5EMeaFgHqygHXz3MUixKKFcJH37oK1Oqdtau5Z5MVvv31VfWVTn799u3Rch3TsvREg7BJKoRBEdmVG+tWxfpHtjGVK3EGHtM51QLh/tiC/nKISqFg0So9c63bgFZs/LjhITgVECA2oialuZ7hnbt/M/16cNXRkarI6X6qHdva/24/XZr9aygZYi95x7n2hfYQxmgMDNgtFcnSoniUdUY+cpBnuMhaghGlfHXXzz1oxVSU7nXTUqKM1EflYO53ICrRBIKvxpoFe3aFgTuQr4pzyp2vxOhDOYoEWz4mBgk5N5KoUIuHFJvGXjoOEVqKpCUFNigf/UqkD27/7XXr/Pww1oDeqNG/NXKF3baNL7h68YNvnIIFqM0jXo8/7z+uWhIqh4NfYhV9DzfjChuGNJHTTjCdCtdrQXuXTkkJSZ5j79YZxp9NjguXuS++nohD7SQBMHevTyx+fjx/ueTkoAHlIETA0BS+TDmjHAIVi2lJBpWDpktOGM4CWS1qjfBekxHg211f4Zyo6UdKlcO/NoYxbXCITkxOXw3k3bOTppk/RrpByDtC9Aywv3+u71+bNmiLpMGvitXzOPyWMFqonmrRINwkIIMCuxTpozx+aQk4/NWefZZ9QRKwupq3YqgSkkBvtCYTDZpwgNBCry4Vjg0LNEw0l2whqTScMKd8s471WWScLAraPQIZvalRTSodORhNwT26NzZ+HyyQ5O0d9/VVlF99pn1lYNSOFSs6Dvu2ZPvji9cmMcJ06K+yEkvx7XCoc3tDqU6NGL/fv7FNAs9bYS0Yvj1V18wPDmHDnEdrBSD/3//850j4qEM4uP9UxLKMdL3B8KTTzrb3jPPmNcRRC9SqBE9kpICT4YkwZj+vpdXX9WfWJ054zv+5ht1PXmK1fHjfb/F/PnFhMECrhUOIY3GevMmD143dSr/4k6ezMsZ4wHTVq70r3/0KN99e/w4P960SfsL/eOP6rJZs/y9MZSD6WOP8dm31lLYDTi1ohFEBrPQJjlzGqtbp03T3lOijGsVLL17q39zGRnAihXq3yvA9wSNGGGt7a5d9e0hMYxrhYNyb8Oaow7qC196iSeMP3TIczPPva5d4/kFmjThSVEkSpYEihThK4CSJYFatayHZLZqANbTxwoEoUTaQ6NHfLx2AEiJ5s2BuzXS+jZpYr0Peg4Fksoyf35tZ4yMDL6fpXFj9bnERF98rurV9e9dtSrfuPnEE9b7GyO4VjiQIvnRt39rhE749Vd7ydhXrQI++cTnuy2lkJSEg3yXsVFoZAA4rRNuasYMoGhR33uzHM9Oh5EWWGfQIO3Q2nLdtpYdKLMgBWk0WsXnzu3/XgoMaceDTG63GjbMdyy1oXd/K/auq1fNd/kDwH33Wd97FCO4VzgovhAZTOOL0LEj8PTT1hpMT+ezjDffVH+pJOEg97wxm/HruZV26mSe6lEQHSQlAVmyqMvlA1u5cs7es2pVZ9uzQ6dO9urnyGF8/vXXfYL0u+94SlXJu8nO/og8ebi6df16/5VMgQJcnTRvnq9Mvov/0UfN286eXfszlpCPM1qh6tu25a9SwMeWLYGlS33nhw7lE0IX4tpNcEo0hYNVTp8GChWSNaYjHOSJ7c2Ew+HDgfdHEB2ULu0bHMqWBQ4c4Md16/K4UgBXSfzyi7X2WrfWzzctsX175AIFTp/u7L0//dR3/NRT/ucqVNC/LmdO/98akS+oonzgjYvzd+AAuKagSBHuZRWIJ1WfPv65teVorUTy5zdeBQ0ZYr8PUYJrVw5KpmybAmZnqXr+vC8WvmRbkFC6nyrPA9YD2QncyZIl2t5lADB7Ng+ktmQJ94iRBIURc+ZYFyLbtjkbinvsWB4GZcIE/l6u1owUtWppl//zD7Bnj/n1enm7CxfmauFAbXSff84N2FLQR7mwlGsOBg4MrH0XETPCAQD2n99vvXLXrvwHuF/jGulLIH0xVmikIXUwq50gyqhdmxtSifgsFPBPY5o7N189NG/OZ6/KXBNatG9vroaRqFbNp6YINLS5XFVSqxZ/pgYNAmvLCs89Z75hTkm5cupIutWr+1bxH3+svqZDB/76wQf67TZoEPj+i6xZuQG7eXP1OSlfef36fNMeoK+2fu89c2N+lBNTwsHWymH+fP5686Z6WSitHIzaGzWK60LF7tvYQ/6Dz56dfw+6d/eVWQ0ZUbOmtXp6ubABro4pW9ZaO3Ik92vAZ/+SBqu8ee23Z8bo0T61m1X27eP7E7RgDHjjDXV55cr83F132e+jHaR4TvKYS9Wq8XuvWQOUKsWPtTyhAL6idDraQJhxtXCY0dnf0GMpdPd///lnBIuLUwsBaTVhFvrhwgX93ZbRhtzLI1rQ+vFHA2aTDCPhIFdDzJ1r7X4ZGcDw4Xz2bUZSEvfPb9lSv84ff/i/l1w1S5XikxqjKLpaKJ932TJ928nSpdxQPHasvXtEG5UqcRXTtGmR7knEcLVwaFDcf5lsaWNcgQI+VQG/SL9uNMQFcgqzMAiRwK2ugVrfGUkHLo/bX6yYseFVIj0dGDCAz76VMOZ/v65duX++kSdOmzZ8cAO4cVV+/fPPAyVKmPdJzq5d/oNk06ZAq1badZs14w4eVr0EoxUivt9J7qiSyXC1cFAKg4CS/nz7LbB8ufa5WMpY5UTEVqdxyjNGLwNcOJH2q5Qq5V++ZQuP6muE1iRE63+zdq3Pk0bu/aMVQqV6deDUKe3wJfIQ2EZJj6S9OuXLA488ol9PEJNE4YgRZqxuoXc7RsKhZUuuj7aqBnEKp4SDU8HfgqF+fT5j//RTPsuWchYkJWlHLv3gA+Ctt/ix1RVq3ry+z9HM/x4wNmZ/+CG3p7Rt6x+gTo5yA9uCBT6PJ0HM42rhoNwlnZ7Bf2TX065j0JJByCTDvjWMhEPVqtx1L5DBukYN66FC5GjFwgmkzUceUc/WI0G2bD5df7Vq5vUHDfIJB7OdvNWrczuYnhAMJF+FFVdM5efTqpW+OkkQc7harVQoh78+cMnBJQCA8ZvHY+Ram8npYx0j4WDV+0Zuq5EIJpGOnnCwasxcvJj7szsdmTbcmK0cJk3ihl69/QmhSmYUqc14gqjA1cIBAMrm8bn5PTuX+x6nZYg9CCqMhIPVePlNm6rLAh2YlIZWgKs5AOMEMnIVSosWfO+A1f4rkdKxhptXX/V/byTcGOPP2KyZfp2HH+avderoq5gCQQiHTI2r1UoAkBDn/wgfrvoQObLobDa6cSMMPYpSjISD1UFAa1OP2cauli2BhQut3UMa5JV1GzUCVq/mx5cuGbelJXT06skJ10CoJUytxAAyom7d0KwehHDI1Lh+5aAUDsMWvoUcpy/4V5KSgmTmeEdWVg7yuDVKypVTe6x07eofulwLaYBZt46rR6RZstYgLqm3iPzzQHz5Jd+d+vXXxvcyY9Gi4K6PBJEaoEeNEsIhk+N64aA0Sk+eCTzVYQji5WpcyWvDqXy3bsTKyqFZM6BKFe06w4ap1TdTpnBffiOka+rW5THx5SGu9YQDwEMkS23nywfMnAn07es7rzRCS2EV9ChfHrj3Xn6cK5dx3UhGRrWLk2okOW634wiCxvXC4Wa6fwC8B/7lr3Faq+zPPw95f0JK+fKBX6slHHr35q9WdPZE/qoLpd4c4CszpdFUb/aptXJQqpWk+FVyv3yArwSVeRamTTMOhb55M389e5Zn61Ny9ixXWR0+rJ2cJlIYqYvOnvXPIigQOIjrbQ5K4SChOSS5fU9DtWrA3r3a56pW1U5MI6HlkSTtlLVqD5APVFrpI0uWVHveGAke5X2V7/WEg1bmsaxZeUROOV99BfTrx1cKkhuoXlwhqTxUM3Etvv6a5ygIFKdiJL3/vi+xlUDgwfUrhy/aaOdW1lw5uB2jQdwo1aHetdJgb3XlIG2Keu45fbdKSThI7qhGfdYTDtKr1Jbd3d2S+lAv5Ha00Lcv8P332ufCqe9/6y3gs8/Cdz+BK3C9cHig0gN+75nnN6USDsuWhadDocRowBg3juv1jVi6lKtehg/nBmJl3go5XyiELhHQsCEwa5Z6INm7l+cLAHxtSgLHzspBWS6tHKzuw5DYs4enfBUGVUEmgYjaENFuItpHRAM0zuciot+IaCsR7SCinmZtul44AEDR29SzWJVw0IrPHg1o7R3QwyhqZ3IycP/9vvd586rbbtaMq14GDOCCRDmQA1wNAwDduvlfW6cOf+3YUZ3i8fbbeb4AQJ0LQykcjAZsaZ+DVGfAAP9yqxQvHhq7wf3320tvGQzSTmSjcN4CAQAiigcwGkBbAFUAPEZESs+S5wHsZIzVANAUwGdEZJAfNUaEw75++7zHkkxwjVrpxx+t123enG/80kNuEzh71ny1JOny5fsX+vTh7cjj6jBmPZGLcoBXCgPpnomJ/ucYU9sW3nqLl0dL0MDffgOuXQvPve68kz97NBnHBdFKXQD7GGMHGGM3AfwM4AFFHQbgNuLRSnMAOAfAcLdwlPzqgiMpUe2iWuVMBDoycyaPhCl3uTSDSO0JZMQPP/hcPEeP5slI8uWz31eAG5UvXwZefFH7/J9/8nDNdli6lKfRlAy7ypVDly48DeagQeZqJacIVXgJgSB8JBDRRtn7sYwxKc5MMQByF7xjAJTpCUcBmAPgBIDbADzKGDMM6hUTKwct1nzncINWUjw++KD18NGSyicpifv/W6VoUZ/6qHhx4LHHfCoIu4NgcjKPIqq3/6NRI5+7q1UqVeLqID17RmIi8MknfGUinZOex+lBXNgcBLFDGmOstuxPHoBM64uu/DG1BrAFQFEANQGMIqKcRjeMiZWDHBaq8cDpgeabb4Bbt7ibpTIqZ/78PGOdHHn2ulD1yUmkgd7MW+nQId8mRSvXCAQCJccAyDM4FQdfIcjpCeAjxnMp7yOigwAqAdD1pY7ZlYPjOD2rTU72uZ8qhUPp0ur68oxUeoNoT48Dgh21Vqiw6iZbqpR65eKUcMiRg4f/njjRmfYEguhkA4DyRFTGY2TuAq5CknMEQAsAIKJCACoCMEz6HXMrB1ciFzydO5snatcTDiVLRo9+XcsTyowyZXjeAqc8guLjeSY2gSCGYYylEdELABYAiAcwnjG2g4j6eM6PAfAegAlEtA1cDfUmY+w/3UYRg8IhJEPjlCk8XMSVK861KR/Y5SsHK4OpHfXLrl3ccyncBCIcpk3jKVulLGoCgcASjLF5AOYpysbIjk8AsJWpSaiVzOjVi0cfNcvWZVcVoiccrHgu2REOlSpFJm+BtO+hUyfr1+TJw436gshhlFNakKkISjgQ0Sue3XbbiegnIgrTDiF9HDdId+9uWuVmuTL6woMx4J13jBuQxyOyIxyimUqVeD87dox0TwR2WLHCHd8vQcgJWDgQUTEALwKozRirBq7r6uJUx6IOyaPmhx9Up45fVkTGtJLmMlxqJYFAIAiAYNVKCQCSiCgBQHao3afcjzIInJQ8XhbvhyktHU8/bd6uPCSEQq2UWvcu42uFcBAIBCEmYOHAGDsO4P/AXaRSAFxkjC1U1iOiZ4hoIxFtTEsz3K0dNGXPAcm3gmjg6695TH850kCsFz7aCspl+s6d/uEpFCuHh+setNaeEA4CgSBEBKNWygMev6MM+K67ZCJSbfVljI2VdvUlhDhGzlKL7uyn9eK4lSnDwz7UU+48B88FUaSILwNZMANz5cr+7wcN8h3HxeH0rfNYWRI43r8P8NFHQP36/vUHD+ab58yisAoEAkGABKNWuhfAQcbYGcbYLQAzATR0plv2yXcVKHXRvF6PB4BCbwCj6xhUWrvW5+EjCYH27YETJ0ITlbNhQ+B///O7X5OngJSXewFvvgmsWeNfv3Fj4MIF/9WHQJMxG8fg0IVDke6GQOA6ghEORwDUJ6Lsnkh/LQDYjNLmHC1N9o1JZGjlvFHmIwbseWyYVbXSlp3EOwJLXL5xGX1/74tmE5tFuisCgesIxuawDsAvADYB2OZpy4KbTmj4aYa1etVO89c/bveVsYoVVPV0h3PJEO0XYM8B1z9ZoDrSTnIqsEmGJ+jk2dQIbAIUCFxOUNNUxtgQxlglxlg1xtiTjLEbTnUsVORP5a+/V/SVHbx0hB/IZvgnLnPHq9NXT/s3EBcH3LjhlynNVDTYWYWIlYNAIIgCMt1IpLVJrkmD3fi2FoB77/WWSbPNk1dOqS/IksX5QbxbN+Cpp3iyd4FAIIgwmU44aNkcjucC+rSHn5vq5ZuXPUcOqIwqVjSvk5QEfPcdD9ctiEpOXjmJ3f/tjnQ3BIKwEHOB98zQEg4SP237CQWTC6JFWYNUnIHw5JNA+fL8LxJB8ASOUOSzIgAANkSElxDEPkI4yOg6sysA/uN//x7gjylAaoWygd2ocWOgalV+TAQ0aMCPbawMmIhxIxAIIoT7hcPGjUAdo00L/lgdbueXB2gogJltwaoHMEivXGn7kovXLyJXtlz27yUwhMROcoHANu63OXz5pa3q8pVD0+5Ag16B37p7R+nIuvBIy0jDf6nqHBvrj69H7o9z45edvwTeIYEmYgUmENjH/cLBLM+CArm30ooywNoS6jqz/52tKjt37Zxq4P7Nsz1iTo0kVf1PVn+CH7f9qCp/Zf4rKPBpAVy+cdlbtillE+qN4yE7lhxY4p3pqgL6CQQCQZhwv3CQ50KwgJXh9sGp6oQznad3xiPTH8GRi0dAwwgf//kxzmcHbhsI/F+rHKr6by5+E4/PfFxV/ssuLmB83lDAkOVDZP0TAkEgEEQedwuH9HS+Ic0GgSYDOniBR0o9evEoAODtZW8DAK5kBTIs/heXHFiCk1dOqsoT43wutOFUgTDGsO7YOqF2EQgEKtwtHBISgFmzTKvd19V3bOStZMSB8zx4U+/fegPwH8SlMA1m3DvJt8lOag8AEuJ8fgEMzDR8BmMMt9KDiU3OmfzPZNT/rj6m75wedFsCgSC2cLdwMGFqVaBZd+APWeikYOfIRy4eCbIFTrsf23mP5cLBCuM3j0eW97Pg8IXDQfVh91m+oWvP2T1BtSMQCGKPmBYOaXHA8jL+ZdOqBtlmBk/6I7cNBKKWuZHmU4dZFQ6MMXy36TuM3zIegG9wDxahVhIIBErcv8/BgHQN7czmIsG1KQkHOYEYkeXX+KmVFAP1qsOrsPXUVrxQ9wUsP7Tcq9bSqmsXSX0Vq0bwWH0ugSAcxLRw0LIvBGJzkAsEyb4gH5i19i3YIZ7081HfM+EeAMALdV/AlZtX/M4FO/jFEV84ipWDQCBQEtNqJaeEQ93/WU/HeerKKdAwezfR28FrNmgHvXLw3NeqQd1tCKEnEAROphAO/Rv295YFMlxsPrnZct2JW60lspYPXHLvJKMBTSlEgl05RJtaaeaumcj1US5cT7se6a4IBJmeTCEc6hevjxbdgJ+qBb7PwSpabqgXr19Er1/943TIB2T5oH/q6ilsOLHB0r2cWjlEywy7/6L+uHTjEo5fOu5Ie9Ei9AQCN5IpbA7xFI+lZYGlAQZY1UJv4FHO7g+cP4ARa0Z4PYy81+sMyL/v/d17/Pm6z037kJaRZujtxBgDA/PaF/z6mknSkQohIRDYJ7ZWDr/9Bkydij871gLgEw6NSjbCyNYjUTp36ZB3QTnglvuyHEZvGK2qZ0XP//P2nw3bfmfZO0h8LxGpt1J122g0vhHi343XPQ84N3jK3XODwan+RMuKSCBwI7ElHMqVAzp3RoNmTwLwCQcC4eX6L2PWo+a7qYPh5JWTljeU+amVApzBbzm5BQD8gvgpWXNsje45J9VKSw8uRbYPsmHV4VUBt5FZVjICgRuILbWSJ69zfDYeJfVqFl6cznhwPrnLaCgoPqK4915Os/30dsfb9LqyOjBTX3JgCQBgxeEVaFyqcVBtOTXjF+okgSBw3Lty0Aq4J+n7e/QABg/Gu03424LJBQGEdrCYtHWSbcHw3O/PWa5b/ZvqjietkWbqTriyOtE3kZRHIIge3CscevZUl3lWDsiSBRg2DKlZwtedbrO72b7mm43fOJKwPtBBNRTeStGk54+mvggEoYSI2hDRbiLaR0QDNM73J6Itnr/tRJRORHmN2nSvcJg7V11mMkhG42CRlpGGn7b/5Fh7Z1PPWq7r5D4HLXvB4gOLkZ5hX80m1EECgXWIKB7AaABtAVQB8BgRVZHXYYx9yhiryRirCWAggBWMsXNG7bpXOGTRWBbEue9x4igOF29cdKStRfsXIf+n+TF/33xL9ZUrjlaTWqHU56WC6oM0sC85sAQtJ7XEB6s+sHyt0wZpIWQEmYS6APYxxg4wxm4C+BnAAwb1HwNgOiN132gqoSUczFYOUThYaO0/0MNs8Fx7bC0AYPWR1bp1ft/zO9YcXYM1R31eTNKKatGBRQGHJFcKmhOXTwAA9p7ba3rttB3TUPrz0iEz5gsEMUACEW2U/T0jO1cMwFHZ+2OeMhVElB1AGwAzzG7oXuGQmKguc6FB0xFDrkdomOWePnThEO7/6X40HN8QDcc3tO2tlHorFc0nNseuM7t069hV3U3+ZzIe/eVRHL542NAlNxCiUY0Y7Xyx9gv0ntPbvKIg3KQxxmrL/sbKzmkNInpf/vYAVpuplAA3CwctQhyoLhTYWTlcunHJ8LzXhsAYes/pjfdXvu93XrlZzq630rKDy7Ds0DK8tvA13XvbgTGGJ2c96Xvv+T4rP6e0jDTHQmoIjHl5wcv4bvN3ke6GwB7HAJSQvS8O4IRO3S6woFIC3CwcKlZUl2W4L7qoHeHQZUYXzXJpUJXaymAZ+G7zd3hn2TuG7dn1VjJbmcjP6Q30WnW973Xqvjz/ZRQfWRznrplOdvyYu4c7LVy5eQWnr562da1A4CI2AChPRGWIKAu4AJijrEREuQA0AfCrlUbdKxy0MBnkKuSrYHg+Euw7ty/oNqRB1crgLUfPW2nwssHYd24fWvzQwm9QNVodSPdec2wNcg7Pie6zu1t/AA9KwSIhxZu6cP2CrfbkiZHGbx5vUDMytJrUynZ4d4FACWMsDcALABYA2AVgGmNsBxH1IaI+sqoPAljIGLtqpV33CgctXb2JcEjOkhyizgRO68mtg25DGkylwfvj1R9r1lMO7q8ufJVfr/i/vbfyPZT/qjyWHlyK0evVcaGMVgML9y/E5Zs+24GRTUXZjp56S64uCxQ7K7RwsejAIkv1jl06hhFrRoS4N9HBxC0Tsf74+kh3w3UwxuYxxiowxsoxxj7wlI1hjI2R1ZnAGNNWP2jg3vAZAQiHWEUaNEM9ABoN9E64oSoH/7OpZ7EpZZPtFZEW0SgcrNLx5474O+VvdKzUEWXzOBhaOArp8WsPAAAbkjl/y9GEe38xWigGl9H3jcaguwf5la3oscIbTiNWYGDIYBl4Y/EbhvV0M84ZDLpa1zgWNVVpc1C8b/djO7Sa3Mqb/MeplcObi97Eg1MfDLitcCOp0wLZUCgQBIp7Vw5aA0XWrH5vn6ujjl10T6l70Ob2Nvhh6w+h6lnYYYxZGjj0ZvfBenGtPrIas/61H/FWT60kle/6j7vM3kq/xcuDEEryZ//kr08AADvP7ESVAlX0LokavGpDF7pqC9yLO4XD2rXAhQv+ZTNmAKWC293rVj768yPsOWctVLgWVl1Z9YTL3d/fHfC9jfohJTGSNsfZEWLHLh3ze6+lVmo+sTlOvn7SbjcjhghpLggn7lMr3boFNGgArFcYrR56yHITVn9k+bPnt9OziDFqwygs3L/QtN7XG77WLDeakQ9bMUxdXzZIHzh/wEIPtVHeVykcpBDrequiJQeWYFPKJs1zJUaW8HsfH6cO134rg69Ijl065qjK5nradWw7tc2x9qJxf44g9nGfcNDay/Dkk+oyB6iYT2MvhYv5cv2XmuWMMUsDkKTWyGAZeG3Bazh84TDKfVnO0r1PXz2N1FupuHzjMtr/1B5HLx5V1VEO0NLKwatuAsOJyyew4/QOAMC9k+5FrbG1LN1fb0Jw7NIxlBhZAoOXDbbUDuALDaJHj9k9cMeYO3D+2nnLbVrBTK108PxB/HnkT0fvKWfo8qHC9TYT4U61kpLcuW1VL5RcyFI9N3u42GHM32PwUv2XDOtksAw8NuMxADwD3ZKDS/DnUWsD0aELh1DmizIAgMr5K2PXf7uQP3t+jGk3xq+eXAgAvtm+XK1UbAQPGfNaA/UubQmtzXJ6ObRPXuFqpfn75+ODFtaCBBYbUQz7+u1DubzagnHl4ZUA+I70PEl5LLVphFVbS9kvuSdTqDx9tFaRgtglqNGPiHIT0S9E9C8R7SKiBk51zBbVqtmq/m6zdy3ViyUDoJld4fO1nxueP5t61jvoJsbzuFanrpyydO/DFw57jyUjM4FUg7gkBKQwIV6bg2dF8eGfH3rrfrbmM817nbl6Bvk+yacq1xP0ge6hMApQmHIlhbcdouRMEsVHFEfrya2RnpGuyjcuEARLsFPjLwDMZ4xVAlADfHdeaFH+iOPigKefttVE1oSs5pUQWwZAM536lZtXLLeVJZ5HxD188bBJTR5YT2vm+/2W71F0RFHNa/ov6g+ArzgAHltJasuIKzevYOTakZrnUq6koOhnRbH3rC9KLBH5hRxxGqe+P3LBdeH6BdAwwm+7f8Pxy8excP9CfLX+K++qTiBwioCFAxHlBHAPgO8AgDF2kzF2waF+6aO0OdSsGVA01r+e+gtTO03VPJc1nguP7InZbbcbrUgDrB7SLF0P+SzYrrrtr6N/2ar/55E/AzLC9vujH4b/OVzz3C87f0HKlRS/kBoAQiocrHL1pjqaAWMMC/cv9Ps/EBF2ntkJAH7PaWYDETgDYwyrDq/KNA4CwawcygI4A+B7ItpMROOISBWfgoiekWKQp6UZD1CWUH4wAX5QDUo0QOeqnTXP5U3i2fN632UeurhDxQ4B3T/cmOVKkDaaadFtVje/89KM3ipvLX3LVn0A2H56u636O8/sxIQtE3TPS6owyR4A8Jm9Mmz5icsnQMPIG7RPD7mw/GLtFzh4/qBhHSO0wp38vP1ntJ7cGmM2jvFbeUlCLJZUntHE9bTrWHpwKRhjGLxssJ/6cMq2Kbhnwj2Y9M+kCPYwfAQjHBIA3AXgG8bYnQCuAlDlLmWMjZVikCckOGD/DoPUlgaSGoVqmNaVBEm0Y6ZWmrpDexUFAJP+mYQ5u1VBHkPKHWPusFX/rm/vMjy/5eQWzXK5BxYA/H3ibwDAt39/a9iepDI6f+08Xl7wMlr80MJOd/3Q8mqSVHZyQUwgzVAp4Z7JxvLM+cU/XkSLH1pg2o5peG/le3hk+iPec/vP7fd7jXWCEQ7HABxjjK3zvP8FXFiEFuUXMwQzKMm/PpYINstaJNUuVriRfiOg66wYpG+k3dANFy79X7VSvVq1OVy5pW/vISK/vimDLALA0kNLDdtfeXgl/kv9z1JfJM5fO297hRgLSGo76f+ltaKOxoySoSBg4cAYOwngKBFJmwFaANjpSK+Mb+xoc9UKVsMbDf1jEkkulNE+INoh2E1ekfxfhOrHKDdIG93jvh/v0/SA8mtLQxBYVf1oCSZNgaAQFBJ6GwElmkxoguYTm+uen7hlIr5a95VfWbERxbzux6q+xfDgqAzyqLT5ZCaC1fP0AzDFk2DiAICewXfJBIeFw7a+fCfruM3jvLND787cGMppHOyzXLyunhmHi1AKJqVBWmvgW3pQf2buhIrFLNqtN18HSJXYyQzp2m2n9XdsS5FQ+9Xr5y27lnbNUvuxhtEmwlhWp2kRlCsrY2yLx55wB2OsI2PM2S2h2jf1f1+njiPNyndDSysHMw8fNxHsymHwcus7iMON1m5rK8gN0noCyGr2OadnlVpCiogsG6QzWAY++vMjTXVX0H3LRINkLK+SzHDfFmDlF/OLLxxp9rfHfvMeSysHKzPW7jW0M54Z7eCNBLvP7o50F0LGoKWDzCvpoDRISzpnaWKgFwLDaBOckyiFgNXcHb/v+R0DlwxEno+1d2inXE6xtbdFzrW0a6p85LGGlgDMbGol9wuHrNY2tJmRL3s+3FPqHgDAAxUfAAAUyF7Ar06urLlU1zUt3VSzvXbl2znSL6cIxpsm2vn3v38Duo6IUP6r8gD4YDBr1ywMXDIQgM+VVm/m2G12N9xI8xnBb6bfVO3nsJybW8NeoWVzAGSurCbGbiPXZAAoOqKoqYeXHoX+rxCSP4y+rIqhIDOtkpS4Tzi8/nrIml74xEKcfeMshjUbhpOvnUSR24r4nbez+SszL0fDzcYTG4Nu4/jl43homvXIvoC/2vHSjUtoNL6Rd+UB8O8AYww0jDBw8UDddow8reRCgDHm/V7J7xMoe8/tNa+kQayvGgDx+wXcKBy+/z5kTWdNyIq8SXkRR3EolEMdnE+5rHy53ssAgDK51V4dTs043BI23I3IP6Ob6TftXw+mGkTOXD3j17400/9o9UcAgEFLBmHU+lFYcWiFt96P237UbFurn1J7UvymQJBvMFR6KQk4Rr/fzLKacJ9wiBC97uyFxU8u9isb0ZonfT/w0gEMaTLE75xTHjaZ5YsYCQzTo1rY/3Ar/ZbK0C9v8+KNi6p7DP9zOPr90Q9NJzbVbXfKP1Ow4cQG3g/ZhERaiQTLh6t8AQxfnP9i0O0Fy59H/vSGYTdj55mdoGHk3awoCB1COFhkXIdxuLPInX7hkOU/3KFNh/rVD1Q4FL3NPxidXlhoQfCcvnpa95wV42PeT/Ki5rc1/crkg3fl0ZWx79w+S30Z+/dYbD25FQDwxKwn/Hak+60iglB3SHaIaFILbTm5BY2/b4xq31iLrPzbbu44orej/8L1C1i0f5Fj/dPzGssMCOEQACt6rMCKHisM6wT6Iy6du7Tf+7a3t0X94vVNr0uMSwzofgJtpJWD2UCgFDDKz73ZxGaW7vfs3GdVggYA/k7525tzwnJSJp2EPA9OfdBSX4LhbOpZQ6GrZMjyIeaVZHg3qen8HzpN64RWk1vhbOpZ07bWHF2j66r8/RZ99XVmWc0L4WDCv8//i+19/YPA3VPqHq9nkx7ZErIFdD/l3orOVTtjVc9VuKOQcawhKeWlwBkCnR0qV4xyG0QgzN833+99MCuH+fvmY+/Zvdh8cnNQfTIi/6f5Uej/tJNp/XnkT/yx94+g2jfz0pIM9WbeWgDQcHxDtJzUUvPc1lN8Fee3Q9pz76u3rmYKASGEgwkV81dE1YJVbV/XpFQT7/GJV62HVJbrsNkQhioFqiAhLgENikcmj1JmJdAkQMp8EvKd6cEOKAwsaFtWhVEVbO3ROHnlpGPpTht/3xj3/XifI20xMNxKv4VSn5fCjJ0zvOXK8Be613s+C7PQI1qMXDsSn6z+xPZ1bkMIBweZ0Zl/SRuVaAQiwuGXDyPltRSVS6ySduXbIWfWnAACD3MhXS9whoMXDqLv3L62r5u3d57uOaMQFlYJ94y1yGdFUPizwravM9uRf+3WNbyz9B3b0X7lK7pz187hyMUjeG7ec77zFoV6sELWKIpxrCCEg4NIm+akL3DJXCVROIf6hzWvq28AYUMY5nadi90v7Mbfz/yt+6My+rKv770er9Z/NZiuCzQY8/cYywZlKxgFv7OCfJ9DOAnEzTfhPeOwbZ+v/Rzvr3rfVpuNxjfyZgn02ySoERzPdOWgcV4vXM6o9aNAw0gVQsaqDcitCOHgIEZfSGlPBAC0Ld9Wdb5wjsK4q8hduiuH5Cz6O1LrFKuTaTwowo1dNYhRRj0rdiGzeF5uixQ8f998LNy/UFX+6+5fDa+bsGWCSjDLd6DvOLMDxy4dA+D/u1OuHCZsmYCJWyaq2tf6Py7Yt0BVdurqKfT7o5+qHODRbuuOq6t5bv3x9fhg5Qea59yCEA4hQMtoNrIN10VrheCQ81mrzzTL3232bvAdE4Qco1wgl25cMr3eaFPai/NfDDqAYqD0X9jf9jWzds1C2ylt0Xpya5WdY93xdTpXcXr+2hO1x9YGwPNpKD2wFuxfgNr/q626Trly6PlrT2/UWTlrj61VlWlNzIwCL646sspvd/63G7/1eknVG1cPby97GwC3a7hxhSGEQwjQW0FMenAS/n7Gt3lHK8Vom9vbaF6bI0sO43u68MsXiwSadEji1YX66sG5e+Z6Yz+Fm/9b83+2r5GHIyn1eSnb10sRZc0CBGp5FBn9HjalbEKTCU1U5VaTMwHq3/jWk1vR5/c+6Da7m1/5jJ0zUGtsLUzZNsVy29GCA3k7I8hC9XI1mnnijie8xzfevuFoxjkRCyZzsP+8tRSV4f4+dPy5o6mqKBzIVw795mmrg1Iua4cesfM/U6adlVxn5+2dh6HLh3rLpaCQu87sstx2tODulUNLbR/lSGNlBpIlPos3b4RVRrQaoXtOrBwEcgJRP03dHrgHTigFgx17mvTbG71+NEZtGKVZRykE5u6ZG3jnwO0a8nhVw1YMC6q9aMHdwiFG2fLsFk3bwysNXtG9RqwcBHICCWPeZUaXEPQk9Jy9dhbjN48H4IucPGKt/kRKSfuf2gd1/56/9kTv33oH1UawEFEbItpNRPuIaIBOnaZEtIWIdhCRcYgHuF2tFGU4NXuvUbgGahSuEZF7C2KDd1eGz4HBifDhQHARYnvN6YVsCdksq920CMVvKByTNiKKBzAaQEsAxwBsIKI5jLGdsjq5AXwNoA1j7AgRFTRr110rhxWmwi6iSJvd7i55d8jv5aS9QiAIhqpf248goIVehFirhuLHZz5uWmdzymbNlYLLdzzXBbCPMXaAMXYTwM8AHlDU6QpgJmPsCAAwxkwDYLlLODz3nHmdCFIhXwXsfG4n3mv2XsjvpYzdFEm10oanN0Ts3oLIcezSMeT6yNg1O9oYt2mcZvmbi98Myf3seEAFQTEA8kTqxzxlcioAyENEy4nobyLqBhPcpVZywUavygUqh+U+ermFI4GdDHmC2KHEyBIhv0fqrVRHJj4L9i3Q3NsgJxQTLAfbTCAiecrDsYyxsZ5jrYFReeMEALUAtACQBGANEa1ljO3Ru6G7ftXyAfEVfeNsZmDSg5P83jv1JZzTxV6sG8BYxXXq9VPBdMcR+tTqE+kuCAKk3Y/tHJn4tJnSBkNXDMXlm5cd6JUx5fL4crC8s+wdp5pNY4zVlv2NlZ07BkAuqYsDUEb7PAZgPmPsKmPsPwArARgaNt0lHOTkzh3pHkSUjpU6+r13auWgJWTMwoUbueTK80wUStYO5axHwxINbdXXo189bX93QWTQSouqx/JDy9FwvDPfA8A4TtRHf37kyD2CMYoHyAYA5YmoDBFlAdAFgHKW9yuAxkSUQETZAdQDYLj5wl3CQb5yKBH6Ja2bCGbl8NPDPxme/73r74bnjdRKcvXXkVesh4oGgEerPmqrvm4fwqP3FVhECp5nlT1ndTUftjGKpmoW0iNaYYylAXgBwALwAX8aY2wHEfUhoj6eOrsAzAfwD4D1AMYxxrbrtQm4yeZw6RKwTRbyuEePiHUlWljSbQmu3rwadDsPVX7I8Lw8HtTj1R9XhQLQEw4nXj3hdy5LfBZb/XLKlmEWzE4QXk5ctp7fRGANxtg8APMUZWMU7z8F8KnVNt2zcuja1f+9C4zToaZ5meZoX5G75d2W5baA25HPrLXUU7dlvc2b27pFmRaq83o2h4S4hKBm7b3u7BXwtXKU0VAr5w+P04BA4GbcIxx2ydRjOYyD0MUyJXOV1Cx/veHr+LSl5UmBH1bCEyQlJAHwV191r9EdD1R8wCs4lDCwoGb/SYlJAV8rR7lyKJFLqCQFAjPcIxziZF1Ny7xqgp3P7cR//f9TlWdNyIrXG74eUJtWBnBJgMjj4Her0Q2zu8w2zGFgNS5O/eL1/d4fePGApeusIDeKA8L1ViCwgnt+JfJB5rp58vBYJTlLMvJlz+dom35qJR3DtlRHLhyqFKgCgAsmPQIdiMvkKRPQdVrULFwT37T7xtsXIRwEAnPc8ysRNoaAuDLwCtgQ5k1hqoWV2X2FfBUA+Gwb91e4XzMFqhw77rXyuq/Ud3YPCxGhT+0+yJMtD38vvJcEAlPcIxzi3NPVaKFivore9KInXjuB9hXUMWX2vODvJqgc0I++wnflT3loCuZ0mYMu1brgg+YfYHyH8Yb37t+wPwomm8b20mREa3VEzc3Pbg6oLS3EykEgMMc9vxIhHIIiIS5B0zZQPl953WuSEpJQPGdxAECubLnQvmJ7EBEGNR6EAsn6KxEA+KTlJyAilb7/x4esb4CSU7NwzYCu08JuHg2BIDPinhFXqJUskT97fu+xnrqodlF17l2JpqWbIne23ACcCcmhHIiVO7sl7N6reM7iWN59eUB9GtpkKDpV6YRtfbeZV3aIfEnO2okEglDjHuEgsMS/z/+LX7toZ+WSBuCBdw/UHazyJOXB8VeP276vXIW08emNuvWSEpPAhgQvdAomF0ST0uo8wAD36DLqQ+EchTH9kemoVrAaqhWsFnRftGhVrpXfe2UUXUHmYtWRVZHugm3cIxwyMszrCJAvez6Uz6uvKgK4QfbASwdw+nXtkO5WkrQrkdRPAFCraC3L10m8WFc7lr8eWeP1PaQqF6iMu4rchZfrvYyl3ZaqzstXVHLvKyeJozg/lZqeIBNkDoRwCBVHj/pvghMETc6sOVV2gza3twHgn6TdKoF6ACUl8JXE43doJ2rZ8uwWHHzpoKrcaG8FwJ9hZJuRaFammV+Zsq/SCmprn622+25EHMXhuTq+/CNf3/e1o+07wbCmsZHrWBAa3CEchg+PdA9iguEthqNhiYa4t+y9qnOnXz+N2Y/OBhDYQP/kHU8G1KdxHbSTr0jUKFwDpXOXVpUr7Smj2monk5ejtRKa2mkq3rnnnYDUS0ZJnZQeUbmyWUuK06lKJ1t9+KhF4JFEBzcZHPC1gtjHHcJBaVjt2DEi3XALt+e9HU1KNcG49v4Db6X8lbD6qdW4Las6DlOB5ALezWyJ8Ykomauk6nojXqj7gu653Nlyq/ZZLH5yMVY/tRpdq3fVucoefev0BWA/uF+R24rg3WbvBuTe+vY9b+uuOAJdSWVPzG65bp2idfDm3f4ZzL5o80VA95Uz/ZHpQbchcD9BR2X1JLfeCOA4Y+z+4LukeRP/90WKhOQ2sUJifCKW91ge8PVxFIfDLx+2dY00k29Wupnq3Jn+Z1SDZYuy6gB+ZkztNBWP/qIdxjuO4vDv8/8iT1Ie0z46iV6uiziKC/lmOy21n5GQy5MtD85fP2/arl6sLEHmwomVw0swSRoRNMoftXBrjTriKA7b+m7DnMfUmeQS4hIc2VsgBf/To2L+igFvvAO4F5dTJMQloFTuUn5lu1/Ybenatre3Dfi+j1R5xHv8RsM3/M61KtcKKa+l4NBLh/DXU39ZbtOp6LgCdxGUcCCi4gDaAbCuf3CCK1fCejuBNaoVrIYcWdwbMffDFh861lZSYhJerMc9sKSQI1Jo87J5yhpeO6PzDM3yj+/92PS+hXIYZ9srnKMwSuUuhQYlGpi2JWFmFxLEJsGuHD4H8AYAXX9AInqGiDYS0ca0QKOpKlcKbQOfWQncSyjUQqEiKSEJcRSH7X23Y2+/vQCse3/phSpXCl4zV+NBjQf5vbfqthtIylkRksScG2k3It0FWwT8iRLR/QBOM8b+NqrHGBsrJcVOSAjQxKEcFLp0CawdgauR6/CfvuvpCPYEmNl5puF5SRVTtWBVzdl87zt7276nXRuG0kPq4o2Ltu8pIQVe1EMIB3NCtacmVATziTYC0IGIDgH4GUBzIprsSK+UuGjGKAg995W/D0/c8YTt60a2Honsidm94UGC4cHKD6rK+tXt5z2uV7ye4fVft9Pe9/B4de39HhJreq2x0Ds1pXKVCsqTySyPsxAO5jgRjiacBPyJMsYGMsaKM8ZKA+gCYCljzP4v1gpCOAh0eLney5Z9/Z+44wlcHXQVifGJ5pVlrOttLfH8l22/tFTPaAWgDLvhdx2RX1IkK4PN2TfOInVQKg69fAiV8ley1D95u0VyWPMM1EsVK/CRmVYO4UMIB4EOI9uMVPn6O03dYnXxZRtrA78Rcl1+Ynyi7R3Kytm5FdtA3qS8hulWpTYvDbikavfuknfjxGsnLPVNHj4lMxCIR1kgtpxI4ohwYIwtD9keB0AIB0HE6Vevn9dO8FTNp/zONSkVWNykwU0G48rAKyiUbOxhJNGtRjfN8mD2JUirmGCN/Uu6LQnq+kjSt3Zf29fMe3webr5909Y1YuUgEISYcM3AJj04CUObDPW+l8Kh3573dr96y3sstxRpVtr9LFfvJGdJRs+aPQEYq2a61eimiuwqqX/W915vem89tOJN1SxcE4VzFMb7zd633I5y5WBFYElpZgPFyspLL0S8nHrFjO1DethVT2Yam4NAEG7C4cr6actPvcftyrfDkKZDVHWMfuSf3PsJNj2zSfNcsZzFMP/x+fjxYf+ER9KKQCvxkpRxTz54K9svlrOYbn/M0LJ/3Jb1NqS8lmIrkqzys6lbrK7pNf/0+UdVNrXTVMv3fOeed0zrWJlIhGvQdtvKIejwGQKBUyzrvgznrp0zrRfKH/PrDV9H/0X9AQTmgdO/UX/D861vb60qk1xdH6v2mOqc9Kx2+rKm1xpcvnHZUl3vyoEIi59c7E0rG2oKJRfS3DXfuWpn3RApeZPy+n0/rEwWrHxXrK5E8yXlw9lrZy3V1UIIh1Bw7VqkeyAIA01LNzU8H+pYRUqUA7I0Q7fqwWOVvEl5cWXgFc2gezmz5gQAVeBCQH9Qk3s0meG1OYBsx7vqdWcvfLf5O1vXSEgDJYEsC/thTYfhraVv4dKNS+aVPaRnpJvWsXp/o3qty7XGgv0LjK/PjAbpkDNpUqR7IIgCahSuASCwDWSBoBQOz9V5DtMfmY4eNXs4fq/kLMmaM+GHKz+Mb+//FsOa+fTrTqrXrObu0BKI8n0dqnZNBLkUIt1OzK06RevYHmDNZutvNjL2dCuTuwzuLHwnAOPB3YpTgttWDu4QDi6TuILQUPS2omBDGB6u8nBY7qcUDnEUh05VOoU1jAcR4Zlaz2imGXVCvWY169/6p/2N3nEUhxqFayB1UCquv3UdgH+4cbO0qDMf5TvM7eyPqFe8nu0B1ux/9NG9xntkutfojsXdFpu25ZSKK5pwh3AQrqyCCBCtsZyqF6yOXnf2ws8P/xx0W5IANBu4iucsjoYlGgLgdgEpj0VSYpI3D8jp108jOZHbLIY2Heq9tnHJxqr2pF3q8v9x9YLVA3sIA5TCRB680IpgTMtIM1wFyVPB2u1LtCOEg0CgQ7SGhIiPi8e4DuNQuUDloNuSkjRZSZL0Vduv0LBEQ0x4YIJm5rzkLMne4ICSrQQA5nadq9umfOXwT1+195ISpRA7+dpJ7Ou3z/u+fYX2WNd7HX7v+ju2PLvFOyBLn2WpXKW82fYkwfRQ5YdQp2gdr2CTk5aR5lO9aQiRf/r+YzkplhAOAkGMEK3CQQstg7UVhrcYjvTB6ZaEw11F7sLqp1Yb7rjWWoHIBYUSu/9jaYCVQpoUylEI5fKWw2etPgPA96DULVYX95W/DzUK1/DWl69UJj/IQ8BJK4I8SXmw/un1KJe3nOp+aRlpXnXZ83WeV52vlL8Set1lLd+FMEiHArFyEEQANwmHPf324OgrR21fR0SOPqc0AFr1LDMSNEbtKzPwKVcIEtLGRS1PMGVd5UZAAKhcoDKyxGdB+uB0vN9cf1Ngm9vbAACmdZqm++xi5SAQxAjhdp0NhtzZckdVfCOlvWZ4i+GaQQVX9Fhhq906xeoAUBuy5a6xcr69/1t8fd/XmgELVbGqZKuezc9uxqZnNnl3r8dRnKENqmbhmmBDGB6p+ohuHSEcQoFYOQgiQCQN0idfO4mU11Iidv9AaVSyEQC1t9KAuwfg0arqzW1WI8VKzH1sLtb2WqsKXSGtKJQDfu5sudG3Tl80KM4z35XNU9ZrRJbvhtfiziJ3BvUdyJXVP59GKL2ViKgNEe0mon1ENEDjfFMiukhEWzx/g83adMcmOCEcBGGkXrF6WHfcWpjuUGGW7jNamfLQFOw5u0fTzmCWMEhJx0odMfvf2X5lubLl0syV4V056IwVr9R/Be3Kt0PF/BUBwFIsrEDRW5mEauVARPEARgNoCeAYgA1ENIcxtlNRdZWdAKnuWDnkcG9eYoH7WPDEAmx5dkuku+FKsidm9+rrldxd8m7seG4HMgb7D5Jf3/e11zUWAH555BcAQM1CvJ0n7ngCZ98wDlvxbO1n0a58O7za4FXN80TkFQx6WLG9rOyxEt+0+8a0HqAOzBdCg3RdAPsYYwcYYzfBk689EGyj7hAORZwNVyAQGJErWy7vbmyBs1QpUEU1u+9bp6+fgfnhKg+DDWEokMw9sHIk5kDepLyG7eZNyou5XeeiYHLBgPs2/ZHppnUal2qMPrX7WGpPmXEwmDStABKIaKPs7xnZuWIA5N4IxzxlShoQ0VYi+oOIqpreMJjeho0M2UyjdOmIdUMgEFhnWNNhmnsHrCIZsJ+s8aRTXTJEGYo9UDpX7YxpO6YhIc5/eL2Zbi//g4I0xlhtnXNaujTlMmUTgFKMsStEdB+A2QDUYYDljYbT9zY5OZldvXrV/oW1agGbPGGKXeYrLBAI3AMN82x4C8ImkZ6RjhvpN1BvXD1sP70dANC1eldMeWhK4P0iSmWMaUpaImoAYChjrLXn/UAAYIwNN2jvEIDajLH/9Oq4Q60kBIJAIAgDc7rMQYeKHYJqIz4uHtkTs3vdat9r9h7G3j/Wie7psQFAeSIqQ0RZAHQBMEdegYgKk0efR0R1wcd+Q0OO+9RKAoFAECLaV2yP9hXbO9KWZFu5v8L9Ic2TwRhLI6IXACwAEA9gPGNsBxH18ZwfA6ATgL5ElAbgGoAuzERt5A7hkG4ek10gEAiiCW9QwzBoPhhj8wDMU5SNkR2PAjDKTpvuUCulpUW6BwKBQGALSa3ktp3REkI4CAQCQQiwmkgpWnGXcFhgnIZPIBAIooVwqpVCgbuEQ+Xg49cLBAJBOJCEg1ArhZJsniBeidazLgkEAkEkkTYARmtGQTPc4a30xx/A9OlAIXcGIxMIBJmPyQ9NxpiNY1CnaJ1IdyUg3LFDWiAQCDIxRjukQ4U71EoCgUAgCCtCOAgEAoFAhRAOAoFAIFAhhINAIBAIVAjhIBAIBAIVQjgIBAKBQIUQDgKBQCBQIYSDQCAQCFSEdRMcEWWAJ5oIhAQAmTk8q3j+zPv8mfnZAfH8CQASGWNhncyHVTgEAxFtNEiwHfOI58+8z5+Znx0Qzx+p5xdqJYFAIBCoEMJBIBAIBCrcJBzGRroDEUY8f+YlMz87IJ4/Is/vGpuDQCAQCMKHm1YOAoFAIAgTQjgIBAKBQIUrhAMRtSGi3US0j4gGRLo/oYCIDhHRNiLaQkQbPWV5iWgREe31vOaR1R/o+X/sJqLWket5YBDReCI6TUTbZWW2n5eIann+b/uI6EtySU5GnecfSkTHPd+BLUR0n+xczDw/EZUgomVEtIuIdhDRS57yTPH5Gzx/dH3+jLGo/gMQD2A/gLIAsgDYCqBKpPsVguc8BCC/ouwTAAM8xwMAfOw5ruL5P2QFUMbz/4mP9DPYfN57ANwFYHswzwtgPYAGAAjAHwDaRvrZgnj+oQBe16gbU88PoAiAuzzHtwHY43nGTPH5Gzx/VH3+blg51AWwjzF2gDF2E8DPAB6IcJ/CxQMAJnqOJwLoKCv/mTF2gzF2EMA+8P+Ta2CMrQRwTlFs63mJqAiAnIyxNYz/Un6QXRPV6Dy/HjH1/IyxFMbYJs/xZQC7ABRDJvn8DZ5fj4g8vxuEQzEAR2Xvj8H4H+lWGICFRPQ3ET3jKSvEGEsB+BcKQEFPeaz+T+w+bzHPsbLczbxARP941E6SWiVmn5+ISgO4E8A6ZMLPX/H8QBR9/m4QDlo6tFj0v23EGLsLQFsAzxPRPQZ1M8v/RELveWPt//ANgHIAagJIAfCZpzwmn5+IcgCYAeBlxtglo6oaZbH4/FH1+btBOBwDUEL2vjiAExHqS8hgjJ3wvJ4GMAtcTXTKs3SE5/W0p3qs/k/sPu8xz7Gy3JUwxk4xxtIZYxkA/gefqjDmnp+IEsEHximMsZme4kzz+Ws9f7R9/m4QDhsAlCeiMkSUBUAXAHMi3CdHIaJkIrpNOgbQCsB28Ofs7qnWHcCvnuM5ALoQUVYiKgOgPLhhyu3Yel6P6uEyEdX3eGl0k13jOqSB0cOD4N8BIMae39PX7wDsYoyNkJ3KFJ+/3vNH3ecfacu9Rev+feAW/f0A3op0f0LwfGXBvRG2AtghPSOAfACWANjrec0ru+Ytz/9jN1zgoaHxzD+BL51vgc+AegXyvABqe35E+wGMgmfXf7T/6Tz/JADbAPwDPiAUicXnB3A3uPrjHwBbPH/3ZZbP3+D5o+rzF+EzBAKBQKDCDWolgUAgEIQZIRwEAoFAoEIIB4FAIBCoEMJBIBAIBCqEcBAIBAKBCiEcBAKBQKBCCAeBQCAQqPh/vK0nliTtrE0AAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "iters = [item[\"iter\"] for item in result]\n", "fig, ax1 = plt.subplots()\n", "ax1.plot(iters,[item[\"loss\"] for item in result],'g')\n", "ax2 = ax1.twinx()\n", "ax2.plot(iters,[item[\"accuracy\"] for item in result],'r')\n", "plt.xlabel(\"iter\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 45, "id": "cab7889f", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "gpu allocated : 2778 MB\n", "gpu reserved : 3272 MB\n" ] } ], "source": [ "torch.cuda.empty_cache()\n", "print(f\"gpu allocated : {torch.cuda.memory_allocated() // 1024**2} MB\")\n", "print(f\"gpu reserved : {torch.cuda.memory_reserved() // 1024 ** 2} MB\")" ] }, { "cell_type": "code", "execution_count": 46, "id": "29ffab84", "metadata": {}, "outputs": [], "source": [ "torch.save(model.state_dict(), \"model.zip\")" ] }, { "cell_type": "code", "execution_count": 21, "id": "4b9b9579", "metadata": {}, "outputs": [], "source": [ "del batch_inputs\n", "del batch_labels\n", "del loss\n", "del optimizer" ] }, { "cell_type": "code", "execution_count": 88, "id": "80d0ee50", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 88, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model.load_state_dict(torch.load(\"model.zip\"))" ] }, { "cell_type": "code", "execution_count": 89, "id": "5c9b570c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "MyModel(\n", " (bert): BertModel(\n", " (embeddings): BertEmbeddings(\n", " (word_embeddings): Embedding(119547, 768, padding_idx=0)\n", " (position_embeddings): Embedding(512, 768)\n", " (token_type_embeddings): Embedding(2, 768)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (encoder): BertEncoder(\n", " (layer): ModuleList(\n", " (0): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (1): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (2): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (3): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (4): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (5): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (6): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (7): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (8): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (9): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (10): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (11): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " )\n", " (pooler): BertPooler(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (activation): Tanh()\n", " )\n", " )\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (lin1): Linear(in_features=768, out_features=768, bias=True)\n", " (gelu): GELU()\n", " (lin2): Linear(in_features=768, out_features=1, bias=True)\n", ")" ] }, "execution_count": 89, "metadata": {}, "output_type": "execute_result" } ], "source": [ "device = torch.device('cuda')\n", "model.to(device)" ] }, { "cell_type": "code", "execution_count": 38, "id": "fff7a7d0", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████| 1250/1250 [00:34<00:00, 36.39batch/s]\n" ] } ], "source": [ "model.eval()\n", "collect_list = []\n", "with torch.no_grad():\n", " with tqdm(dev_loader, unit=\"batch\") as tepoch:\n", " for batch_i,batch_l in tepoch:\n", " batch_inputs = {k: v.cuda(device) for k, v in list(batch_i.items())}\n", " batch_labels = batch_l.cuda(device)\n", " output = model(**batch_inputs)\n", " loss = BCELoss(output, batch_labels.double())\n", " \n", " prediction = (output > 0).to(device,dtype=torch.int64)\n", " correct = (prediction == batch_labels).sum().item()\n", " accuracy = correct / prediction.size()[0]\n", " \n", " collect_list.append({\"loss\":loss.item(),\"accuracy\":accuracy, \"batch_size\":batch_labels.size(0),\n", " \"predict\":prediction.cpu(),\n", " \"actual\":batch_labels.cpu()})" ] }, { "cell_type": "code", "execution_count": 23, "id": "4e9a90b5", "metadata": {}, "outputs": [], "source": [ "def getConfusionMatrix(predict,actual):\n", " ret = torch.zeros((2,2),dtype=torch.long)\n", " for p_s,a_s in zip(predict,actual):\n", " ret[p_s,a_s] += 1\n", " return ret" ] }, { "cell_type": "code", "execution_count": 39, "id": "b7a513c9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "average_loss : 0.33939967472716237, average_accuracy : 0.86565, size :20000\n" ] } ], "source": [ "total_loss = 0\n", "total_accuracy = 0\n", "total_size = 0\n", "confusion = torch.zeros((2,2),dtype=torch.long)\n", "\n", "for item in collect_list:\n", " batch_size = item[\"batch_size\"]\n", " total_loss += batch_size * item[\"loss\"]\n", " total_accuracy += batch_size * item[\"accuracy\"]\n", " total_size += batch_size\n", " confusion += getConfusionMatrix(item[\"predict\"],item[\"actual\"])\n", "print(f\"\"\"average_loss : {total_loss/total_size}, average_accuracy : {total_accuracy/total_size}, size :{total_size}\"\"\")" ] }, { "cell_type": "code", "execution_count": 38, "id": "1ac327de", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[8716, 1638],\n", " [1194, 8452]])" ] }, "execution_count": 38, "metadata": {}, "output_type": "execute_result" } ], "source": [ "confusion" ] }, { "cell_type": "code", "execution_count": 122, "id": "3e71d4d2", "metadata": {}, "outputs": [], "source": [ "def getF1Score(confusion,c):\n", " TP = confusion[c,c]\n", " FP = confusion[c].sum() - TP\n", " FN = confusion[:,c].sum() - TP\n", " precision = TP / (TP + FP)\n", " recall = TP / (TP + FN)\n", "\n", " f1Score = (2*precision*recall)/(precision + recall)\n", " return f1Score" ] }, { "cell_type": "code", "execution_count": 242, "id": "6756408c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "f1 score : 0.7763830423355103\n" ] } ], "source": [ "print(f\"f1 score : {getF1Score(confusion,1)}\")" ] }, { "cell_type": "code", "execution_count": 86, "id": "f28f64e9", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "MyModel(\n", " (bert): BertModel(\n", " (embeddings): BertEmbeddings(\n", " (word_embeddings): Embedding(119547, 768, padding_idx=0)\n", " (position_embeddings): Embedding(512, 768)\n", " (token_type_embeddings): Embedding(2, 768)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (encoder): BertEncoder(\n", " (layer): ModuleList(\n", " (0): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (1): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (2): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (3): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (4): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (5): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (6): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (7): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (8): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (9): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (10): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (11): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " )\n", " (pooler): BertPooler(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (activation): Tanh()\n", " )\n", " )\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (lin1): Linear(in_features=768, out_features=768, bias=True)\n", " (gelu): GELU()\n", " (lin2): Linear(in_features=768, out_features=1, bias=True)\n", ")" ] }, "execution_count": 86, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model.eval()" ] }, { "cell_type": "markdown", "id": "2da5789b", "metadata": {}, "source": [ "한번 테스트해보기" ] }, { "cell_type": "code", "execution_count": 51, "id": "cc727fd9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "웹 gpu\n", "긍정적 output : 54.185086488723755 %\n", "부정적 output : 45.814913511276245 %\n" ] } ], "source": [ "sen = input()\n", "inputs = tokenizer(sen, return_tensors = 'pt', padding='longest')\n", "output = model(**{k: v.to(device) for k,v in inputs.items() })\n", "prob = torch.sigmoid(output).item()\n", "print(\"긍정적 output :\",prob * 100,\"%\")\n", "print(\"부정적 output :\", (1-prob) * 100,\"%\")" ] }, { "cell_type": "markdown", "id": "2faa8141", "metadata": {}, "source": [ "```\n", "5471412\t맘에 들어요~ 0\n", "```\n", "라벨이 잘못 붙어있는 것들이 있다. 별점가지고만 긍정, 부정을 매긴 것 같다." ] }, { "cell_type": "code", "execution_count": 40, "id": "b40f071c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████| 1875/1875 [00:51<00:00, 36.45batch/s]\n" ] } ], "source": [ "model.eval()\n", "collect_list = []\n", "with torch.no_grad():\n", " with tqdm(test_loader, unit=\"batch\") as tepoch:\n", " for batch_i,batch_l in tepoch:\n", " batch_inputs = {k: v.cuda(device) for k, v in list(batch_i.items())}\n", " batch_labels = batch_l.cuda(device)\n", " output = model(**batch_inputs)\n", " loss = BCELoss(output, batch_labels.double())\n", " \n", " prediction = (output > 0).to(device,dtype=torch.int64)\n", " correct = (prediction == batch_labels).sum().item()\n", " accuracy = correct / prediction.size()[0]\n", " \n", " collect_list.append({\"loss\":loss.item(),\"accuracy\":accuracy, \"batch_size\":batch_labels.size(0),\n", " \"predict\":prediction.cpu(),\n", " \"actual\":batch_labels.cpu()})" ] }, { "cell_type": "code", "execution_count": 52, "id": "871b72d4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "average_loss : 0.3305340794156926, average_accuracy : 0.8672333333333333, size :30000\n" ] } ], "source": [ "total_loss = 0\n", "total_accuracy = 0\n", "total_size = 0\n", "confusion = torch.zeros((2,2),dtype=torch.long)\n", "\n", "for item in collect_list:\n", " batch_size = item[\"batch_size\"]\n", " total_loss += batch_size * item[\"loss\"]\n", " total_accuracy += batch_size * item[\"accuracy\"]\n", " total_size += batch_size\n", " confusion += getConfusionMatrix(item[\"predict\"],item[\"actual\"])\n", "print(f\"\"\"average_loss : {total_loss/total_size}, average_accuracy : {total_accuracy/total_size}, size :{total_size}\"\"\")" ] }, { "cell_type": "code", "execution_count": null, "id": "62333944", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.11" } }, "nbformat": 4, "nbformat_minor": 5 }