1840 lines
112 KiB
Plaintext
1840 lines
112 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"id": "1cc9cec9",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"'C:\\\\Users\\\\Monoid\\\\anaconda3\\\\envs\\\\nn\\\\python.exe'"
|
|
]
|
|
},
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"import sys\n",
|
|
"sys.executable"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "f9c786f1",
|
|
"metadata": {},
|
|
"source": [
|
|
"파이썬 환경 확인.\n",
|
|
"envs\\\\nn\\\\python.exe 으로 끝나기를 기대합니다"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "4c0a08d7",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"100%|████████████████████████████████████████████████████████████████████████████████████| 5/5 [00:00<00:00, 9.31it/s]\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from time import sleep\n",
|
|
"from tqdm import tqdm, trange\n",
|
|
"\n",
|
|
"lst = [i for i in range(5)]\n",
|
|
"\n",
|
|
"for element in tqdm(lst):\n",
|
|
" sleep(0.1)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "56a66a44",
|
|
"metadata": {},
|
|
"source": [
|
|
"### tqdm 소개\n",
|
|
"tqdm은 다음과 같이 Progress bar 그려주는 라이브러리이에요. 와! 편하다.\n",
|
|
"```\n",
|
|
"from tqdm.auto import tqdm\n",
|
|
"from tqdm.notebook import tqdm\n",
|
|
"```\n",
|
|
"이건 0에서 멈춰이있고 작동하지 않더라고요. 왜인진 몰라요."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"id": "d00b25e6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from preprocessing import readPreporcssedDataAll\n",
|
|
"import torch\n",
|
|
"from torch.utils.data import Dataset, DataLoader\n",
|
|
"from dataset import make_collate_fn, DatasetArray\n",
|
|
"from transformers import BertTokenizer\n",
|
|
"import torch.nn as nn\n",
|
|
"from read_data import TagIdConverter"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "a1d7bb15",
|
|
"metadata": {},
|
|
"source": [
|
|
"대충 필요한 것 임포트"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"id": "433419f4",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"tagIdConverter = TagIdConverter()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"id": "79fb54df",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"cuda available : True\n",
|
|
"available device count : 1\n",
|
|
"device name: NVIDIA GeForce RTX 3070\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"print(\"cuda available :\",torch.cuda.is_available())\n",
|
|
"print(\"available device count :\",torch.cuda.device_count())\n",
|
|
"\n",
|
|
"if torch.cuda.is_available():\n",
|
|
" device_index = torch.cuda.current_device()\n",
|
|
" print(\"device name:\",torch.cuda.get_device_name(device_index))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "dd22dd5e",
|
|
"metadata": {},
|
|
"source": [
|
|
"cuda가 가능한지 먼저 확인해보아요."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"id": "2cd9fe37",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"PRETAINED_MODEL_NAME = 'bert-base-multilingual-cased'\n",
|
|
"tokenizer = BertTokenizer.from_pretrained(PRETAINED_MODEL_NAME)\n",
|
|
"\n",
|
|
"my_collate_fn = make_collate_fn(tokenizer)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "2177c793",
|
|
"metadata": {},
|
|
"source": [
|
|
"데이터 로딩 준비"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"id": "e738062d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from transformers import BertModel"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"id": "70296ee3",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertModel: ['cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
|
|
"- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
|
|
"- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"PRETAINED_MODEL_NAME = 'bert-base-multilingual-cased'\n",
|
|
"bert = BertModel.from_pretrained(PRETAINED_MODEL_NAME)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"id": "6cbc236d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class MyModel(nn.Module):\n",
|
|
" def __init__(self,output_feat: int,bert):\n",
|
|
" super().__init__()\n",
|
|
" self.bert = bert\n",
|
|
" self.dropout = nn.Dropout(p=0.1)\n",
|
|
" self.lin = nn.Linear(768,output_feat) #[batch_size,word_size,768] -> [batch_size,word_size,output_feat]\n",
|
|
" self.softmax = nn.Softmax(2) #[batch_size,word_size,output_feat] -> [batch_size,word_size,output_feat]\n",
|
|
" #0부터 시작해서 2 번째 차원에 softmax.\n",
|
|
"\n",
|
|
" def forward(self,**kargs):\n",
|
|
" emb = self.bert(**kargs)\n",
|
|
" e = self.dropout(emb['last_hidden_state'])\n",
|
|
" w = self.lin(e)\n",
|
|
" return w"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "7dec337a",
|
|
"metadata": {},
|
|
"source": [
|
|
"모델 선언\n",
|
|
"`nn.CrossEntropy`는 소프트맥스를 겸함."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"id": "e5214de8",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"MyModel(\n",
|
|
" (bert): BertModel(\n",
|
|
" (embeddings): BertEmbeddings(\n",
|
|
" (word_embeddings): Embedding(119547, 768, padding_idx=0)\n",
|
|
" (position_embeddings): Embedding(512, 768)\n",
|
|
" (token_type_embeddings): Embedding(2, 768)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (encoder): BertEncoder(\n",
|
|
" (layer): ModuleList(\n",
|
|
" (0): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (1): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (2): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (3): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (4): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (5): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (6): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (7): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (8): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (9): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (10): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (11): BertLayer(\n",
|
|
" (attention): BertAttention(\n",
|
|
" (self): BertSelfAttention(\n",
|
|
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" (output): BertSelfOutput(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (intermediate): BertIntermediate(\n",
|
|
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
|
" )\n",
|
|
" (output): BertOutput(\n",
|
|
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
|
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" )\n",
|
|
" )\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (pooler): BertPooler(\n",
|
|
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
|
" (activation): Tanh()\n",
|
|
" )\n",
|
|
" )\n",
|
|
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
|
" (lin): Linear(in_features=768, out_features=22, bias=True)\n",
|
|
" (softmax): Softmax(dim=2)\n",
|
|
")\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"model = MyModel(22,bert)\n",
|
|
"model.cuda()\n",
|
|
"bert.cuda()\n",
|
|
"print(model)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "72df7ac3",
|
|
"metadata": {},
|
|
"source": [
|
|
"Tag의 종류가 22가지 입니다. 그래서 22을 넣었어요."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "0bba477c",
|
|
"metadata": {},
|
|
"source": [
|
|
"생성과 동시에 gpu로 옮기자.\n",
|
|
"`cuda` 저거 호출하면 됨."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"id": "5aa129e3",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"bert current device : cuda:0\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"print(\"bert current device :\",bert.device)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"id": "61d356b6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#for param in bert.parameters():\n",
|
|
"# param.requires_grad = False"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cf6690b2",
|
|
"metadata": {},
|
|
"source": [
|
|
"bert 는 업데이트 하지 않는다. 메모리를 아낄 수 있다."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"id": "f28a1a61",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"device(type='cuda')"
|
|
]
|
|
},
|
|
"execution_count": 13,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"device = torch.device(\"cuda\")\n",
|
|
"device"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 28,
|
|
"id": "8844beef",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"inputs = {'input_ids': torch.tensor([[ 101, 39671, 8935, 73380, 30842, 9632, 125, 9998, 9251, 9559,\n",
|
|
" 9294, 8932, 28143, 9952, 8872, 127, 9489, 34907, 9952, 9279,\n",
|
|
" 12424, 102]],device=device), \n",
|
|
" 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],device=device), \n",
|
|
" 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]],device=device)}"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "4046945a",
|
|
"metadata": {},
|
|
"source": [
|
|
"적당한 인풋을 정의한다"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 29,
|
|
"id": "c37c3c1b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"emb = model(**inputs)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 30,
|
|
"id": "261d4cc7",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"torch.Size([1, 22, 22])"
|
|
]
|
|
},
|
|
"execution_count": 30,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"emb.size()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "d492e37b",
|
|
"metadata": {},
|
|
"source": [
|
|
"결과가 잘 나왔어요."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 31,
|
|
"id": "c773fdba",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"entity_ids = torch.tensor([21,21,7,17,17,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21]\n",
|
|
" ,dtype=torch.int64,device=device)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "b0b69822",
|
|
"metadata": {},
|
|
"source": [
|
|
"잘 이해하지는 못하겠는데, int64면 실행이 되고 int32이면 실행이 안된다.\n",
|
|
"```\n",
|
|
"RuntimeError: \"nll_loss_forward_reduce_cuda_kernel_2d_index\" not implemented for 'Int'\n",
|
|
"```\n",
|
|
"이런 오류를 내면서 죽음."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 32,
|
|
"id": "0f97b71a",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"torch.Size([22, 22])"
|
|
]
|
|
},
|
|
"execution_count": 32,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"emb.view(-1,emb.size(-1)).size()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 36,
|
|
"id": "2a35055b",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor([19, 13, 21, 21, 19, 21, 13, 13, 16, 16, 21, 16, 19, 13, 2, 13, 13, 16,\n",
|
|
" 13, 20, 19, 19], device='cuda:0')"
|
|
]
|
|
},
|
|
"execution_count": 36,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"predict = emb.view(-1,emb.size(-1)).argmax(dim=-1)\n",
|
|
"predict"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 39,
|
|
"id": "778c99b7",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor([False, False, False, False, False, True, False, False, False, False,\n",
|
|
" True, False, False, False, False, False, False, False, False, False,\n",
|
|
" False, False], device='cuda:0')"
|
|
]
|
|
},
|
|
"execution_count": 39,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"(predict == entity_ids)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 38,
|
|
"id": "798091aa",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor([[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],\n",
|
|
" device='cuda:0')"
|
|
]
|
|
},
|
|
"execution_count": 38,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"(predict == entity_ids) * inputs[\"attention_mask\"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 19,
|
|
"id": "d7d0164a",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor(2.9564, device='cuda:0', grad_fn=<NllLossBackward0>)"
|
|
]
|
|
},
|
|
"execution_count": 19,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"nn.CrossEntropyLoss(ignore_index=tagIdConverter.pad_id)(emb.view(-1,emb.size(-1)),entity_ids)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "9fee44d6",
|
|
"metadata": {},
|
|
"source": [
|
|
"크로스 엔트로피를 계산하는 데에 성공.\n",
|
|
"ignore_index는 padding class를 넣어요."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 40,
|
|
"id": "197cfa62",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"del inputs\n",
|
|
"del entity_ids\n",
|
|
"del emb"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "733cb548",
|
|
"metadata": {},
|
|
"source": [
|
|
"본격적으로 학습시켜봅시다."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 21,
|
|
"id": "40a3e52c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"datasetTrain, datasetDev, datasetTest = readPreporcssedDataAll()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "66d41fee",
|
|
"metadata": {},
|
|
"source": [
|
|
"데이터 셋이 적어도 어느정도 성능이 나와하야 할지 생각해봅시다.\n",
|
|
"`O` 토큰으로 범벅이 되있으니 전부 `O`로 찍는 것 보다 좋은 성능이 나와야 하지 않겠어요?\n",
|
|
"한번 시도해봅시다."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 22,
|
|
"id": "80c37a04",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"100%|██████████████████████████████████████████████████████████████████████████| 4250/4250 [00:00<00:00, 236146.99it/s]"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"151572/190488 = 0.7957036663726849\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"total_l = 0\n",
|
|
"total_o = 0\n",
|
|
"\n",
|
|
"for item in tqdm(datasetTrain):\n",
|
|
" entities = item[\"entity\"]\n",
|
|
" l = len(entities)\n",
|
|
" o = sum(map(lambda x: 1 if x == \"O\" else 0,entities))\n",
|
|
" total_l += l\n",
|
|
" total_o += o\n",
|
|
"\n",
|
|
"print(f\"{total_o}/{total_l} = {total_o/total_l}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "c0b67c41",
|
|
"metadata": {},
|
|
"source": [
|
|
"79% 보다 높아야 해요."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 23,
|
|
"id": "619b959f",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"BATCH_SIZE = 4\n",
|
|
"train_loader = DataLoader(\n",
|
|
" DatasetArray(datasetTrain),\n",
|
|
" batch_size=BATCH_SIZE,\n",
|
|
" shuffle=True,\n",
|
|
" collate_fn=my_collate_fn\n",
|
|
")\n",
|
|
"dev_loader = DataLoader(\n",
|
|
" DatasetArray(datasetDev),\n",
|
|
" batch_size=BATCH_SIZE,\n",
|
|
" shuffle=True,\n",
|
|
" collate_fn=my_collate_fn\n",
|
|
")\n",
|
|
"test_loader = DataLoader(\n",
|
|
" DatasetArray(datasetTest),\n",
|
|
" batch_size=BATCH_SIZE,\n",
|
|
" shuffle=True,\n",
|
|
" collate_fn=my_collate_fn\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "7d45dd29",
|
|
"metadata": {},
|
|
"source": [
|
|
"BATCH_SIZE 를 4로 잡는다.\n",
|
|
"bert paramter를 freeze 안했을땐 batch를 8 정도로 했어요. 그 이상은 메모리가 부족해서 돌아가지 않아요.\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 24,
|
|
"id": "efd1837a",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"({'input_ids': tensor([[ 101, 39671, 8935, 73380, 30842, 9632, 125, 9998, 9251, 9559,\n",
|
|
" 9294, 8932, 28143, 9952, 8872, 127, 9489, 34907, 9952, 9279,\n",
|
|
" 12424, 102]]),\n",
|
|
" 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]),\n",
|
|
" 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])},\n",
|
|
" tensor([[21, 21, 7, 17, 17, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,\n",
|
|
" 21, 21, 21, 21]]))"
|
|
]
|
|
},
|
|
"execution_count": 24,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"my_collate_fn(datasetTrain[0:1])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "c4a2e2e1",
|
|
"metadata": {},
|
|
"source": [
|
|
"데이터를 한번 더 확인"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 25,
|
|
"id": "c575cb55",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from torch.optim import AdamW"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "627cb2f8",
|
|
"metadata": {},
|
|
"source": [
|
|
"tqdm 확인"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 26,
|
|
"id": "56844773",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"optimizer = AdamW(model.parameters(), lr=1.0e-5)\n",
|
|
"CELoss = nn.CrossEntropyLoss(ignore_index=tagIdConverter.pad_id)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "eaa08ab2",
|
|
"metadata": {},
|
|
"source": [
|
|
"옵티마이져 준비"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 27,
|
|
"id": "78e46670",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"[1, 2, 3, 4]\n",
|
|
"[5, 6, 7, 8]\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from groupby_index import groupby_index\n",
|
|
"\n",
|
|
"for g in groupby_index([1,2,3,4,5,6,7,8],4):\n",
|
|
" print([*g])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "ed61ce06",
|
|
"metadata": {},
|
|
"source": [
|
|
"`groupby_index` 그룹으로 묶어서 실행"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 43,
|
|
"id": "109259b4",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"epoch 0 start:\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 0: 100%|███████████████████████████████████████| 1063/1063 [00:45<00:00, 23.15batch/s, accuracy=0.923, loss=2.24]\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"epoch 1 start:\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 1: 100%|███████████████████████████████████████| 1063/1063 [00:46<00:00, 23.07batch/s, accuracy=0.961, loss=1.52]\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"epoch 2 start:\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 2: 100%|██████████████████████████████████████| 1063/1063 [00:46<00:00, 23.06batch/s, accuracy=0.976, loss=0.793]\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"epoch 3 start:\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 3: 100%|███████████████████████████████████████| 1063/1063 [00:46<00:00, 23.07batch/s, accuracy=0.935, loss=1.88]\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"epoch 4 start:\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 4: 100%|███████████████████████████████████████| 1063/1063 [00:46<00:00, 23.09batch/s, accuracy=0.98, loss=0.524]\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"TRAIN_EPOCH = 5\n",
|
|
"\n",
|
|
"result = []\n",
|
|
"iteration = 0\n",
|
|
"\n",
|
|
"t = []\n",
|
|
"\n",
|
|
"model.zero_grad()\n",
|
|
"\n",
|
|
"for epoch in range(TRAIN_EPOCH):\n",
|
|
" model.train()\n",
|
|
" print(f\"epoch {epoch} start:\")\n",
|
|
" with tqdm(train_loader, unit=\"minibatch\") as tepoch:\n",
|
|
" tepoch.set_description(f\"Epoch {epoch}\")\n",
|
|
" \n",
|
|
" for batch in groupby_index(tepoch,8):\n",
|
|
" corrects = 0\n",
|
|
" totals = 0\n",
|
|
" losses = 0\n",
|
|
" \n",
|
|
" optimizer.zero_grad()\n",
|
|
" for mini_i,mini_l in batch:\n",
|
|
" batch_inputs = {k: v.cuda(device) for k, v in list(mini_i.items())}\n",
|
|
" batch_labels = mini_l.cuda(device)\n",
|
|
" attention_mask = batch_inputs[\"attention_mask\"]\n",
|
|
" \n",
|
|
" output = model(**batch_inputs)\n",
|
|
" loss = CELoss(output.view(-1, output.size(-1)), batch_labels.view(-1))\n",
|
|
" \n",
|
|
" prediction = output.view(-1, output.size(-1)).argmax(dim=-1)\n",
|
|
" corrects += ((prediction == batch_labels.view(-1)) * attention_mask.view(-1)).sum().item()\n",
|
|
" totals += attention_mask.view(-1).sum().item()\n",
|
|
" losses += loss.item()\n",
|
|
" loss.backward()\n",
|
|
"\n",
|
|
" optimizer.step()\n",
|
|
" accuracy = corrects / totals\n",
|
|
" result.append({\"iter\":iteration,\"loss\":losses,\"accuracy\":accuracy})\n",
|
|
" tepoch.set_postfix(loss=losses, accuracy= accuracy)\n",
|
|
" iteration += 1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 44,
|
|
"id": "f0d9b2d7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%matplotlib inline"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 45,
|
|
"id": "19ca6da1",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"import matplotlib.pyplot as plt"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 46,
|
|
"id": "0bee685c",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD7CAYAAACBiVhwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAABMJElEQVR4nO2dd5gURfPHv3WJOw4k53ggKCAKimAEBUURxYygL4qgiIqv4VVB/b0ChlcwooIYUEGCKIiAgAIqqCBZMkeOR07HHQfc3e7W74/e3p3ZnY2XdqA+zzPPzvT09NTc7XZNVVdXEzNDEARBEIzElbQAgiAIQuwhykEQBEHwQ5SDIAiC4IcoB0EQBMEPUQ6CIAiCH6IcBEEQBD9CKgciqkNE84gonYjWE9HT7vJBRLSXiFa5t1uKXlxBEAShOKBQ8xyIqAaAGsz8DxGVBbACwB0AugI4yczvFrmUgiAIQrGSEKoCM+8HsN+9n01E6QBqRXOzuLg4TklJieZSQRCEc5ZTp04xMxfrMEBIy8FUmag+gD8BXATgOQA9AWQBWA7gP8x8PNj1qampnJOTE62sgiAI5yREdIqZU4vznmFrIiIqA+AHAM8wcxaAkQAaAmgBZVm8F+C6PkS0nIiWOxyOgkssCIIgFDlhWQ5ElAhgBoDZzPy+xfn6AGYw80XB2hHLQRAEIXJi0nIgIgLwJYB0o2JwD1Rr7gSwrvDFEwRBEEqCcNxKVwPoAaC9T9jq20S0lojWALgewLNFKaggCIJgDRF9RUSHiMjyJZ0UHxHRViJaQ0SXhmoznGilBQDI4tSs0CILgiAIxcBoAMMBfBPgfCcAjdxbG6gx4zbBGpQZ0oIgCDaHmf8EcCxIldsBfMOKxQDK+wwN+CHKQRAE4eynFoA9huMMhJivZgvlMGPzDAxZMKSkxRCEs5MzZwCJIgwPlwsIJyT/11+BTZsK884JekqAe+sT4fVWQwNBQ1VtoRx+2foL3v1bsnQIQpFw4YVAmTLRXbt0KXDypH95fn5whcMMbN4MHDkCLFgQ3b0BwOkEHn4Y+PZbYNUqICPD+l7vvQcQAVlZwOnTqiwSsrOBkSOBfv2AxETv9bm5wF9/+de/8Ub1dy08HMzcyrB9HuH1GQDqGI5rA9gX7AJbKIeEuAQ4XDKBToiA7Gz1RmwnJk1SHVyojuvAAWDo0Mg7uEDs2hX8fIUKwGOP+ZefPAm0aQN06uR/7pZbzArH5QI6dwZGjQI6dADuuw+44AKgShXg2mv9FclrrwF16nifccMGoFcv4K67gN27vfWmTwdGjwbuvx9o2VJds3SpOnf8uLp+9Gjg+edV2VtvAaVLA999p46ZgcaNgRdfBObP97bLDPToododMwa49FLgiSeUggCUvHv2AM89B7RtC6Snq3t06WJ+FiJg/Pjgf9/iYTqAB91RS1cAOOFOjRQYZi62rXTp0hwN/5n9H059MzWqawUbk5/PfO+9zEuXessmTmTeuDH0tQBzixZFJ1ukbN7M/Pff5jKXi/ngQeZTp5jHjFEyA8xvvsl85ox/GydOMB8+zNyxo6r3ww/Mp097z8+dy3zBBao9TU4Oc26utUzDhjGvWuW9ry+ZmcyPPeZ//tgx5uefV/8Xq2sPHvSWr1un5D5xwltmtS1YwPzGG8zvvMP8zz/e8u3bmT/7zFz3nntUuxs3Mvfr59/WhAnM+/er/fbtzedq1PC28f776m9gPD9vnpJ19Ojg8t5+e+Bz6en+ZY88EuzbERIAORykbwXwLVSminwoK6E3gL4A+rrPE4ARALYBWAugVbD22P1fjXnl0H9ufy71eqmorhVszLZt6itap463LFBH5ks49Q4cYH7ySea9e5n/+kt9Gunbl/nTTwNff+yY/zXMSqmNHGnulK3kefllVVahgnUn8/DDSgEwMzscqmOrVo25aVNvnSeeYL74Yubrr/eWzZ9v7vRbtVJtLFnC/O676njHDv/7MTN/+CFzo0bMb71l3emdOaP+ZgBz587ecy+9pBRXXp71s7zxRvDO1thJt2rl3a9b17/uPfeoz+Rk5nbt/M+XLcvcqVPw+9WrF905vdWpoz6bNfM/d+GF1tccPBj8+xiEUMqhKLZivVm0yuGV317huMFxUV0r2IDPPlOdmS/r1nl/WOvWqbJwOv2TJ731jG/gx48zz5yp3grvvJO5dWtV5+671WfDhuZ2dBvGTj49XXXaeXnMNWuypxN//nl13vjW/Prr6u174UJrubUFEGpjZl6/Pry6APPgwf5lRmUBMPfu7V9nwYLQbffp47UmdAept549mZcvD1/OYFv79qqTD1WvfPng5594wro8NbXgMt5/v/rfWP29rbapU4N/b4MgyiEAA+cNZAwCO13OqK4XfJg+PbCrIRCHD6u3YZdLHZ88yfzf/zKvWFEwWfSbJpHqWLdtU+VPP8383HPmH5fT6d1nVrI4HP5taosDYF60yN91EGzbulW9kX7yibfM+Man3RS//+5/rcvF/Ntv3uMePZivuspcx+jyMb7tB9uYzQom1FapUug6l18efnuRbFrhhto+/jjwuTp11P91wgRz+YAB1vUnTmR+8cXAf7v9+5k3bPCWNWgQ2TNdeCHz2LH+5f/5j2p/3jz/cxMmMGdleY/HjSvQz0SUQwDe+OMNxiBwniMvquvPaebNUx0Ws+qIZ8xQ//Znnw3veodD+bK1+b55syp/6SXvF3/dOtUxfvYZc3a2Ov/tt8wtW6oOnVm5Wnbv9rZ79Kj6tDLLjx+3/pG+8or5R3/ttcy1a6s3uE8+UfdwOpVvX9cbOtT/Dde49erl3S9bVv3gfev8+adq29jB+HZcgHq+JUu8x0a3i94GDWK+6CKlbC++OHTHVLeuslZmzoysQwu1JSeH/nsU1mb1P7ayUsqVU2MIO3Z4vyfG587NZa5YUe3fcou3PDNT1f3zT+YbbjC3afwe67IePcx1brhBvf0nJlrL36SJ9TjCu++qto0WLsCckKC+w8zm5y0AohwCMOSvIYxB4FN5p0JXPts5fly9CQXim2/Uv/XYMXVs/JF07eo9bthQvf2H4u23zV/8RYtUed++3rK5c5l//lntP/WU+b7aJ3/FFep43jzmF15Q+6H8wsG2UaP8yypWVAOyRh/+rbda+631pv9egHqjbNTIup7vW+v77/vXOf98s0Whn9lqe+AB6/Jy5azLfQdlly0zH1uNEQTq6Fq0CHze5WLu3z/6/4vVZuU+27rVv+zbb/2/f/v2mWXbtEl91/TLyauvmuuvXWtu04gu+/RT737lyl4r2ugeqlLFu9+0KfPOnf7yjh2rrjtwwFzesqX/Pf/5J/RvLQgloRxsE8oKQMJZ33lHhRXWCDLr/aOP1Ofmzf7nvv/eu79tmwo1XBcime7PP5uPMzJUqJ5xIlBcHLDfHRX355/me2/bpj4XL1af11+vnsOq7Uh45BH/smPH1MSj48e995oxwxz66MvFF3v38/OBLVus6w3xmYRp1ebWrcCUKd5j/cxWBApvTEuzLt+503zcqpX5uEMH6+v++1/zcYUKKiwzEERA5crmsqws7/cqGlq29C+rVs18/McfQNeu/vVSDVmqiVTY6c03A6VKqbIGDcz1zzsvsByLFwMTJwKNGnnLPv8cSEpS+8bQ22XLVOitpnRp//YqVlSflSqZy5OTgz+HXShOTRSt5TBs0TDGIPCxU8eiuv6s4Ngx/zeirCzmtm2ZJ09W4Y+XXcbcuLE6//vvqo6uv2uX9Vvdl19a32/oUOZ//Uv5W+++2xuhYrX9+ivzBx8EPp+ZWbA3zy1bwqtnfNsDzG6oQNvBg4HdCeFut90W/bVt25qPje4S41a1qv//33hsfMPW29tvq3pvvsn844+qbMoU6/BPQFlOzP7jAfn5qnzJEv/Q0H//23x8551mK+bdd1W4bXy8uZ7L5d3fsCHw9z4/3/zMmpwcJafvmJPRJdm2rXWbxt+CHuNiNlsUzMxz5qj9Zs2Uu9T372UMTc7MVL9DQI0laXTdPXsCP2MYQNxK1gxfMpwxCHzwZPShYDGPHujNz1c/Zt/oHV+z1uViHj7c+yPwHYScPl1dp4/fece6Q7j2WmWKax8psxo0Ndbp18/8I/XdZsyw9tVH20H6bi6XeTA60LZxo/l4/Hjres884913OFRHc+ut0cm+cKF5jCPYNnIk86WXeo8vv1x1TocPe8vC9fkzmwdhjT71efOYR4wI/F178EFVr3lzc5tffKHOf/GF/700xgF3QCkbQEX/dOnCfOQI86RJqqxJE+91s2aZ3UvG72Yowq3HbP6eZmUFrpeT4/3NacaNM9/rr7/U/kUXmf++ektPN1//00+q/Oab/WXXY2xRUhLKwRZupcT4RABniVvp9Gnr8gsuAJo0UVPzX3kFuPdeVe5yKddPZqa5fqdOwJo1aj8nR5ncRrRrJdR9//oLaN5cuRvefRe44gp/E7p6dSAhSHb348eBRYv8y19+OfA1Roxmfvv26rNCBfW5caN6trg4oG7dwG1cfrn6G/7vf96yJk2s6z72mPfvFR+vnvfEidByDhzoX3bVVdZuhI4d1WezZt6yvn2BFSu8bsHx45VbxOjGMbpJ7r8fGD7c3O6UKcpVBqhZ0pr4eO/+ddep2byh6ONOz1O3rnKTaVed1fNotCtFo+vWqAFMm6ZcLLqM2VuvUydg9mzztZ99Zp16oiAYv6dlywauV7q0/2/G1/WTkqI+icx/X43v3yI3V31a/f2s3FIxji2Ug63HHHJyvGkctm5VX5Lx44H164GePZWvfuVK5eveuNF7nf4y9umjOu/ly83tzp6t/KWA6nCOHDGfP3oUWLjQe3wsWDZfNy+8ACxZ4l9evXrw63r0AP7+27+8XbvQ9wS8/ufq1YGL3CvNPvKI6lwuuMBbT/+YL7vMv41evdTns4Y1p6z8919/rdrculWNj2gOHgwtp29no5WV7kSMlC9vrmNk2jSge3dr+Yy++IQEpaw1v/wC3Hmn2Rdu5P33VcK3UAwdqv7XffoATz6plE0dQ9qdYMrB6NP/8ktvXWPnqctcLv/rjR1ynz7ANdeEljcafDv+cPDNL6WfKVBb+gVGo8dBjH9L33M2QpRDYTNokPoyOZ3quEwZ7xvs+vXqc+JE1TmMGQPUrGk9QLhxo+rovvxSHfftG5kcx46Zf3jDhgX/0QdDd3SRkJys8uaEg1Y+eXneH5HVm1ac++tq1a5+NuOPsHx51fkZ6dlT/X8aNFA5fTTBLCONUTm8/TYwZ47at1IOukOpWtX/3OWXAxMmmO85ebJSsL5vr8Z73nSTf1vGjujZZwMPTBupXl3Jn5SkLJPmzc3nw1UOvXp5n91KORgtB83evSpPUlGyaBGwY0fk19VyZ7C+7jr1qeUPpBwSE83Ht9wCjBjhH7wQrI0YRpRDYaMjcQ4f9pbpSBP9ZcvPt85k6cuwYd79cNIEG/F1QwHeqIxICfetZ9MmlbUSUD+cFi3Mz6DZvNnrLrntNmUlVK4MfPih90dkdU99zko5GF0ARu67LzzZp04NXcfYUb/wgteqsVIO+uVAR7KE6hzuvhu48kqzcnjkkdDZUtevBw4dCl4nUoIpB1/rSf+fwlUONWoEdvcFI5T1auSKK4B69SK/R5MmwPbtykIDgKZNlfX72WfqePZslWDvzTdVJJwvcXHKnWd8sakVdMmEmMZWyiHfmV/CkgTB4VAmvf7xbN0K9O7tPe9yKVcPoL5kwd5srLJc+hIqHbCvmwlQIYma336zvu6pp/w7O/1jD9bBlSunwgwbN1bHTqeq//TT/nUbNVJv9Mwqq2ZKilKm//qXt0OJs/hqBrMcrDpoQHUUPXsGltsoUygC+bCt7q1dKtovHY5lYmyrfXtl2YRSDmXLhm+hhUsw5eCrtK3+X8HcStGQleUNiS5q0tK8z1iqlMrU2sa9mmbHjup39/LLwO+/h9fe6tVmd7GNsJVyiGnL4fXXVQ537bu+9lrgq6+855ctA74JtLyrD0OHKrM/EI8+6n2bAVQaYgCoX99bNnFi8Htcc411h/j66/4dWTjuqHfd622UK6c+86NU5LpDCaYcrGLZA8mYmKjGGaKhWzfzcSTKQb9d9ujhlSMcdD1tKWrlUJxuiXAVGeD9fxktB925WlkO0VC2rC0HdAEoy9E4bmYjbKEcEuNsEK3kO2DsyxVXmAdAA1GmjPKHv/CCykNvZP58pThef908GKZN6EgWF0lMVLnofSlXzr9jDtTxzp3rX0dH3kSrHIL5eXWZVUcRyHLQzJ8ferDWd/LXt9+aJ7IFUg5WnWnjxurtsk4dZY0Z1woIhu5ktXKIi1MDzatWhXd9YWD82+vIqEA0bao2o/tQP0NhKQehRIjgFaHkiHnLYcYMs8umINxwg9fv/OCDykerByLbtfNGABnHIPQA2osvev2loSBSg9yjR6sIpSuvBN54Q53zVQ76TTAhwdzpt27t3dfKoXZt/3sNHqzcRr5hmVYE61C0XNpnXaaMsiL27TMrsF9+8Xd/hBM5tWaNGh+66ipvmXYp6PuFi5aVKLLZxfo5jArQGIFVnLRqFTgySlO6tDfQQqOfvbDcSkKJIMohErZsUQOc333ndcn8/bcaVC0sjBYBkTdePli9GjWif0vTg9RXXeWdYxDIcvjrLxWG+/HH6tjYAQdTDq++qj4jUQ5WloOWq149FSKckqLk3rdPRTpprKJ6wqFGDbVNn25t+UQb7RUJrVopZWqVHqS40IPogdJsDB4c3M1Tt64ab9PBCYItEeUQCY8/ruYkzJvnVQ564kthESgHi+8gdSg3Srhol4jxrTiQcmjTRm1aORijn3Qd39hvI+PHh45U0dEdvrl3AK/CYPZ2To8+qlw/wSbIRUogZR8s2mvaNKBhQ+88jWgh8irTkuL881U4qFVOJCC0fHFxajlQwdbYSjnku0o4WknPomVWbpIdO6zfoOrUUevLRoOV6yLYZCI9kzoQTz+tQkQ1EyYA//zjPfYd+AQCu5UCyQCYI5oaNFDJ0Xy5//7gsgLAM88o68MqBNXKXdGrl4pyijZMNxKC3aNLF/V52WVqUqLdMU6+E85J7DEgHQvpM5i9HWBmpprc1KaN9czjcN9iP/7YPKgLWFsORNZuFofDu1C6xjdE1fe67t29czEArzvGqByuvtp8TTjuFGOdbdvUZKBoSEhQUUJWz6utDt+/UXEohnDvs2CBf+oSQbAhtlAOhe5WOnLEO/M4HIYOVW+t2g89YID3XEaGf/1wJ+D06qUGoI0Yc/GEIj7evxNt394bDjl+PNC/v3fA2gor5fDNNyr0VhNMOeh7FYc/ftQoYObM8OYkFAXhKIfk5OhmlAtCjHFuKod771UDftu3q+Nu3ZRrQpOVpQY5NTraxCrv0N69/mW+6QiMGDsY39j36dNV7pyCMnky0LatshKqVw8ewqnHTIxhmqmp5vUCgsXoF6dyKFNGpSgoKRITlWXmG50jCGch56Zy0EpB89135sVXLr1UDYwuWaIsDKOi8MUqPUTlyuZJQTVrerOFGpWDb6ZHXysiWrp0UYunGDOPBsLKcvAl2ASs4lQOJQ2RssyaNi1pSQShyLGVcihw+ow1a9QPXK/iFSgOW0/Vv+KKwCGFDz+sPq1SPZ93nsp3o0MBhw1TcxYA89yAcAd+i5JwlIMvixd7cxFp5RDuDGBBEGyBrZRDgS2Hn34yH+flRT8/IJjrqGxZlVNH5+Z3uZQlMm8eMHZs4OusUkYUFsOHq8FSX6JRDm3aALffrvb1BK1gufMFQbAdtlAOOn1GgUNZfV0f2dnRh5zWrAn85z/W53RHqS0B7de/7jpv7qHi5skn/aOQgODK4Ztv/FNe+/LSS0r52TX3jSAIlthCOSTFKz99njMvRE0LjJaBb9rrm28OP7LIN++Oy+VNNueLDrX0VQ5A7LlfrAakNT16hJ7VHCjM9mzi3Xejn3UtCDYlpHIgojpENI+I0oloPRE97S6vSERziWiL+zPI1NiCUSpBdbIRK4fbblNzDrKz1bFOma3xXfMgM9N6Oc177gG++ELtt22rlou8++7A9/VNLW1UCJFkvCwOtNILNDNbUBZiuDmrBOEsIZyeygHgP8z8DxGVBbCCiOYC6AngN2YeQkQDAAwA0L8ohIzKcmD2ZpRctUql0LZa48BI7drWK2lVqOCNMkpJUau9WXHNNWp+QMOG6vjVV5UbyRgmW5TjCtHw++9qs+EyhoIgFB0heypm3s/M/7j3swGkA6gF4HYAY9zVxgC4o4hk9FcOvknRVq5UycL271dplnv2NFsJ6enq09dy8CUnR8018KV0aW84qF7hy4rbblPrRWv/fenSyicfa9aCkQYNSjbJmyAIMUlEr7FEVB9ASwBLAFRj5v2AUiAALBbLLRz0gHSeM0/Nkk1KMs89+PxzlcZi0iSVv2fMGPNcBr0SU3a2WunJOMErEMb1l1NTvR18sOU6o01RvGmTed0AQRCEEiZs5UBEZQD8AOAZZg578QIi6kNEy4louSPSdZC9bSAxLhFXjpmnsnAC5mU209L8y3bt8u7ruQgnT6pVmXwX0bEiN1ct0gIo5aBXcwq2JnG0yqFxY/O6AYIgCCVMWMqBiBKhFMN4Zp7iLj5IRDXc52sAsFzlnJk/Z+ZWzNwqoQDulfOzEnDTaMNKasaOWC8baVxn1mhZnDqlXEpr1yqXj3GW8jPPWK9z/OyzXishNVWNR5w+DTz2WGAhZXETQRDOEsKJViIAXwJIZ+b3DaemA3jIvf8QgGmFL56Xejk+isW48pqO1V+0yFu2f7/6LFtWderNmqnxgNRU8+Br+fL+4aWlSqm8RHp8QSuT5OTgYZvBxiMEQRBsRDiv8lcD6AFgLRGtcpe9DGAIgO+JqDeA3QBCLCxQMMq4fETVKTAAb6y+MRpJWw4VKyrL4eBBd0M+lkPlykppGNFzI3RnH67FI5aDIAhnCeFEKy1gZmLmi5m5hXubxcxHmbkDMzdyf1osbFB4pPoqhyeeUNFFgHmJSI2vctAkJpqVQ7163s5fzyDWnbxWDsES12luvDH0bGJBEIQigIhuJqJNRLTVPbXA93w5IvqJiFa756s9HKrNGAu6D0xpl7uD7m+YSrFyJfDDD14Xk3GJSqNyWLjQW56b668ctKXgO0s4EuUwZ45aAEgQBKEYIaJ4ACMAdALQFEB3IvJNHfwkgA3MfAmA6wC8R0RBFyiJ4QB8Mx7l8NBDavEdQE1sA9Sat0lJKmpJr8KlV2jznfkbTDno+Qn6WA9IB1MOCxYAW7ZE/kCCIAiFQ2sAW5l5OwAQ0USoeWgbDHUYQFn3GHIZAMegJjgHxH7KwWom74EDqtw4dpCZqcp802Hk5poHoM87z+tG0snj9HHt2urTN6+Skauvtk5oJwiCUDzUAmDMIJoBwDc2fjhUENE+AGUB3MfMQQdJbeNWSjEqB980GCdPKmugWjVvWW6uSnWhxyU0Zcv6RxxpS0FbGfr49deBCRMk6ZogCCVNgp4v5t76GM5ZhVD6rkVwE4BVAGoCaAFgOBGdF/SGBRC2WCntcD9/qVIqVcb69Wqltl69VPnRo2o1t6VLgTvuUGXJyebBaMDrkmrUyLsAj7YUfF1QOqS1sFm48NxYOU0QhMLCwcyBUjtkAKhjOK4NZSEYeRjAEGZmAFuJaAeACwEsDXRD2yiHFJfbyNGdatOmatPKAQBq1FCL0CQlqQimlBSzchgxwjthbvNmb7mv5VDUXHVV8dxHEIRzgWUAGhFRGoC9ALoBuN+nzm4AHQD8RUTVAFwAwGe9ZDO2UQ7JToPlELJyslIOycnedN27dqn03Vb4jjlcdlnBhBUEQSgmmNlBRP0AzAYQD+ArZl5PRH3d5z8F8DqA0US0FsoN1Z+Zg6aptpVycBEQF86EtORkFd6akgL873/Ac88pqyIQWjnExyuXz4UXFo7QgiAIxQAzzwIwy6fsU8P+PgAdI2nTNgPSpZyEvIQwVx3TrqfkZDX+sH178BXYtHKIi1Mun4oVCyyvIAiCnbGNckh2sFIO4aDHEJr6zgMJUT/WFuIRBEEoIWzTGyY5GLnhOsH2uEN+b701vPpGy0EQBEGwkXLIZ+RaWQ46+6oV9eqF17hOeyHuJEEQBAA2GpBOcjJyrbJYVK8OrF5tHcVknBQXjAEDgFq1imZOgyAIgg2xjXJIcAL5geyciy+2Lq9SJbzGk5KA3r2jkksQBOFsxDZupXgX4IjznREeggKsPCcIgnAuYx/lwICTwlQO7duHl2ZbEARBsMQ2r9bxDOT65ZIKwK+/esNTBUEQhIixleXgIIDD6fSJJCxVEAShANimB41zAc44wBU8BbkgCIJQCNhHOTDgJCDflV/SogiCIJz12EY5xLsAFwEOV9CV7QRBEIRCwDbKIY4Zzjgg3ymWgyAIQlFjG+UQ71JuJbEcBEEQih7bKIc4ZrhkzEEQBKFYsI9ycEcrieUgCIJQ9NhIObhUtJKMOQiCIBQ59lEOLJaDIAhCcWEb5UBOGXMQBEEoLmyjHOJcLNFKgiAIxYSNlINL5jkIgiAUE7ZRDsQyQ1oQBKG4sI9y0NFKMuYgCIJQ5NhGOcQ5XRKtJAiCUEyEVA5E9BURHSKidYayQUS0l4hWubdbilZM5VaSAWlBEITiIRzLYTSAmy3KP2DmFu5tVuGK5Q85XXARkOfMK+pbCYIgnPOEVA7M/CeAY8UgS1BIopUEQRCKjYKMOfQjojVut1OFQJWIqA8RLSei5Q5H9C4hPSAtloMgCELRE61yGAmgIYAWAPYDeC9QRWb+nJlbMXOrhISEKG8HkIvFrSQIglBMRKUcmPkgMzuZ2QXgCwCtC1csf8gdrSShrIIgCEVPVMqBiGoYDu8EsC5Q3ULD5RS3kiAIQjER0s9DRN8CuA5AZSLKADAQwHVE1AIAA9gJ4LGiE9Eth0stEyrKQRAEoegJqRyYubtF8ZdFIEtwnE4ZcxAEQSgmbDNDGk5xKwmCIFhBRDcT0SYi2kpEAwLUuc49aXk9Ef0Rqs3ow4eKG/c8B6coB0EQBA9EFA9gBIAbAWQAWEZE05l5g6FOeQCfALiZmXcTUdVQ7drDcmAGuVyguDixHARBEMy0BrCVmbczcx6AiQBu96lzP4ApzLwbAJj5UKhG7aEcXC71GR8vykEQBMFMLQB7DMcZ7jIjjQFUIKL5RLSCiB4M1ag93EpOp/oU5SAIwrlJAhEtNxx/zsyfu/fJoj77Xg/gMgAdAKQAWEREi5l5c8AbFkTaYsNjOSSIchAE4VzEwcytApzLAFDHcFwbwD6LOkeYOQdADhH9CeASAAGVgz3cSm7LIU4sB0EQBF+WAWhERGlElASgG4DpPnWmAbiWiBKIqDSANgDSgzVqD8vBrRxILAdBEAQTzOwgon4AZgOIB/AVM68nor7u858yczoR/QJgDQAXgFHMHDSzhT2Ug3YrJSRIbiVBEAQf3GvqzPIp+9Tn+B0A74Tbpr3cSnHiVhIEQSgObKUcKEHcSoIgCMWBKAdBEATBD3soB/eYQ1x8Is44zpSwMIIgCGc/9lAObsshITEJp/NPl7AwgiAIZz/2UA5uyyEhoRROO0Q5CIIgFDW2Ug6JCWI5CIIgFAf2UA6s0oQkJpTCqfxTJSyMIAjC2Y+tlENSfJK4lQRBEIoBWykHGZAWBEEoHuyhHNxjDknxpZDvyofD5ShhgQRBEM5u7KEctFspoRQAiPUgCIJQxNhKOSQmJAGAjDsIgiAUMbZSDtpykIglQRCEosVeyiFe3EqCIAjFga2UQ2KiWzmIW0kQBKFIsZVyKJWYDEDcSoIgCEWNPZSDTp8hbiVBEIRiwR7KQY85JEq0kiAIQnFgK+VQKkHcSoIgCMWBrZSDTIITBEEoHuypHMStJAiCUKTYSjmIW0kQBKF4CKkciOgrIjpEROsMZRWJaC4RbXF/VihSKT0D0ko5iFtJEAShaAnHchgN4GafsgEAfmPmRgB+cx8XHe5Q1vj4BCTGJYpbSRAEoYgJqRyY+U8Ax3yKbwcwxr0/BsAdhSuWnxDqkwilE0uLW0kQBKGIiXbMoRoz7wcA92fVwhPJAoNySElMEbeSIAhCEZNQ1Dcgoj4A+gBAUlJSdI0YlUNCiriVBEEQiphoLYeDRFQDANyfhwJVZObPmbkVM7dKSIhSF4lbSRAEoViJVjlMB/CQe/8hANMKR5wA+LqVDJZDVm4Wjp8+XqS3FwRBONcIJ5T1WwCLAFxARBlE1BvAEAA3EtEWADe6j4sOrRzi4pRbyTDmUO3daqj4dsUivb0gCMK5Rkg/DzN3D3CqQyHLEhh3KKt2K2WeyfScOuM4U2xiCIIgnCvYaoa0lVtJEARBKHzspxwSJJRVEAShqLGdcpBoJUEQhKLHdsohJSEFe7P3YtD8QSUqkiAIQqxARDcT0SYi2kpEAdMZEdHlROQkontCtVnkk+AKBYNySHZnZh38x2CUTSrrqeJ0OREfF18S0gmCIJQYRBQPYARU5GgGgGVENJ2ZN1jUGwpgdjjt2styiIszRSc9P/d5z/6of0YVt1SCIAixQGsAW5l5OzPnAZgIlf/Ol6cA/IAgk5aN2EM5GEJZs/OyLav0ndm3GAUSBEEoVhKIaLlh62M4VwvAHsNxhrvMAxHVAnAngE/DvmFBpC02DG6lQMpBEAThLMbBzK0CnCOLMvY5HgagPzM7iayq+2M75ZDvzC9ZWQRBEGKLDAB1DMe1AezzqdMKwES3YqgM4BYicjDz1ECN2sOtZFAOw28ZXrKyCIIgxBbLADQiojQiSgLQDSr/nQdmTmPm+sxcH8BkAE8EUwyADZVD3XJ10aJ6ixIVRxAEIVZgZgeAflBRSOkAvmfm9UTUl4iiHoy1nVsJAE7mnQQA3HHhHZi6cWoJCSUIghAbMPMsALN8yiwHn5m5Zzht2sNy0NFKcUpcp8sJAGhQvoGnSlr5tGIXSxAE4WzFlpbD1G5T8fXKr9G0SlNPlTxnXklIJgiCcFZiS+VwcbWL8cHNH2Bn5k4AQMMKDZGVm4U8Zx6OnjqKGmVrlJCggiAIZwf2cCv5KAdN/fL1wQMZtza+FXnOPDz444Oo+X5N5Dpy8dofr+HDxR+WgLCCIAj2x5aWgy9J8UnIc+bhu/XfAQBaj2qNNQfXAACevuLpYhFREAThbMLWloNGKweNVgyR34Zx/Zjr8WP6j1FdLwiCcLZgL+UQZy1uUnwSnOws8G0cLgfm75yPu76/q8BtCYIg2Bl7KAdD4j0rkuKTLMsbVGhgWR6IXGeuuo1lqhJBEIRzB3sohzDcSgBQNbWqqTzXkRvRbXT9OLLHn0UQBKGosEcvGEI5JMYlAgCuqnOVqVyv/XA6/zQu/exSzN8533OOBhOe/eVZAMCqA6tAgwkbj2wEIMpBEATBHr1gCOWQk58DQM13MKKVw7pD67DywEo89fNTpvPDlgwD4F0oaPKGyQBEOQiCINijFwyhHI6cOgIAqJZazWNFAF7lkJGVAQAonVja3Zw51bkeY9D1RTkIgnCuY49eMEzlUKl0JU/HXimlEpzsxEdLPvJEH2nl4BvZpBe/OO04DUCUgyAIgj16wRChrK+2exU3NLgBdze521NWJbUKAOC9Re95yk7nq87fd8EgrQyKSzn0+LEHnv5ZJucJghC72EM5hAhlbVChAeb2mItyyeU84aj1y9cHAFQvU91Tb+uxrch35psmzOU78z1uJa08ilo5jFszDh8t/ahI7yEIglAQ7KEcQriVrNCD0w6XAwDwxvVv4Ojpo5i3cx7yXV7LISs3S9xKgiAIPtijF4xCOegJcBlZGbixwY14uOXDAIAdx3eY3Eonck943UrFZDkIgiDEOvboBaNQDjXL1gQAHMo5hCqpVVC5dGXPsdGtlHkms9jHHARBEGKdsyIrq5Ffe/yKv/f8jZSEFE9Z65qtkRSfhArJFTB69WjTeg8nzpzwjDn8s/8fAEB8XHwhCi8IgmA/zjrl0KFBB3Ro0AF/7PzDU9bnsj4AVHqNTUc34dGfHvWcO5F7wjPmoIkmt5KLXWJxCIJw1mCP3ixEKKsVtc6r5dlPSVRWhG/uJUBZDr6duj7Oys3C2wvfhotdQe+1cPdCxL8WjwW7F4QtnyAIQixTIOVARDuJaC0RrSKi5YUllB8hQlmtqFHGf6lQPe5gJPNMpp+lsDd7LxbuXogX576I/r/2x/RN04Pea+72uepz29yw5RMEQYhlCsOtdD0zHymEdgITxYB0alKqX1n55PJ+Zf1/7e+ZG2HkyVlPommVpgCAnLycEOKxWzxJ9S0IwtmBvdxKUXS+xjUdyiSV8TvvqxjSyqd56uqB6WALCc3ZNgcfLlFrVTtdTuzK3BWxjIIgCLFGQZUDA5hDRCuIqE9hCGR9l+iUw/H+x7Gmr3fJUGNSvkDocNaFexbip00/AVCd/pFTR0CDCVM3TjXVv2ncTTiRewIA8MZfb6D+h/WxP3t/RHKWJIdzDgd0h609uNbzNxAE4dyioMrhama+FEAnAE8SUVvfCkTUh4iWE9Fyh8MR3V2iVA7lk8ub3EvhhKieOHPCu+/u9J3sxLpD6wAAHyz+IGQbOhGgHeg0vhM6juvol28KAC7+9GJ0mdilBKQSBKGkKZByYOZ97s9DAH4E0NqizufM3IqZWyUkRDnEUQC3kpF4MiuH1rX8xPVYDqay/NPecYWzbAnR9YfXA/CmKxcEQQAKoByIKJWIyup9AB0BrCsswUxEEcpqRUKcWTl92eVLLH/UHGR1Xf3r/K7Lzsv2hLMaB51914XwlMO6PNA12bnZOHrqaMBrihL9NzmVf6pE7i8IQmxSkN62GoAFRLQawFIAM5n5l8IRy4coQlmtuPn8m03HiXGJqJBSwVT2U3d/H/usLbM8CwYZLYes3CzL+xjTc/hiNWei0ceNUPkd/zDb4kArByuLSRCEc5eolQMzb2fmS9xbM2Z+szAF87mZ+iygcri67tXI/2++J413YnyiKbx10r2TLCOaFu5ZiJ7TerpF8MowZMEQy/vkOswRUDSY0GtaLwDeLLFGDuYcjOg5omF/9n7LkFyPcsgX5SAIgpezPpTVl4S4BCQnJLubZZxX6jzPuXua3hPyem055DnzMGRhAOVgMW/i61VfA7BWDsVBzfdr4qqvrvIr1+Mw4lYKjzL/K4M7v7uzpMUQhCLnnFMOANCqZisAQFJ8kt84RChW7F+BZ355JuisaV/LwUhJKQcAWHNwjV+ZuJUiIyc/xy+cWRDORs5J5TD69tH4tcevqFOuTsTXZp7JxIdLPsS9k+4NWEdbDoszFoMGm2UuSeVgRThuJd9xEofLYRn6GoohC4ZIihFBsAnnpHJITUpFhwYdCqUtK7TlMOqfUX7njLOtfSOXAkU/FZRg7YZjOfgqgiu/vBJJbyQBUBMEj58+HpYcL/32EjqO6xhSVklgKAglj72UQwFDWSPl4PMHUTGlYsTX6TkDVn58o+Vg9UZeFARrV08MDDbm4Hv98n3e8N8X576Iim9XRHZudgGlVHy24jNc+/W1+DH9x0JpTxDOBYjoZiLaRERbiWiAxfkHiGiNe/ubiC4J1aY9lEMhhbIGYvu/t2P7v7d7ju9rdh8S4xJRNbUqRnYeGfTaSimV/Mq0W8m3w31u9nPYcHiD5/hE7gmsP7Tec1xUE9GCtRuOWymYchm/djwANRekMNh4ZCMAYGfmzkJpT9NpfCdcN/q6Qm1TEGIBIooHMAIqU0VTAN2JqKlPtR0A2jHzxQBeB/B5qHbtoRwK2a3kS1qFNKRVSPMcT7xnIvL+q+YqdG3WFS2qtzDVNw5if9zpY7/2ch252HRkE6ZtmmYq/2DxB7h94u2e49u+vQ0XjbzIe51FlFOkMLOfG8nY7qGcQ9h0ZJPnOCy3kivw+IK2fkKteVHS/LL1F/yx64/QFS1YfWA1pqRPCavuxHUT8cwvz0R1H0GIktYAtrqnF+QBmAjgdmMFZv6bmbX/dzGA2qEaFeUQBl91+QqvXPsKbmp4EwB4UnkD3rWqjeQ6c/H3nr8t2zJaE751thzdUmBZ416Lw03jbjLLY4ieavRxI1w44kLPcTgzpINZDno2uPEeP6b/aHI9zdk2Bz9v+Tks+WMx/XmLz1rg7u/vhtMVODuvpvsP3T1ZegWhmKgFYI/hOMNdFojeAEL+IM+6ZUKLgpY1WqJljZb415R/AQAebvEw6pxXBw0rNvTL1wSojvLY6WMR3+eqr67C7w/+juvTrjeVNx3RFO3T2mP4LcPDamfu9rl4/Y/X8c2ab7DlqS0my8F3VreWPxq3kotdHovBeI+7vr8LAMAD1f/NV1nZlWAz3wWhiEnwWVDtc2bWriGrjtEyCoWIrodSDteEvGHEIpYEJawcNHXOU6GvmWcy8cwVzwCw7lRznbmedBuR8veev/2UQ/qRdKQfSQ9bOQDAq/NfBaA68GDzLnTnHkm0kibXketVDkHuEQ2xmOAwmHtNEIoYBzO3CnAuA4AxLr82gH2+lYjoYgCjAHRi5pDJ3MStFAGdGnUCANMYREpiCnY+vROl4kt5yg7lHMK4teP8cjmFw8m8k2HXfWT6I5iwdkLQOsdPHw84lsHMHkvC6FY6lX8KK/at8BwHshzOOM74WQ7huF6CESppYUmm+RDLQYhRlgFoRERpRJQEoBsA0yxdIqoLYAqAHsy8OZxG7aUcijmU1Ze29dri4PMHcceFd5jK65WvZxqHWJyxGCfzTuKJVk9EfA9f5WA1F2LS+kk44ziDL1d+iQemPIB3/34XNJjw564//dr7If0HtPyspeW9PlzyIbYcU+Mcxk73oakPodUX3peUQOG3px2n/SyHwopasuK9Re+h9P9K41DOoSK7RzBCKaZYm+BYVHQa3wldJ3UtaTEEN8zsANAPwGwA6QC+Z+b1RNSXiPq6q70KoBKAT4holY+LyhJ7uJWKOJQ1EqqmVg1ZZ/XB1QBgioAKF925MjMY7BeG+s/+f9B1clfP4DgAvDD3BQBAu9Ht/Np7bMZjlvdxupwmq8PoVlq4e6GprtGdYuwg526b61Fm2nIwLpZUEKwGpMeuGQsA2Ju1N6z/Q6Rk52bj2OljmLllJvKceR7XoSaUVdfjxx6efafLGdbiUnbkl61Fk3xZiB5mngVglk/Zp4b9RwA8Ekmb9rIcYkA5BMKY3VVTr1y9iNvRbp77Jt+H+NfiserAKs+5tQfXet7oZ2+bHZWcmjxnHlISUzzH0zZNw9jVqvP1de0Y34hz8r2ZXXWmWsBrOejV86JFW0rBZnWfcZzBVV9ehWV7l4GZseP4joB1j58+7pfCJBBtR7dF/Q/r48lZT+LZ2c/6nQ9lFU1cN9GzH+n4ROaZTIxbMy6iawShKBHlUEjc1URF6BizvJYtVTbidnRyvEkbJgEArv7qas+5iz+9uCAimshz5nmy0wJKKT049UHLuiblYJH2Gyh8y8Gqc9UKY+WBlViUsQj//uXfePOvN9HgowbYdmybZTuBQoqtMCpiKyKZBR5p7qmeU3uix489kH44HSfOnCiyVCqCEC6iHAqJJy9/Etv/vd2TbuO7e76Lqp1tx7fh4MmCr+9wTd3gkWrDFg/DnG1z/MrznHl+HZOxozNaDkYisRx2n9iNpXuXWp7TVotV56rHN/S5eIr3vG0fP+Of3ynPmYdbv701pDwAsC/bL7gDgFkZBlrcyYpILYcdmcr62Zm5E+WHlsd7i96L6PriQpTWuYO9lEMMQ0RIq5CG2uepiYc3NrgRADDuznGYdb/JFYh7m96LpPikgG1tP7494LlwqVK6StDzg/4YZFn+n9n/8evYCmI5WHUm9YfVR5tRbSzb0fWDRQbp8ZEzjjOekOH/zvuv370iedPvNL6TZXmZt7yLPwVzKw1dMNR0bFRuY1aNwZ4Te5CTl4OXf3vZMuxX19+TpeYyGWdkO11OPDHzCaw9uDaMJyk4ec483DHxDssU74Uxi7+k2X1iN2gw4bftv5W0KDGNfZRDDFsNRibdOwkzus/wLD/6wMUPoFOjTujY0JuNlIhMEU8XVb3I1MYNY28osBzRuLQAYPiy4X4T+LRyOHDygMmvbqT39N5gZj/LwSqCR1sHPX7sEfBeVspBX6fDblfsX+GxZH7Z+otHUeQ6cpHnzLMcQA705rsrc5df2aD5g0zHwQakB/xmznWm5c/Jy0HPaT3R4ZsOeGvBW3hrwVuW2Xr1c+sAhEUZi3Dk1BEAwJZjWzBy+Uh0/6F7wPv7cijnEPac2BO6ogWrDqzCtE3T8Mh0//HLSKynWEVbrSOWjShhSWIb+yiHEg5jDZfqZaqjc+POfuXTuk3D+x3fB6A6gpGdR2L8XeMxo/sM/NHTnPOnMFZlK5vkrxysFjbqkBY6dbm2JDqN74RhS4YFrLf7xG6/ziPYm+a4NePw4WKVauK37b/h2OljyHOpTtVKOWi3UuaZTMv2tKIoN6QcGn/c2PJNX3fCR08dNaUrMY6/aAb/Mdh0bLREsnKzguaT0n8zbeXsOrHL87excjlpuYyW2YBflcLRLq9IUopUe7ca6g6rG3Z9I/q54sj/N2elHE7ln7KVu0n/NkIpOofLgQemPGBKjnkuYY8e1+WyjeUQiOSEZHRv3h1J8Ul47ornUDGlIu5vfj86N+4cVVrwUJQrVc503LVZV8u3+HAsDH2d0a1hFZ317OxnPW4lAmFK+pSAobSaOIpDdm42bhh7A+767i6PUrC0HNwdUKDUJHpdiVxnLnad2GX5pq875otGXoTGwxt7yksllPKr64tR2ZQbUg7vLHwHgPXEP51VVit6p8vpqWelpLVcRstLWxEdvlEKPCUhxe86X07lnwo7Okuz4/gOk6KLRDkcyjmE1P+lYtjiYWHdK9eRG9FEz6JAP18o5bD24FpMWDsBD0x5wO/6gk72tAP2UA42cisFo3qZ6sj9v1xcXffq0JUBXFn7yqjvVa1MNdPxJdWs07efV+o8XFwteBSUw+XAX7v+Mi1UZDXP4MeNP3rW1WYw7v7+7pAzuBPiEnDg5AEAwJK9SzxKYcaWGZi/c76prnYrBbIcfJWG1ZiDbl/fU2Oc4R4I3/ZGrx4NwNrS0x26Rzmw0/P3s8rHpRWw8Rmc7DTNcymVUAr5zvygkVCR5vRad2gdGnzUAB8s+sBTpttflLHIlEUY8O9QtSvvmzXfhHW/1qNao+xb/i8k+7L3RWR9bDi8ARcOvzBgYEMwtDVnZVkePXXU8zfUlpqvhdhtcjckvG6PKWIFQZRDjLDu8XVY+/ha3H6B98eYk5+Dtzq8hQ9u+gBLHlliqv/LA7/g6jqBlYyvNWLlNgGAaqnV8GfPP7H1qa2eMj2Yrsl35qPt6LZB24+WP3b9gf0n9wNQb8p6sHb78e24fsz12Je9D1uPbUWpN0ph81E1698qMgkAvlv/nemt1OoN9amfnzJZUPp+gf4+Rj5a+pHpWFsqgSK4+s3qh8M5hz3H+r5WbiXdIR897U1542KXSRFmnslE3WF1ccHwCwLKGMkgfJ4zD81HNgcALNjjXX3POCHSd610X+WgO/Rw36SNg9yztszChsMbsOXoFtR6v1ZE2WxX7l+JTUc3oc9PfcK+RqMVtlXYdeV3KqPS2/5rtBjRYeZ2cqVFgyiHGKFZ1Wa4qOpFmNptKr65Q72FDbh6AAZcMwDPXPGMn5vopvNvwrV1rwUALO69GI0rNTadL59c3rSA0W2Nb7O8b40yNVAuuRwaVmyIGd1nYFq3aZjTwxziuuuE/2CtUTmMv2s8Prr5I786Vvgm1Ju7fa7Juvhp80+m8zuO78B3674zuZkCzV0Yu2asqS0r5TBuzTjTOtbajROOW8mXgzkHcTr/dMAIrhHLRpjGaLTlYCWXcSzEU9/l9CiH1MRUHM45jAMnD3jCXq0I1HlbsT97v2ff6LIKNuZlbJ+ZPc8SKnUIM/tZPJ0ndEazT5p5XHC+658Ew8oNFy56lv/hU4eD1tMvDoHyfQWyYM8WRDnEID0u6QEeyOje3BudUi65nF+9V9q+gqWPLEWb2m3wxW1fYPTto731S5Uzpe9oVKmR5b1uaXSLZ79z487ockEXAMBTrZ8CACTFJ3nScxgxDngnxSehcunKYT2b1RjHP/v/CVh/4rqJlr7vQBjHOAKFnt4ywfvMmWcyMWvLLKQfTg/7HkYWZywOaDkAwLK9yzz7qw+otCpG5ZDvzMewxcM8bRgthyV7l6DJiCYAVCoWo8XkdDkxa8ss0GBCi09bAABe+vUljFk9xnR/HRprhVFuo+UUrnLId+V7/sZGl6Nm89HNoMGEFftW4OXfXvasOw6YlZZ2nUWyEqKWMZBiDudah8sR1EWn5QmkYEMpF7sjysEm6AHg+5vfj13PqDf5MkllcHmtywGopIAPtXjI8wZ/QWV/14NxIPSauteAB7JlPQD48OYP4XzViUYVrZWKsTNJjEs0WRJj7vB2UF0u6ILr6l/nObYaBFy2b5lfmWb4suEYunBowPPB8B1XsCLzTCY6T+gctIO3Qv8t23/THi/OfTFgPWPnvPLASgBKOThdTqw5uAYfLP4Az85+1tJyMKZ9TyufZrKeMrIy0HmCiorTubyGLByCkcvNy9rWG1YPGw5vMLlQFmcsRt0P6poSNYajHM44zqDfrH6eY+Pgsm9yxv5z+3vGK0avGu0Zi9IYn0WH7EaS9t2jHCL8vwFmt9nuE7u95T6JFT3KwW05rDu0zjRZ0pgActL6SdibtReTN0w2tZPryC20rAHFjX2Ug01CWYuK5IRkHHvxGL654xvULRc4RLFf637I+788ywHjBQ8v8Axy+7qpfCEixFGcXztvdXgLX3X5yjSAmxif6Oks729+Px68xJuGY2TnkZj30DzP8evXv45mVZqZLBZfFvVeZDoO5Tp4te2r+Kn7T56JhTpkeOK6iWhQoYFnYqIVxgHc8yueH7CeDvnVis44byXSPFczNs9AwusJuOTTS9D/1/6mc4HeRtPKm5M4rju0znQc7A262SfNUH5oefT5qQ/2Ze/DvB3zsCdrDx6f+binzsjlIz0z5n2Vg35z/nDxhyYXS54zzzPG4XA58MvWX8DM2H58O97++23PeuBW4cxGC0m7CQOFPU9aP8nkCjTKeCr/VMRL1Bo7b6NyOJhjzkyglYhuv/nI5qj1vneBNa0ccvJy0HVyV9T+oDbunXQvbvvW68LtMrELyg8tH5F8sYI9etyzIJS1MKiQUiFkpk8iQmJ8oud4ce/FWNhLZVltU7sN5vecj94te+PTWz8N1ISJl655CYB3oaNO53fCwy0fRrOqzTx10sqn4br61+HN9m9ieCe1IFH7tPYA4HE3XVBJWSgDrhmAdU+sw8z7Z+LOC+803WvsnWPxyrWvoHWt1mHJBgDb/70dg68fjFsb34q1j6/F3B5zUSVVzQ7fcmwLHmj+gGXoqMY41yE1MTVgvda1WiP9yXRM6zYNY+4Yg7F3jrWsN+DqAZblRoKNGQTy3dcvX9907JsWRA/WB+OLf75ArfdrBbTUek3rBcD/DXrB7gU4cPKA37yPEctGeNxKu0/sRqfxnRD3Whx+2mQeN7JyFxkH6ketVJMCA71hd53cFR3HdTSVGRWYb4TWtmPbMHnDZI/lOHzpcHy2/DPLa42uOqOlyeyfEdkXrRx8LdTfdnhnXmuFa8fJg/aIxxK3UtS0qW1OU5EUn4RRXfxn6AbixoY34swrZ5Cdl40f03/0hL32a90PTSo3wfkVz/eMbbx87cue62bePxM7M3d63uYX9lqIUgmlTB31lPumYO62uZ4f/r8u/lfEz2ccV2lcqTEaV2qM2Vu9b/LNqza3nJOhWbjHm578+JnjyBqQhSrvVDG9xT7Q/AH0bNHTM+hvtIyMPN3maXRs2NHjQmletTnWHjKnvGhTqw2W7F2C8XeNx2t/vIZNRzeFfMbWtVoHdP9p1h8OPlGrznl1PC6uHzf+aFlnb/ZePPbTY/j8n89N5b6RapqB8weibT3/c8/Nec50bLX+hpWFdODkAVOq8yUZS0zuLhpMOPnSSaw8sNKkSHaf2O15Cck8k4nzP1YWYONKjbGizwo89bMaP6uYUhHNqjYzuZW0Yjl2+hiu/NIbOp6RleGJ1Np2bBvSPvRPv6+fS0fbGfm/3/8Pg64b5DneemwrLq1xqV+9WEaUgxCSUgmlUCqhFB697FFPWRzF4caGNwa8JjkhGRdWvtBzXKm0dXigcTzCyNJHlqJ6mep4b9F7eOTSR9C0SlNk52Z7TPQXrnoBy/dZr1ditGqaVGmCt294G8/Pfd4yV9B3670JEo+cOoKypcrixIATGPXPKPT7WfnXP+n8iSnbri9j7xyLzUc3Y2C7gaZkhpfWuBSnHaex9Zg3THhqt6n4auVX6NqsK75a+VVYymHJI0tMz5pWPs3P+jCuJWHFnB5zPIPbgIoas4rC8VUMobBaYMoXK7ebr8K4pNolWH1wNZqPbI5NRzeh20XdLOfIDJw/0C8p4cYjGzFz80zk5OeYxqc2H91smlPRdbJaoCg1MRXlk8sj80wmHv3pUczbOQ+9WvQytdl6VGuPRZDvyvdEVBnR1o/V2Nabf72JplWaIjEuEfmufGw+utl2yoGKM1Y3NTWVc3IiH0DC008DY8YAmZmFLpNQ8gxbPAxX17naM7geDD37lwcGX0407jXlMc3/bz4S4hLAzFi+bzneX/w+WtdsjZUHVnoWD2qf1h6/7/jdr93dJ3Zj27Ftfmt6B5MlJy8Hrb5ohazcLCzuvRgJcQlYuGchapatiYysDHRt5l1B7eFpD2P0qtEhn5kHMo6eOorK71RG9TLVUaV0FY9FUqtsLezN3huyDeerTsS/5nVJtq3X1q9jX/boMlz+ReD/wejbR+PWxrfihbkv4OtVX1vWefaKZ/HB4g/8yptUboL0I96IsHiKN0U49b2sLz5dEZ6r05cbGtyAX7f/GtE1vladr3zh0LVZV9QrVw9zts3xBAUEYsDVA/DWDW9F1L4RIjrFzIH9nkWAWA5CieO74low3urwlmd+RyCICF/c9gXqlqvrcWMRES6vdTm+vftbT70GFRpgzcE1+KHrD4h7LQ7X1zcrgbrl6gYd/J//0Hy/jjk1KRXpT5o7mXua3mN5/UVVLrIsB4D+V/dHx4YdPS6xSqUrYetTW+FiF+6ddC8AoNtF3TCo3SBMXDcRpx2n8cHiD5DnzMNdTe7ClPQpGNhuIIYtHoY2tdsgjuKQkpCC047T6HJBFwzpMARNP2lqumermq3w0jUvoX1ae9w41t8qfKjFQwCAge0GYsuxLWhRrQWGLxvuabd3y94Y2G4gLqh0AfrO7Gu69pPOn+D6Md6/r2/oa+fGnbFs3zI4XA6/jrZjw45YsW+FaRBbk1Y+zVIxVEut5jfAbKRljZYm5aAVw2e3fhYy5QugQrm/X/99yHqaIQuHoF39dlGtK19S2MNyeOopYPx44FhkqQEEIVz2nNiDiikVkZpUfC9n6YfT0fSTpmhXrx26X9Td06F+2vlTPNYqcAfVc2pPjFk9Bhuf3Ggai1icsRhDFw7F2DvHYuORjWhZvSUcLgfi4+KREJeA/dn7kXkmE02qNPFYV0+3edozM9loAV0/5nrM3zkfb9/wNl789UW/8wAwdeNU3PndnUgrn4b5Peejznl1PCknthzdgh2ZO9AhrQOOnDqCamWqYcHuBZi9dTbe+OsNv2fa+fRO1CuvVk7Md+bjp80/oVHFRqhcujJqlK0Bp8vpSVkx6/5ZnrkqXZt1teykv779azw87WG/8trn1UZGVgY+7vSxZyzCiOtVF95b9J7l3B4jNzW8yeQu69iwo8elOP6u8X75mADg4PMHo17etiQsB3soh379gIkTgSNHCl8oQShBcvJyPAqp+w/d0SGtAx65NPhSvzl5Ofhz15/o1Mh6DYpwcbELBPK44Iydv8PlgMPlQHJCMmgwoXxyeRzvb05bsuP4DjT4qAFqlq2Jvc+Fdm1pbp1wK2ZumWkqy3k5B6UTSwe9bs3BNahXrh7KJZcDDSZUTa2KV9u+6hkbAoAvbvsCl1S7BC1rtETv6b3xVOun8MfOP/D83OcBAHP+NQe/7/gd/233X0xaPwnlksth/s75qJpaFTXK1MDDLZVCycnL8azl8fmtn6PPDHOajoPPH8TOzJ1o+3Vb5DpzMfW+qWhUqREO5RxC23ptTS68iikV8WnnT3Fvs3vD/hv5UhLKQS1kH+UG4GYAmwBsBTAgVP3SpUtzVDz+OHPlytFdKwhCUFbuX8l/7Pwj4PlV+1fx/uz9fuUul4sfn/E4L9qzKOJ7ph9O5y7fduExq8Zwu6/bRXz9ydyTnJOXw6fyTvFNY2/if8/6N3+y9BPLutm52fz2grc515Eb0T3Grh7LT858kpmZRywdwRgEz5bvzGdm5lxHLq8+sNrvWgwCp7yRwpPXT+bDOYcjfDp/AORwAfrqaLaoLQciigewGcCNADIALAPQnZk3BLomasth1Chg8WL1KQiCUAIcP30ce7L2YM62OXj+queD1l19YDWqpFZBzbI1C+XetnIrEdGVAAYx803u45cAgJkDDslHrRwEQRDOYUpCORRkhnQtAMasXhnuMhNE1IeIlhPRcocjeOZGQRAEITYoiHKwii31M0OY+XNmbsXMrRIS7BE5KwiCcK5TEOWQAaCO4bg2gH0B6gqCIAg2oiDKYRmARkSURkRJALoBmB7iGkEQBMEGRO3nYWYHEfUDMBtAPICvmDl49i9BEATBFthjEpwgCMI5jN2ilQRBEISzFFEOgiAIgh/F6lYiIheA0yErWpMAwI4TJUTu4kXkLj7sKDNgT7lTmLlYX+aLVTkUBCJazsytSlqOSBG5ixeRu/iwo8yAfeUubsStJAiCIPghykEQBEHww07KIbLFbWMHkbt4EbmLDzvKDNhX7mLFNmMOgiAIQvFhJ8tBEARBKCZsoRyI6GYi2kREW4loQEnLY4SIviKiQ0S0zlBWkYjmEtEW92cFw7mX3M+xiYhuKiGZ6xDRPCJKJ6L1RPS0TeROJqKlRLTaLfdgO8jtliOeiFYS0Qy7yOyWZScRrSWiVUS03F0W07ITUXkimkxEG93f8StjXeaYpLiXnot0g8rbtA1AAwBJAFYDaFrSchnkawvgUgDrDGVvw71sKoABAIa695u65S8FIM39XPElIHMNAJe698tCrejX1AZyE4Ay7v1EAEsAXBHrcrtleQ7ABAAz7PAdMci9E0Bln7KYlh3AGACPuPeTAJSPdZljcbOD5dAawFZm3s7MeQAmAri9hGXywMx/AjjmU3w71BcU7s87DOUTmTmXmXdArb3dujjkNMLM+5n5H/d+NoB0qIWaYl1uZuaT7sNE98aIcbmJqDaAzgCM69zGtMwhiFnZieg8qBe2LwGAmfOYOTOWZY5V7KAcwlpxLsaoxsz7AdURA6jqLo+5ZyGi+gBaQr2Fx7zcbvfMKgCHAMxlZjvIPQzAiwBchrJYl1nDAOYQ0Qoi6uMui2XZGwA4DOBrtxtvFBGlIrZljknsoBzCWnHOJsTUsxBRGQA/AHiGmbOCVbUoKxG5mdnJzC2gFpdqTUQXBale4nIT0a0ADjHzinAvsSgrye/71cx8KYBOAJ4korZB6saC7AlQbt6RzNwSQA6UGykQsSBzTGIH5WDHFecOElENAHB/HnKXx8yzEFEilGIYz8xT3MUxL7fG7SqYD+BmxLbcVwPoQkQ7oVyi7YloHGJbZg/MvM/9eQjAj1Aul1iWPQNAhtuiBIDJUMoilmWOSeygHOy44tx0AA+59x8CMM1Q3o2IShFRGoBGAJYWt3BERFA+2XRmft9wKtblrkJE5d37KQBuALARMSw3M7/EzLWZuT7Ud/d3Zv5XLMusIaJUIiqr9wF0BLAOMSw7Mx8AsIeILnAXdQCwATEsc8xS0iPi4WwAboGKqNkG4JWSlsdHtm8B7AeQD/UW0htAJQC/Adji/qxoqP+K+zk2AehUQjJfA2U6rwGwyr3dYgO5Lwaw0i33OgCvustjWm6DLNfBG60U8zJD+e9Xu7f1+rcX67IDaAFguft7MhVAhViXORY3mSEtCIIg+GEHt5IgCIJQzIhyEARBEPwQ5SAIgiD4IcpBEARB8EOUgyAIguCHKAdBEATBD1EOgiAIgh+iHARBEAQ//h9ZEqdrWRdEWAAAAABJRU5ErkJggg==\n",
|
|
"text/plain": [
|
|
"<Figure size 432x288 with 2 Axes>"
|
|
]
|
|
},
|
|
"metadata": {
|
|
"needs_background": "light"
|
|
},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"iters = [item[\"iter\"] for item in result]\n",
|
|
"fig, ax1 = plt.subplots()\n",
|
|
"ax1.plot(iters,[item[\"loss\"] for item in result],'g')\n",
|
|
"ax2 = ax1.twinx()\n",
|
|
"ax2.plot(iters,[item[\"accuracy\"] for item in result],'r')\n",
|
|
"plt.show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "2bb67740",
|
|
"metadata": {},
|
|
"source": [
|
|
"학습 그래프입니다."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 47,
|
|
"id": "0defca72",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"torch.cuda.empty_cache()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 48,
|
|
"id": "2f45cae0",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"gpu allocated : 2739 MB\n",
|
|
"gpu reserved : 2910MB\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"print(f\"gpu allocated : {torch.cuda.memory_allocated() // 1024**2} MB\")\n",
|
|
"print(f\"gpu reserved : {torch.cuda.memory_reserved() // 1024 ** 2}MB\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "68f18f4d",
|
|
"metadata": {},
|
|
"source": [
|
|
"gpu 메모리 사용량을 보는 코드"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 49,
|
|
"id": "73b01630",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"del batch_inputs\n",
|
|
"del batch_labels\n",
|
|
"del loss\n",
|
|
"del optimizer"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "02a3f367",
|
|
"metadata": {},
|
|
"source": [
|
|
"한번 테스트 해보자"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 50,
|
|
"id": "af93a3ec",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"김광현 을 둘러싼 주변 의 우려 는 너무 많이 던진다는 것 .\n",
|
|
"두산 은 주포 인 김동주 와 안경현 이 빠진 상황 이 라 타력 에 적 지 않 은 문제점 을 안 고 있 지만 29 일 잠실 롯데전 이 우천 으로 취소 되 는 바람 에 시간 을 벌 었 고 더구나 삼성 이 4 연패 로 2 위 로 추락 해 어부지리 로 1 위 로 올라서 는 행운 을 잡 았 다 .\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for data in datasetTrain[100:102]:\n",
|
|
" print(tokenizer.convert_tokens_to_string(data[\"tokens\"]))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 51,
|
|
"id": "9b2cd5c4",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"{'input_ids': tensor([[ 101, 8935, 118649, 30842, 9633, 9105, 30873, 119091, 9689,\n",
|
|
" 118985, 9637, 9604, 26737, 9043, 9004, 32537, 47058, 9076,\n",
|
|
" 65096, 11018, 8870, 119, 102, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [ 101, 9102, 21386, 9632, 9689, 55530, 9640, 8935, 18778,\n",
|
|
" 16323, 9590, 9521, 31720, 30842, 9638, 9388, 18623, 9414,\n",
|
|
" 65649, 9638, 9157, 9845, 28143, 9559, 9664, 9706, 9523,\n",
|
|
" 9632, 9297, 17730, 34907, 9633, 9521, 8888, 9647, 9706,\n",
|
|
" 19105, 10386, 9641, 9655, 31503, 9208, 28911, 16617, 9638,\n",
|
|
" 9604, 38631, 29805, 9773, 22333, 9098, 9043, 9318, 61250,\n",
|
|
" 9559, 9485, 18784, 9633, 9339, 9557, 8888, 9074, 17196,\n",
|
|
" 16439, 9410, 17138, 9638, 125, 9568, 119383, 9202, 123,\n",
|
|
" 9619, 9202, 9765, 107693, 9960, 9546, 14646, 12508, 12692,\n",
|
|
" 9202, 122, 9619, 9202, 9583, 17342, 12424, 9043, 9966,\n",
|
|
" 21614, 9633, 9656, 9529, 9056, 119, 102]],\n",
|
|
" device='cuda:0'),\n",
|
|
" 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0],\n",
|
|
" [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
|
|
" 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
|
|
" 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
|
|
" 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
|
|
" 1]], device='cuda:0'),\n",
|
|
" 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
|
|
" 0]], device='cuda:0')}"
|
|
]
|
|
},
|
|
"execution_count": 51,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"inputs, labels = my_collate_fn(datasetTrain[100:102])\n",
|
|
"inputs = {k:v.to(device) for k,v in inputs.items()}\n",
|
|
"inputs"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 52,
|
|
"id": "6b31782c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"model.eval()\n",
|
|
"with torch.no_grad():\n",
|
|
" predict_label = model(**inputs)\n",
|
|
" sp = model.softmax(predict_label)\n",
|
|
" p = sp.argmax(dim=-1,keepdim=True).squeeze().cpu()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 53,
|
|
"id": "7f4d43ce",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"['O', 'B-PS', 'I-PS', 'I-PS', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-PS', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
|
|
"['O', 'B-OG', 'I-OG', 'O', 'O', 'O', 'O', 'B-PS', 'I-PS', 'I-PS', 'O', 'B-PS', 'I-PS', 'I-PS', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-DT', 'I-DT', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-OG', 'I-OG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for data in p.numpy():\n",
|
|
" print(tagIdConverter.convert_ids_to_tokens(data))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 54,
|
|
"id": "5ade3317",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"['O', 'B-PS', 'I-PS', 'I-PS', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
|
|
"['O', 'B-OG', 'I-OG', 'O', 'O', 'O', 'O', 'B-PS', 'I-PS', 'I-PS', 'O', 'B-PS', 'I-PS', 'I-PS', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-DT', 'I-DT', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-OG', 'I-OG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for data in labels.squeeze().numpy():\n",
|
|
" print(tagIdConverter.convert_ids_to_tokens(data))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "b48e22ff",
|
|
"metadata": {},
|
|
"source": [
|
|
"It work!"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 55,
|
|
"id": "383dd24a",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"torch.Size([194, 1])"
|
|
]
|
|
},
|
|
"execution_count": 55,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"sp.cpu().view(-1,sp.size(-1)).argmax(dim=-1,keepdim=True).size()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 56,
|
|
"id": "ff74fced",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor(120)"
|
|
]
|
|
},
|
|
"execution_count": 56,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"correct = (sp.cpu().view(-1,sp.size(-1)).argmax(dim=-1) == labels.view(-1)).sum()\n",
|
|
"correct"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 57,
|
|
"id": "3f6ad5d8",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor(120, device='cuda:0')"
|
|
]
|
|
},
|
|
"execution_count": 57,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"inputs[\"attention_mask\"].view(-1).sum()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 58,
|
|
"id": "986fd52b",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor(1., device='cuda:0')"
|
|
]
|
|
},
|
|
"execution_count": 58,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"accuracy = correct / inputs[\"attention_mask\"].view(-1).sum()\n",
|
|
"accuracy"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "8cc47437",
|
|
"metadata": {},
|
|
"source": [
|
|
"accuracy는 다음과 같이 구해져요."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 59,
|
|
"id": "1f3f8666",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"100%|█████████████████████████████████████████████████████████████████████████████| 125/125 [00:01<00:00, 74.40batch/s]\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"model.eval()\n",
|
|
"collect_list = []\n",
|
|
"with torch.no_grad():\n",
|
|
" with tqdm(test_loader, unit=\"batch\") as tepoch:\n",
|
|
" for batch_i,batch_l in tepoch:\n",
|
|
" batch_inputs = {k: v.cuda(device) for k, v in list(batch_i.items())}\n",
|
|
" batch_labels = batch_l.cuda(device)\n",
|
|
" output = model(**batch_inputs)\n",
|
|
" loss = CELoss(output.view(-1, output.size(-1)), batch_labels.view(-1))\n",
|
|
" \n",
|
|
" prediction = output.view(-1, output.size(-1)).argmax(dim=-1)\n",
|
|
" correct = (prediction == batch_labels.view(-1)).sum().item()\n",
|
|
" accuracy = correct / batch_inputs[\"attention_mask\"].view(-1).sum()\n",
|
|
" \n",
|
|
" collect_list.append({\"loss\":loss.item(),\"accuracy\":accuracy, \"batch_size\":batch_labels.size(0),\n",
|
|
" \"predict\":output.argmax(dim=-1).cpu(),\n",
|
|
" \"actual\":batch_labels.cpu(),\n",
|
|
" \"attention_mask\":batch_inputs[\"attention_mask\"].cpu()})"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 60,
|
|
"id": "b7567f48",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def getConfusionMatrix(predict,actual,attention_mask):\n",
|
|
" ret = torch.zeros((22,22),dtype=torch.long)\n",
|
|
" for i,(p_s,a_s) in enumerate(zip(predict,actual)):\n",
|
|
" for j,(p,a) in enumerate(zip(p_s,a_s)):\n",
|
|
" ret[p,a] += attention_mask[i,j]\n",
|
|
" return ret"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "a898cd34",
|
|
"metadata": {},
|
|
"source": [
|
|
"단순하게 confusion matrix를 계산하는 함수. 클래스 22개 정해져 있음."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 61,
|
|
"id": "15cd73a5",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor([[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])"
|
|
]
|
|
},
|
|
"execution_count": 61,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"getConfusionMatrix(torch.tensor([[0,1]]),torch.tensor([[1,1]]),torch.tensor([[1,1]]))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 62,
|
|
"id": "de9c7932",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"average_loss : 0.16166621172241866, average_accuracy : 0.9605389833450317, size :500\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"total_loss = 0\n",
|
|
"total_accuracy = 0\n",
|
|
"total_size = 0\n",
|
|
"confusion = torch.zeros((22,22),dtype=torch.long)\n",
|
|
"\n",
|
|
"for item in collect_list:\n",
|
|
" batch_size = item[\"batch_size\"]\n",
|
|
" total_loss += batch_size * item[\"loss\"]\n",
|
|
" total_accuracy += batch_size * item[\"accuracy\"]\n",
|
|
" total_size += batch_size\n",
|
|
" confusion += getConfusionMatrix(item[\"predict\"],item[\"actual\"],item[\"attention_mask\"])\n",
|
|
"print(f\"\"\"average_loss : {total_loss/total_size}, average_accuracy : {total_accuracy/total_size}, size :{total_size}\"\"\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "24539b98",
|
|
"metadata": {},
|
|
"source": [
|
|
"test로 보면 결과가 나왔어요. 96% 나와요. F1 스코어는 아직입니다."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 73,
|
|
"id": "f6047991",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"confusion = confusion[0:21][0:21]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "4830938c",
|
|
"metadata": {},
|
|
"source": [
|
|
"Outside 토큰에 해당하는 곳을 짜르겠습니다."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 74,
|
|
"id": "000d1e68",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAUsAAAEICAYAAADWe9ZcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAABNf0lEQVR4nO2dd3wVVfrGv28SivQOSSBACqRXerOggKhrRUAFV7Htqj91XV1su9ZVV9e1YFksawexIwLSBEGUKiC9Sa8JCRASUt/fH3cSLyFlktybO3eYh8/53CnvnOe8c29eZs57nnNEVXHgwIEDB5UjwNcNcODAgQN/gBMsHThw4MAEnGDpwIEDBybgBEsHDhw4MAEnWDpw4MCBCTjB0oEDBw5MwAmWZwjEhf+JSKaILK1FPQNFZJMn2+YriEiYiGSLSKCv2+LA+hBnnOWZAREZCEwCuqvqCV+3x9sQkR3ATao6x9dtcWAPOE+WZw46AzvOhEBpBiIS5Os2OPAvOMHSghCRTiLyhYgcFpEMEZlgHA8QkYdFZKeIHBKR90WkuXGui4ioiFwvIrtEJF1EHjLOjQPeAvoar52PicgfRWRRGV4VkUhje7iIrBeR4yKyV0T+ahw/R0T2uF0TIyLzRSRLRNaJyB/czr0rIq+KyLdGPUtEJKICn0vaf4OI7Da6C24TkZ4issaof4KbfYSIzDPuT7qIfCQiLYxzHwBhwDeGv/e71T9ORHYB89yOBYlIKxHZIyKXGHU0EZGtIjK2tt+nA5tAVZ1ioQIEAquB/wCNgYbAAOPcjcBWIBxoAnwBfGCc6wIo8CZwFpAE5AExxvk/AovceE7ZN44pEGls7wcGGtstgVRj+xxgj7Fdz2jPg0B94DzgOK5XfYB3gSNALyAI+AiYXIHfJe1/w/B5CHAS+ApoB4QCh4CzDftI4AKgAdAW+AF40a2+HcD55dT/vnFfz3I7FmTYDAEOGHxvAp/5+vfgFOsU58nSeugFhAD3qeoJVT2pqiVPgNcCL6jqdlXNBh4ARpV5pXxMVXNVdTWuoJtUw3YUALEi0kxVM1V1ZTk2fXAF7WdUNV9V5wHTgNFuNl+o6lJVLcQVLJOr4H3C8HkWcAKYpKqHVHUvsBBIAVDVrao6W1XzVPUw8AJwtgm/HjXua27ZEwbnp8Bc4CLgVhP1OThD4ARL66ETsNMILmURAux029+J64mtvduxA27bObiCWU1wJTAc2CkiC0SkbwXt2a2qxWXaFFqL9hx0284tZ78JgIi0E5HJRhfBMeBDoE0VdQPsruL8RCAe+J+qZpioz8EZAidYWg+7gbAKEhD7cCVqShAGFHJqQDGLE0Cjkh0R6eB+UlWXqeqluF5JvwKmVNCeTiLi/jsKA/bWoD3VxdO4XqETVbUZcB0gbucrGuZR4fAPYwjRf3G9qv+ppP/WgQNwgqUVsRRXf+EzItJYRBqKSH/j3CTgHhHpKiJNgH8Cn1TwFFoVVgNxIpIsIg2BR0tOiEh9EblWRJqragFwDCgqp44luILu/SJST0TOAS4BJtegPdVFUyAbyBKRUOC+MucP4urbrQ4eND5vBJ4H3nfGYDoogRMsLQZVLcIVcCKBXcAeYKRx+h3gA1zJjN9wJUDurCHPZuBxYA6wBVhUxmQMsMN4xb0N15Nb2TrygT8AFwLpwGvAWFXdWJM2VROPAanAUeBbXMkudzwNPGxk0f9aVWUikgb8BVf7i4BncT2Fjvdoqx34LZxB6Q4cOHBgAs6TpQMHDhyYgBMsHThw4MAELB8sRWSYiGwy1BSV9h+ZtfW0ncNtD267+eMN7jMavh4VX1nBpWbZhiurWR9XBje2NraetnO47cFtN3+8wX2mF0smeNq0aaOdO3chOzub/fv3ExUVBcCBA/sB6NAh+LRrzNp62s7htge33fzxFPfOnTtIT0+X0y6qBgKbdVYtPE0wVS409/B3qjqsNnxeg6+jdXklNTVNcwtUP5r8qf7xhnGaW6CaW6D69v/e11v/dHvpvnsxa+tpO4fbHtx288dT3KmpaVrrJ7Kz2mnDlDtNFWC5r+NPRaVWfZZV9XOICy8b59eISGp1A3k5ddbK1tN2Drc9uL1Rp924awwBRMwVC6PGwdJQNryKa0ByLDBaRGLLmF0IRBnlFuD16nCEhnZkz57fpbx79+4hJCSkVraetnO47cFtN3+8wV0rSIC5YmXU9JEU6At857b/APBAGZv/AqPd9jcBwWZfw4/nFmiXrl11w+btevREniYkJOqKVWvLfZUwa+tpO4fbHtx288dT3B55DW/UThv2uMdUwcKv4bWZLTqUU2dw2QP0NmETikv7XCWCgoL4z0sTuOSioRQVFXH9H28kNi6uVraetnO47cFtN3+8wV1zCAT4v8S+xtlwERkBDFXVm4z9MUAvVb3TzeZb4Gk15mMUkbnA/aq6opz6bsH1qk6nsLC0zdt2ljVx4MBBHaN/7x6sWLG8Vp2JAU06aIP4603ZnlzyrxWq2qM2fN5CbToJ9uCae7EEHXFN2VVdGwBUdaKq9lDVHm3btK1Fsxw4cGAtmEzu2DXBAywDosQ1XVh9YBQwtYzNVGCskRXvAxxVVVOv4A4cOLARbJDgqXHr1DWH4h3Ad8AGYIqqrhPXIlO3GWbTge241ml5E/hzdXlmfTeTxLjuxEVH8ty/nvGIraftHG57cNvNH29w1xg2eLL0eYapsmx49slC7Roerus3bSvN0q1cva7cjJ5ZW0/bOdz24LabP57i9kg2vHGwNuz/sKmChbPhln7uXbZ0KRERkXQND6d+/fqMGDmKad98XStbT9s53Pbgtps/3uCuMQRXNtxMsTAsHSz37dtLx46/54dCQzuyd2/5y7uYtfW0ncNtD267+eMN7ppDzuw+y7pAecOa/Fku5nBbl9sbddqNu1YIEHOlCojIOyJySETWuh1rJSKzRWSL8dnS7dwDhtx6k4gMdTueJiK/GudeFhMOWzpY2k0u5nBbl9tu/lhK7ih48snyXaDsrETjgbmqGoVrzffxAIb8ehQQZ1zzmvy+AN3ruMZ1l8ixq57pyNedppUleOwgF3O4/YPbbv5YSu7YNEQbnveUqYKJBA/QBVjrtl8qowaCgU3G9ikSbFwjd/oaNhvdjo8G/lsVb23kjl6H3eRiDrd1ue3mjx/LHduIyHK3/YmqOrGKa9qrMX5bVfeLSDvjeCjws5tdidy6wNgue7xSWHLy37S0HvrjkuVVGzpw4MCr8IjcsVlHbdDnLlO2J2ffX6XcUUS6ANNUNd7Yz1LVFm7nM1W1pYi8Cvykqh8ax9/GNfZ7Fy4Z9vnG8YG4ZNiXVOqHKQ8cOHDgoKYwOyC95kmlgyIS7KKSYOCQcbwiufUeY7vs8Uph+WBpNwWEw21dbrv5Yy0Fj1eHDk0FSmbquB742u34KBFpICJdcSVylhqv7MdFpI+RBR/rdk3F8HUyp7IEjx0UEA63f3DbzR9LKXiaddSGw14wVagiwQNMwjXFY0m/4zigNa4s+Bbjs5Wb/UO4FmPbBFzodrwHsNY4NwGjS7KyYuknS7spIBxu63LbzR9LKXg8OChdVUerarCq1lPVjqr6tqpmqOpgVY0yPo+42T+lqhGq2l1VZ7gdX66q8ca5O9RE8sbSwdJuCgiH27rcdvPHUgoem8gdLT10qLxg788KCIfbutzeqNNu3DWHWF7KaAaWDpZ2U0A43Nbltps/llLwgPWnXzMDXydzKkvw2EEB4XD7B7fd/LGUgqd5mDb8wxumChaeos3ST5Z2U0A43Nbltps/1lLwYIsnS0fB48CBgwrhEQVPyy7a4JyHTdme/Opm+y1YJiKdROR7EdkgIutE5DQ9k4icIyJHRWSVUf5eu+Y6cODAHyEBAaaKlVGb1hUC96pqDNAHuN2YEqksFqpqslEery6J3RQQDrd1ue3mj1UUPIIru26mWBqe6vzEJRe6oMyxc3AJ3muU4LGDAsLh9g9uu/ljJQVPQMvO2uiqd0wVLJzg8chzrzELSAqwpJzTfUVktYjMEJEKe41F5BYRWS4iyw+nHwbsp4BwuK3LbTd/rKbgscOTZa2DpYg0AT4H7lbVY2VOrwQ6q2oS8ArwVUX1qOpEVe2hqj3atmkL2E8B4XBbl9tu/lhKwYM9XsNrFSxFpB6uQPmRqn5R9ryqHlPVbGN7OlBPRNqYrb+8TL0/KyAcbutye6NOu3HXBgEBAaaKlVGbbLgAbwMbVPWFCmw6lCwEJCK9DL4Msxx2U0A43Nbltps/llLwSDWKlVGLhM4AQIE1wCqjDAduA24zbO4A1gGrcU3v3q86CR47KCAcbv/gtps/VlLwBLbqqs2v+cBUwcIJnhoreFR1EVX8X6CqE3DNFVcj2E0B4XBbl9tu/lhNwWP1/kgzcBQ8Dhw4qBCeUPAEtQ7XZsOfNGWb+eG1llXwWFob7sCBA3vADk+WTrB04MCBdyEgAf4fLK2dq8dacrFbb7qRsJB2pCXHn2L32oRXSIzrTmpSHA+Ovx+AjIwMhp5/Lm1aNOHu/7vDa22sjm1F7a8Lbn/w21/88YRdeffk888+JTUpjkb1A1ix3HPdYGKTQek+zzBVlg23mlzszXfe08VLVmhsXFyp3czZ8/Tc8wZrVvZJzS1Q3bn3oOYWqKZnZeuc7xfqyxNe11v/dLtPpGply+x5C05rf11w+4Pf/uKPp+zKuye/rFmvq9du1IGDztZFPy3zWDY8qHW4tr3hE1MFC2fDLf1kaTW52P59e2nVqhUAu9Jz2JWew4svT2DsrXdz8HgRu9JzOBnQhF3pOWTkCp2iUzlRKB7h9oRUbcDAQaXtrwhWu+d15be/+OMpuwEDB9GiRUtUobhYKS5WunWPJjKqG+B6iCouVjyW/rXBOEtLB0t/kIvt2L6F5UsWM2L42Vx3+VDWrFpRa398KVXzh3teXVuz8Ad/vMHtdYg95I6WTvCUN6zJanKxosJCjh3NYsq38/l11QruvmUMc5es81upmj/c8+ramoU/+OMN7rqA1QOhGVj6ydIf5GLtg0O5YPgfEBESU3oQEBBAZkZ6nbSxurZm4A/3vLq2ZuEP/niD29sQ5MzWhtcFevTsydatW9jx22/k5+fz6SeTuejiP9TK1tN25w+7hJ8XLQDgt21bKCjIp2Xr8ucK8TR3dW3NwB/uuTf89hd/vMFdJ7BBn6XPM0yVZcNzC1S/nPqtRkZFadfwcH308ScrzWaata2p3YiRo7RDhw4aFBSk7YND9Ml/v6a/7szUS64YqVHdYzQ2Pknf/fRb3bT/hG7af0JDO4Zp8xYttXHjxhoSGqorV6/zeBurY+ve/pDQUH39v2/VGbc/+O0v/njCbsTIUdre7Z689sabOmnK5xoSGqr169fXtu3a6eDzh2iKB7Lh9dpGaMhtX5gqWDgb7sgda4hd6TmmbcPaNPJiSxw4qBmKi6v+2x/Qtycrayl3rN8uUttd9bwp272vX+7IHR04cHDmwknw1AGsqoAoix/mzWLogGQu6JvAFUMH0De+Mxef8/t/kEeOHOGiYRcQHxPFRcMuIDMzs/Tcc88+TVx0JIlx3Zk967vS40VFRfTpkcIVl15cIa8v1SneqNNZsKzuuffs3s2FQ84jNTGWHsnxvPrKS5W2syaQADFVLA1f9wNU1mdpZQVESb/kpv0ndP2eY9qpc1ed8/Na/XVnpnbq3FVfmvihRnWP0U37T2hugeo9996njz/1tOYWqD7+1NP6l7/er7kFqitXr9OEhETNyj6pGzZv167h4Zp9slBzC1Sf+de/9epRo/XC4RdV2C/lK3WKL+65v9Vpde4TecV6Iq9Yt+7Yq4t+Xq4n8or1QPpRjYyM0uWr1uqJvGKP9FnWbxepYXdONVWwcJ+lpZ8srayACGvTqLQc3L6W7t2i6J8WR2RIC26++WZ2bFpNvaAAwto0IiM7n6+//oqLrhhNRnY+F10xmq+++pKM7HymfPYFF102guwCoUvXrkRERLJs6VL27NnDzBnfcsONN1V6j3ylTvFGnc6CZXXLHRAgBAQIoaEhpKWlERAgNG/ejOiYGA7s30dAgHgsQW2HQemWDpb+ooAoz/bggQOn2KQfOkT7DsEAtO8QTMZh1wqW+/fvJTi04ynX7tu3l/vuvZunnv6Xx8ae2U114g91+gu3O3bu2MGqVb/Qs1fvKm2rA08GSxG5R0TWichaEZkkIg1FpJWIzBaRLcZnSzf7B0Rkq4hsEpGhNfWhtguW7RCRX0VklYiclr4WF142GrpGRFKrU395mXorKiDKHVFg9n/Jcq79ZeUK2rVtR2pamrk6TNHYS3XiD3X6C3cJsrOzGX31lTz37xdp1qxZpbbVhofGWYpIKPB/QA9VjQcCgVHAeGCuqkYBc419RCTWOB8HDANeE5HAmrjgiceWc1U1uYJ0/4VAlFFuAV6vTsX+ooAoz7Z9+/an2LRp146DB/YDcPDAflq3dS33GxzSkf1795xy7b69e5k2bSrdI7sw9tpRzP9+HjeMva5cbrOwm+rEH+r0F26AgoICRl99JSNHX8tll19RoV1N4eHX8CDgLBEJAhoB+4BLgfeM8+8BlxnblwKTVTVPVX8DtgK9auREbTo8gR1Am0rO/xcY7ba/CQg2m+Dx5wWfvvn2u9Kky57MPL31jnt0/N+f1D2ZeTr+70/qbXf+Rfdk5uncxb9oTFyCbjtwTDds3q5dunYtTfDkFqh+N+f7ShM8uQWqG7f8VmWC50xaZMsqdfoLd05+sV5z7Ri9/c67TjvniSnaGrSP1Ih7p5sqmEjwAHcB2cBhXMtwA2SVsck0PicA17kdfxu4qiZ+1HacpQKzRESB/6rqxDLnQ4Hdbvt7jGP7y1YkIrfgevqkU1gY4L8LPjVu3Jibx11Peno6EV06cs/fHuGOe+7jthuuYfKH/yO0YyfeeHcSAN1jYrnksqs4r08S9evX48WXXyUw0PxbwtjrRrNwwfxSrkf+/hh/vHGcT/y24iJbZ6o/1eFe/OOPfPzRB8THJ9A7LRmAx578J8MuHF6uffVRrafGNmW69Ca6xxWjL/JSoCuQBXwqIpW9dpVHXCMlTq0UPCISoqr7RKQdMBu4U1V/cDv/LfC0ulaCRETmAverasXzmOEfCp7qICM735Rd6yb1vdwSBw6qB08sWNawQzcNG/uyKdstz11YqYJHREYAw1R1nLE/FugDDAbOUdX9IhIMzFfV7iLyAICqPm3Yfwc8qqo/VdePWvVZquo+4/MQ8CWn9wXsATq57XfE1b/gwIGDMwge7LPcBfQRkUbiumAwsAGYClxv2FwPlIyTmgqMEpEGItIVV/5kaU18qHGwFJHGItK0ZBsYAqwtYzYVGGtkxfsAR1X1tFdwBw4c2BjiGhxiplQFVV0CfAasBH7FFcMmAs8AF4jIFuACYx9VXQdMAdYDM4HbVbWoJm7U5smyPbBIRFbjitTfqupMEblNRG4zbKYD23FloN4E/lxdEk/LxbwhD6yO7VtvvMLgvimc1zeZt153vZoUFRUxdFCvUlljRQtH7dyxg5ZNz6J3WjK905K588+3OXLHKmx3797N0PPPJTkhhtSkOCa8XLGUz+y9rM49t+piaSdPnmRA3170Sk0iNSmOJx77B+CdRcuE3wfAV1XMQFX/oarRqhqvqmOMTHeGqg5W1Sjj84ib/VOqGqGq3VV1Ro0dqW2myxvFm3JHT8sDzdiWyMqWrlyjMbFxejgzW4+eyNdzzh2sq9du0qeffV5HjBytCf3O04k/7dDHJs3RxyfP1W4pvfXBd6bqxJ926MSfdug/v1ioIeHddOJPO7zmj9UletW13b5rny5eskJzC1QPHTmmkVFRtV7YzKydL7+fquxy8ot17+GjejS3SNOPndS0Hr10zvwfdekva3X56vU6YODZ+v2iJZrsgWx4ww5RGvfQLFMFR+5YM3hDLuZpeWB1bDdt3ECv3r1p1KgRQUFBDBw0iA/ef5eZM6bzxxt+z2AHd4mkQ+eIKtvoDX+sLtGrrm1wcDApqS4tRNOmTYmOjmHfvvKVLGbvpVm76tjW9b0UEZo0aQK4xlgWFBYgInSPjiGqW3dTvpmGB1/DfQlLB0u7LdwVGxvPjwsXkpGRQU5ODt/NnMEXn03hqaefNS1rTN+3myfGDueC885m0aKFHvfHXyR6Nfm+vSXl8wR8cS+LiooY0DuVyLAOnHve+fTw0n0R7KENt/R8luUNa6qtXMyX3NExMfzlr/dzyfAhNGnShObNm5OTc4KU1DR+WDC/yjY1b92OZ75aTJPmLYmTdK6+6jJWrl5XtTPVaKM3/PYldwm8KuXzAHxxLwMDA1m0ZCVZWVlcN/JK1q9bS2xc1X2r1Yf1A6EZWPrJ0m4LdwFcf8M4Fi9Zway5Czh69Ci7d+0mpltXrh8zmo0rFvP2o3dX2KZ69RvQpLlrfoDUtDTCwyPYsnmzR/3xF4ledWy9LeXzBHx5L1u0aMGAQWczx20uVU/DeQ33Muy2cBfAoUOHANi9axdZmZms27SNDZt/470PJhGd1o9xj75YYZuOZ2ZQXOQa9fDb9u1s3bqFruHhHvXHXxbZMmurqtx28zi6R8dw1z1/qeo2+Qx1fS8PHz5MVlYWALm5ucyfN5du3T3cV1kC8Ww23GfwdYapsmx4boHnF3zyxuJVVdmWZMNP5BVrv/4DNDo6RuMTEnXajNmlx2fMmleaDf/T029oi7YdNKhefW3aso3G9h6oE3/aobf+83UN7hqlHSOjNTk5RT/7cqpPF+PyRp2e5p7z/UIFND4+QRMTkzQxMUm/nPptrX4b1bnnVl0sbemK1ZqQlKxx8QkaExunDz7yqB7NLdIPJ3+mISG/L1rWtGmzWmfDG4V009TH55kqWDgb7ixYVgcwszAUwEcrd5muc0yPzjVtjgMHAOQXFldpc3b/XvxSS7lj49DuGvOnN0zZrnjkPGfBMgcOHJy5cBI8dQA7qUlK7JLjo0mIieL55061e/fJ+7h3eBqPXjuk9NjEh2/nseuGcvs50fxpYCR/PrsbU998AYA1q1dz9oC+9EhO4MrLLuHYsWOAa4xdicqnV2oSX3/1pSX89hW3r1Q0mzdtKv0eeqcl065VM1556cU64a7Kbvfu3Vw8dDA9k+PonZrA6xNcarI1q1cxeFA/BvRO5ez+vThx4kSF9VcHdkjw+LwfoLI+S7uoSUr6JY/lFGjXruG6dsNWzTx+UuMTEksXhjqRV6x/fe0TfejdaaUqnZLy38W/6ctz1+kFo2/Si2/8P+0Sm6TzF/6kqWk9dNbc+ZpboPrGxLd1/IMPa26BasbRE3o8t0BzC1wKlrZt22pW9skzUsGTW+BblZP7Ne3bt9eNW3fUCXdVdtt37dMFi5fp0dwi3XMoSyMio3TJyl/13MHn62dfTdOjuUX66ZffaOMmTWrfZxnaTXv9c76pgoX7LC39ZGkXNUlJpm/F8mVEREYSERlBw4YNuHrkKKZPm1p6/h83XsXoPjE0b1iP0SmdSss1qWH8sV931v8wkwf/dAPN6wcwb+sh1m/cyL6mXZmyajc5wXG8P2kyU1btZtrmDL5Yu58pq3aTd/IkIsLyZWemggfqXkVT3h/avLlz6BoeQVhY2CnHfXUvg4ODSUlJRYBmTZvSPTqa/fv2IiIcP3YMAY4dPUr9evWqvG9VQTCXCbd6NtzSwdJuapLaqIx+WDCfI0cyuOCcAZw7+Hwi4lPoGN6dX36YDcCyud9y5ODvEzptW/sLD1w9mB4pCbz86hscPHjQ8n77UrHlrXaW4NMpkxkxclSdcVenjTt37mDNqlX06NmbZ5/7D488+DdiIjvz8AP3ExISWqlfZmGH13BLB8vyMvX+rCapTp1l8eXnn/LwPx5n47ZdrFi2jD1bNzHu788x59P3+PuY4eTmZBPo9hQQEZ/C01PmsuinZTz37NPk5+d5vI3+cM+rA299j/n5+Uyf9g1XXDmizrjN2mVnZzNm9Aieee4FmjVrxlsT3+Dpf/2bDVt38vS//s3OXTsrbHN1YAe5o6WDpd3UJDVVGRUWFjL16y+58qqradGiBQMHnc2an+YT0iWS+yd8xOMfTKfvkEtpF3r6cKLomBgaN25M3sk8y/vtS8WWt9oJ8N3MGSSnpJ62iJ03uc3YFRQUcN3oq7h65DX84TKXsmnSR++Xbl9+5QhyPJHgcSbS8D7spiapqcroqy8/Jzw8ktCOHcnNzeX7eXMJ6RLBsSPpABQXF/P1Oy9z3pWupUgO791FUWEhADt37mTz5k0MvXC45f32pWLLW+0E+PSTyl/BvcFdlZ2qcvttN9G9ewx33HVP6fEOwSEsWrgAgAXz59GgQcNK220GzkQadQB/WfDJU3XeMOYaFi5cQEZ6Ot0jwnjw4X9w/Q3jmDJ5Env27KJPj2SKi4u54soRdBt4PrMmvc2cz94HoMc5wxh4ydUAbF69jGnvvkZQUD1aNW7AS6+8RocOHSzrtze5wbeLuuXk5DBv7mxeea3yQdl1fS8X//gjkz/+kLj4BPr3dk1h9/fHnuSVV//L3+67h8LCQho0aEhY57BK220WVg+EZuAoeCyEwqKqFRUl+OJXc8mMq5M7VW3kwKOozt+UL4NIgUkFz8paKniadorWtHvfMWW74J7+llXw1GYNnu4issqtHBORu8vYnCMiR91s/l7rFjtw4MC/cKb3WarqJlVNVtVkIA3IwbXCY1ksLLFT1cery2M3NYlZu9mzZpKSEENSbDf+/dyzlXKvWTyfv115DvddPpBp775aejw/r/x1VgBem/AKiXHdSU2K48Hx91vGb19ye8ufpLho4mOieN6i93LL5k30751aWkLbteDVV16qtK3VgWCuv9Lyr+qeGNmOa2XHH8s5fg4w7UxX8Ji1O36ySI+fLNKsE/natWu4rlm/RTOO5Wp8QqIu++XX0vPHTxbpe8t26XvLdun/fv5N24aG6XNfLtS3F2/VTlEx+s9P5uh7y3bpu0t36uHM45pboHosJ1979Oyl8xf+pDNnz9NzzxusWdknNbdAdefeg7ZV8Piizpz8Ys3JL9bjuQXaNTxc123cqlnZJzUhIVFXrFpbej4nv9in9/JYbtFpJTM7X9u1b69rN27XY7lFmuKBNXiadorW815ebKpwBih4RgGTKjjXV0RWi8gMESm/FxwQkVtEZLmILD+cfhiwn5qkKrugwACCAgP4ZcVyIiIjiYqKpNFZDbl65ChmfPtN6fmgwACuTu7E1cmd6JK/n6TYaO64eADX9Izg5uvHULhlKVcnd2JkShj1GpxFXkER2Tknyc8voLComDdef427/nIfBASRV1BE85atfeq3Fbg9WWfJU9LyZcuIiIgkPCKCBg0aMGLkKL6dNrXcJylf3MugQDmtLFwwj/DwCMLDuxAU6LknvQARU8XKqHWwFJH6wB+AT8s5vRLorKpJwCvAVxXVo6oTVbWHqvZo26YtYD81iS+4XeuspBHVOZhzBw+mR6/ebN2yhcU/LmLwoL4MH3IuK5cvs53fVqjTX7jdUZXSqCYQm0z+64knywuBlap6sOwJVT2mqtnG9nSgnoi0MVux3dQkvuB2rbOygnVbdrJi+TLWr1tLUVEhWVlZzFmwmCeeepY/jhldWo9d/LZCnf7CXQIzSqOaIkDMFSvDE8FyNBW8gotIBzG+IRHpZfBlmK3YbmoSX3K3aNGCAQPPZu7s7wgJCeWSSy9DREjr2YuAgADS09Nt6feZ6k9NFE5mlEY1xRmf4AEa4Qp+zd2O3QbcZmzfAawDVgM/A/2qk+A5nlugXbp21Q2bt5d2Uq9Ytbbcznmztp62szL3rn2HdMe+dM3KKdT9Gce1b7/+Ovmzr/SFl17V+8Y/pFk5hbp89XoNDe1Ymmywg99WqdPq3O6Jppz8Yr1qxEh94823TznmiQRPs7BoHf7GElMFCyd4aqXgUdUcoHWZY2+4bU8AJtS0frupSeqa+8D+/Yy74XqKiovQ4mIuu+Iqhg2/mPz8fO647Sb69kiiXr36vPbmO6X/q9vBb6vU6S/cYF5pVBMIruFD/g5HwWNz5BUUmbJrUC/Qyy1xYDWY+dvv36dnrRU8LTrH6KCH3jdl+82tvSyr4LG0NtyBAwc2gFg/020GTrB04MCBVyFg+TGUZmDpKdrA91I1O3HPmTWTHkmxpMR35z/PVyyhtJvfZ6o/1eU2K8usCeygDfd5hqmybPiZKr1zuP1b7mgn7lQPZMNbdI7RK99ZYapg4Wy4pZ8sz1TpncPt33JHu3LXFGafKs0+WYpICxH5TEQ2isgGEekrIq1EZLaIbDE+W7rZPyAiW0Vkk4gMrakflg6W/iIXc7j9n9tu/vh68beyCBQxVUziJWCmqkYDScAGYDwwV1WjgLnGPiISi2vuijhgGPCaiNRo6Ielg2V5QxusKBdzuP2f2xt12o27NvCUgkdEmgGDgLeNtuerahZwKfCeYfYecJmxfSkwWVXzVPU3YCvQqyY+WDpY+otczOH2f267+ePrxd/c4cqGm9aGtymZfcwot5SpLhw4DPxPRH4RkbdEpDHQXlX3Axif7UrcA3a7Xb/HOFZ9+LrTtLIEj9XlYg63fbjt5o+nuD2R4GnVNVav/WCVqUIVCR6gB1AI9Db2XwKeALLK2GUan68C17kdfxu4siZ+WHqcpb/IxRxu/+e2mz/eWvytpvDgW/0eYI+qLjH2P8PVP3lQRIJVdb+IBAOH3OzdF6LqCOyrCbEjd3TgwEGF6N+7BytqKXdsHR6nFz1Z0dzgp+KDa5OqlDuKyELgJlXdJCKPAo2NUxmq+oyIjAdaqer9xoTjH+PqpwzBlfyJUlVzOmA3WPrJ0oEDB/4PAQI9K3e8E/jImHh8O3ADrvzLFBEZB+wCRgCo6joRmQKsx/X6fntNAiVYPMED/qWAcLj9m9tu/niDu6YQk8UMVHWVulZVSFTVy1Q1U1UzVHWwqkYZn0fc7J9S1QhV7a6qM2rshK+TOZUleOyggHC4/YPbbv5YScHTJjxWx03+1VTBUfDUDHZTQDjc1uW2mz9WUvCAPbThlg6WdlNAONzW5babP1ZT8NhhWYkqg6WIvCMih0RkrduxCnWYZa4dZugxtxoZqmqhvEy9PysgHG7rcnujTrtx1wZnypPlu7g0le4oV4fpDkN/+Squ1R9jgdGGTtM07KaAcLity203fyyl4BEhMMBcsTTMdGwCXYC1bvubgGBjOxjYVM41fYHv3PYfAB6oToLHDgoIh9s/uO3mj5UUPG0j4vTPX6w3VbBwgqem4yxP0WGKSLtybMrTZPauqEJDA3oLQKewMMB+CgiH27rcdvPHagoeSydHTMKUgkdEugDTVDXe2M9S1RZu5zNVtWWZa0YAQ1X1JmN/DNBLVe+sis9R8DhwYA14QsHTPjJeRz7/mSnbVy6PseyCZTUN+AcN/SVldJju8Jgm04EDB/6Nasw6ZFnUNFhOBa43tq8HyhuUtQyIEpGuhixplHFdtWA3BYTDbV1uu/ljFQWPCGdGggeYBOwHCnA9LY4DWuPKgm8xPlsZtiHAdLdrhwObgW3AQ2Y7Uh0Fj8PtKHiswe2JBE/7yDj96zcbTRUsnOCp8slSVUerarCq1lPVjqr6tlagw1TVfao63O3a6araTV26zKeqG8jtpoBwuK3LbTd/HAWP52HpJJXdFBAOt3W57eaPlRQ8gmvdcDPFyrD0FG3lZer9WQHhcFuX2xt12o27NrD0U5lJWDpY2k0B4XBbl9tu/lhJwQPWf8U2BV93mpZXHAWPw+0oeKzB7YkET3BUvD4yc7OpgoUTPJZ+srSbAsLhti633fyxnILHBk+Wzho8Dhw4qBCeUPCEdkvQW1/90pTtP4ZEWVbBY+knSwcOHNgDduizdIKlAwcOvAs/kDKageUz+naTiznc1uW2mz9WkTsCiMl/loavM0yVZcPtIBdzuP2D227+WEnuGNotXp+dt9VUwcLZcEs/WdpNLuZwW5fbbv5YT+54BqzB40vYTS7mcFuX227+WE/u6P9TtFk6wVPesCZ/los53Nbl9kadduOuMfxgkgwzsHSwtJtczOG2Lrfd/LGa3NHqk2SYgq87TStL8NhBLuZw+we33fyxktyxU/d4fXnRdlMFCyd4LP1kaTe5mMNtXW67+WMtuaMQYPVhQSbgyB0dOHBQITwhd+wcnah/e8fcijK39+9qWbljldlwEXlHRA6JyFq3Y8+JyEYRWSMiX4pIiwqu3SEiv4rIKhFxop8DB2ciTGbCrZ4NNzN06F1gWJljs4F4VU3EtcbOA5Vcf66qJtf0fwu7KSAcbuty280fKyl47DBTuqmOTaALsLaCc5cDH1VwbgfQpqYJHjsoIBxu/+C2mz9WUvB0jk7QiT/vMFUwkeABAoFfgGnGfitcD3BbjM+WbrYPAFuBTcDQ2vjhiUHpNwIzKorFwCwRWSEit1RWiYjcIiLLRWT54fTDgP0UEA63dbnt5o/VFDweXgr3LmCD2/54YK6qRuFabXY8gIjE4lqCOw7X2/FrIhJYUx9qFSxF5CGgEPioApP+qpoKXAjcLiKDKqpLVSeqag9V7dG2TVvAfgoIh9u63Hbzx3IKHpOlyrpEOgIXAW+5Hb4UeM/Yfg+4zO34ZFXNU9XfcD1h9qqpHzUOliJyPXAxcK1WkFJX1X3G5yHgS6rZ0PKq9WcFhMNtXW5v1Gk37hpDqqUNb1PyhmmUsm+kLwL3A8Vux9qr6n7Dl/1AO+N4KLDbzW6PcaxGqNE4SxEZBvwNOFtVcyqwaQwEqOpxY3sI8Hh1eOymgHC4rcttN3+spuCpRuhNrygZLCIXA4dUdYWInFND2pqPlTTRmToJ2A8U4IrM43A9zu4GVhnlDcM2BJhubIcDq42yDniougkeOyggHG7/4LabP1ZS8HSNSdAPlu82VagkwQM8bcSgHcABIAf4EFfyJtiwCQY2uSV3HnC7/jugb039qNVN8FYpCZa5BapfTv1WI6OitGt4uD76+JPlftnVtfW0ncNtD267+eMJbk8Fyw+X7zZVKguWZQLnOfyeDX8OGG9sjwf+ZWzHGQ9rDYCuwHYgsKZ+OAoeBw4cVAhPKHjCY5P0qY+mm7K9JrWjKQWP8Rr+V1W9WERaA1OAMGAXMEJVjxh2D+EasVMI3K2qFY3cqRKW1oY7cODA/1GSDfckVHU+MN/YzgAGV2D3FPCUJzgtPfkv+IcC4tabbiQspB1pyfEe88fTfu/evZuh559LckIMqUlxTHj5pTrj9oZdWduHHxxfrn9rVq/m7AF96ZGcwJWXXcKxY8cAmDtnNv16pdEjOYF+vdKY//08r7Sze2QXeiQn0Dstmf69K35gquvfUFW/h/+88Dxn1RMKCwurbI8Z2GGmdJ/3T1bWZ+kvCojZ8xbo4iUrNDYurtJ+IV8qP7bv2qeLl6zQ3ALVQ0eOaWRUlK1UJ9HRMfrR5E9P8y81rYfOmjtfcwtU35j4to5/8GHNLVD9aelK3bZzr+YWqC7/5VcNDgnxSjvDOnfW3fsPV/q78MVvqLLfw+btu/T8C4Zop7AwTUxMqnWfZXhMok75Za+pgoWnaLP0k6W/KCAGDBxEq1atPOaPN/wODg4mJTUVgKZNmxIdHcO+facPPPaXe17W9prrxrBt65ZS/9p1imDu8g1s3LiRoJA4ft6WQcuoHkz6ZAo/b8vgZLMwduU2ACA2Lo68kydZ/OMij7SzTCKiqkQFUPe/oeDgYBKSkiksKuasRo3p1j2a3bt3U1hUzH333s3jTz2DILUYZ+OG6o2ztCwsHSz9RQFhFlZRX+zcsYNVq36hZ6/edcJd137v3LGDrRt+JSYpja5RMfw4z9WnP3/m1xzaf3p9X37xOUnJKRw+fNjj7RQRLhk+lH69e/D2WxPLtakOvHEvd+7YwZpVq+jRqzffTptKSEgoCYlJtW5rCQQIFDFVrAxLJ3jKy9RbUQFhFt7grm47s7OzGX31lTz37xdp1qxZnXDXpd8l/t3+wFM0btKU+//5Mq88+QDvv/o8/c8bRr169U+5Zv26dTz84N+YNn0Wv/yy0uPtnDt/ESEhIRw6dIhLLhxC9+7RDBhYoeq3Snj6XmZnZ3Pd6BE88/wLBAUF8fyzT/PVtJk1bl9FsHYYNAdLB0t/UUCYha/VFwUFBYy++kpGjr6Wyy6/otb1WU110r59+1L/eg+5GICw8Ciee+czAHb/tpWfF8wuvebwgX3cfPOVvPXO+4RHRHDo0CGPt7PkeLt27bjk0stYvmxprYKlJ+9lQUEB1426iqtHXcOll13BurW/smPHb/TrmVJ6zf79+xCRDqp6oMaNxh4Llvm807SyBI+/KCByC1Q3bvmtys55Xyo/cvKL9Zprx+jtd97lkTb6+p6XtY2PT9CLLr6k1L/vN6br9xvT9YsfN+j3G9N17vpDesGlV+t9T72k329M12+WbtPw7nH68SefebydOfnFmpNfrIczj+vBjKOl27379NWvvpleej4nv9hnv6Gc/GIdfc11+uc7/k+Pnywqt4SFddYEDyR4ImMTdeqaA6YKFk7w+LwBlQXL3AL/UECMGDlKO3TooEFBQRoSGqqv//etOuM2azvn+4UKaHx8giYmJmliYpJ+OfXbOuGuC79vGHfTKf5FRMfr0/+dpLc/+JR27ByuHTuH6+ib/0/nbTis329M1xvvekAbntWo9F4kJibpzr0HPdLOkkC4buNWTUhI1ISERI2JidV/PPbEKYHSPVjW9W+o5PcQF5+gCYlJmpCYpJ999Y3XguU3vx4wVawcLB0FjwNb4udtGabs+kS09jh3df6mfJkBLiwqrtJmUL9erKylgicqLllf/GSWKduLE9pbdg0eS/dZOnDgwP9Rkg33dzjB0oEDB96F2CPBY+lxluCf0juH21rcSxfOZeyw3lw7pCcfTzxV1jfh5ZdIS44nNSmOV156EYDH/vEIPVMS6Z2WzMUXDmHfvn3lyhGffPxRwjuH0jstmd5pycyccepkEXt272bYBeeRkhBLWlI8r75SscS0Ov54414WFRXRv3caV11+SaV2NYWIuWJp+LrTtLIEj79K7xxu33OXZMPnrDuoIZ266Eezl+usNfs0vHuc/m/aj/r9xnR9Z+pCjY2L04yjJ/R4boGee95g/XX9Zj2YcbS0nudfeElvuvnWcuWIDz3yD/3ns8+d1raS5M22nXv1xyXLNSe/WA9mHNXIyChdsWptuQkeX9xL92TOP599XkdcPUqHXjj8lOMpHpiiLSouSWetP2yqYOEEj6WfLP1Veudw+567T0Rr+kS0JjBjG7HR3bjinDQGxgTzxzHXsnvVAvpEtKb+if0kpfbiZHEQx04Wk9ZnAJOnfEphQEOOZOdzJDuf9Mxj5BUWE5vSh+YtWqIKRcVKUbFSrIoa2+6lRLoXEhJCamoaIkKzZs2IjokpGbd4mrzPF/cyKDCAoMAADuzfx6yZ07nxppsJECk9HhQY4JHB5MKZs264z2AH6Z3DbV3uuLh4fl68kCNHMsjNyWHe7Jns27sHgGee+Ds94iL48tNJ3PfgP8ptA8B/33iV3mlJ/OmWG8nMzKzQrjKJqaf8qYkdwH333s1TT/+LgADvhQM7rBte5d0RkXdE5JCIrHU79qiI7BWRVUYZXsG1w0Rkk4hsFZHx1W1ceUMw/El653Bbmzs6Jobb7/oroy8fzrVXXUJsXAKBQa6c5/hHHmf5um1cPmI0/3vz9XLbcNMtf+LXDVv5adkvtO8QzIN/u7dcu6okpp7ypyZ207+dRru27UhNSyu3XZ6CmPxnZZj5r+RdXGvulsV/VDXZKKdNg2ysz/sqrmVwY4HRxjq+puHP0juH2z+4R4+5ge8WLOGL6XNp0bIVXcMjT7n+8qtGMn3ql+W2oX379gQGBhIQEMANN97M8mXLTrMxIzH1pD/Vtftp8Y9MmzaV7pFdGHvtKOZ/P48bxl5XYTtrAru8hpvq2AS6AGvd9h/FNaV7Zdf0Bb5z2z9l8SAzCR5/ld453P7DvXrzbt2bmadL12zRiKhuuu63A7pw+Vrdm5mnezPz9IlnX9Dhf7hc92bm6bpN2zUmNk6z84o1O69Yt+7YW7r9zL/+rVeOGKnZeb8nbcxKTH19L0vKd3O+1wuHX3TKMU+swdM9Lll/2HTEVMHCCZ7ajLO8Q0TGAsuBe1W1bIdNeWv2lt9hAxjrA98C0CksDICgoCD+89IELrloKEVFRVz/xxuJjYsr93qztp62c7j9m/vmsaPIzMwgKKgeTz33Ei1atOS+/7uNbVs2ExAQQGinMJ55YQJ/HjeGnxf/QEZ6Ot3CO/HQI4+y8IcFrFm9ChGhc+cuvPzqG6dwL/7xRz7+6APi410zpQM89uQ/GXbh6b1WvryXXoc/DAsyAVNyRxHpgmsltXhjvz2QjmsN3idwLUN5Y5lrRgBDVfUmY38M0EtV76yKz5E7OqgrHMnON23bvFE9U3aBln+fNA9PLFgWnZCib38xz5TtgG6t7CV3VNWDJdsi8iYwrRyzPUAnt/2OwL6a8Dlw4MB/YRe5Y43GCohIsNvu5cDacsyWAVEi0lVE6gOjgKnV5bKTmqQ6i1KZtfXGQmS+vOfgUpP06ZHCFZde7BHul1/8D6lJcaQlxzP2utGcPHnyNL4hg3oxduRlADzxyHgG9UogJboz3Tu14ezeifx53BhOnjzJmtWrOHdgX/r2TGFg354sX7a0tJ61v67hvEH96NC2JU0aBpGS+Ptr7+effUpqUhyN6gewYnn5b03e/v2W95uqrF3PPfs069atxRjRMrRCIjMQk8XKMJHcmQTsBwpwPS2OAz4AfgXW4AqAwYZtCDDd7drhwGZgG/CQ2Y5Uuyp4zC5KVR1bTy9E5st7XlKe+de/9epRo09LNtSkzq079mjnLl30yLEczS1QveKqETrxrf+Vnt+bmad/f/JZvezKkTp4yIW6NzNPP/58mi5Zs0U7hXXWW/78f/rn/7tXL77sSn3jzXf0vMEX6Bdff6vZecX6+VfTdMCgszU7r1izTuRrXHyC/rTsF509b4HOmDVXY2J//+5+WbNeV6/dqAMHna2Lflrmk9/vm++8d9pvqqJ2rVy9ThMSEjU5OUWBrsbfcGBNEiPR8cn689YsUwULJ3iqfLJU1dGqGqyq9VS1o6q+rapjVDVBVRNV9Q+qut+w3aeqw92una6q3VQ1Ql3r91YLdlGTlMDsolTVsfX0QmS+vOcAe/bsYeaMb7nhxpsq9bs6i4YVFhaSk5NDQUEBOTk5dAgOLj2XeXg/8+fM5IZx4wgMEBo1COTCC4fR5Kz6FBUVkZKSwoH9e8g/mct9n2zmh/UHGPP8TMJunszYf89k6a4Cwm6eTMdLHmLLiWZc9MoG+vUfSOfOXQEoLlaKi5Vu3aOJjOpW2raS4966l+XZ7d+395Tf1OFjebQODadlcBcKiorJPJHP4WN5HD6Wx+RPP+fCS6+iSAVV/Q3YCvSq9EupBHbQhjsKHh8tGuYNeGIhMl/eczCvJjFbZ2hoKHffcy/dIzoTHhZC82bNOf+CIaXnH7z/Lzz25DOn8YWEhnLn3X/hjttuYtaM6TRr3px6IYk07jmW3BUfk/XZHeQs/4izUkcCUHzsAIhwbPbT9OudxltvnpoZrwpWW3Dv4P59hIR2dD+0B9cIlxrBDm/hlg6W5WXq/VlN4k14aiEyX97z6qhJzNaZmZnJtG+msn7zdrbt3MuJEyeY9NGHpXxt2rYjOfV0vqzMTN549RXOHXwB2/YcJOfECfK2L+Lk5jk06jmGFldNoFHPMZxYPNFoTxGFhzbRZODtzPl+IXPnzOLEiRNV+lFdf7zxPZptD9RiZVwbREtLB0u7qUm8BU8uRObLe14dNYnZOr+fO4fOXbrQtm1b6tWrx6WXXc7PPy8G4OfFPzLz229IjI5g3NhrWbjge265cSwATz32d05kZ/Pex1OoX78+l1x6OYWHNpO/7QfqhfUEoH7n3hRmbAcgoFErgtrHENCwGY0aNWLQ2edwMje33LbXxp+6WnCvQ0hoqU7eQI1Hs4icIdpwX6JHz55s3bqFHb/9Rn5+Pp9+MpmLLv5DrWw9bVddW09DVbnt5nF0j47hrnv+UqGdL/02a/fEU0+zbcceNm3dwfsfTeacc8/jf+9/WCvujmFhLFuyhJycHFSV+d/PIzo6BoDHn3qadVt3smbjNt5+/yMGnn0uE995nzmzZjLruxm0aNmy9B4vmD+PwOahSKOWFB7cAEDhgXUENm0PQL2QRIoyd6GFeRQWFrJ0yVIaNGhQbtvr4l7W9jd5wYUX882Xn1JcXIyIdAWigKVVXVcRbPBgae35LHML/HPxrIpsq7MolVlbbyxE5st7XlLKk95Vp073OSMfeOgR7datu8bGxunoa67TzOO5pecycwo1M6dQv5k5R4cMG66ZOYXaNTxCQ0I7att27bV+gwbaokVLvXr0tdry2ve06dC/a2CrLhrYMkwD20Ros4ue1FZjP9ZWYz/WxgP+rIHNQ7VZs2baqHHj0u/utTfe1ElTPteQ0FCtX7++tm3XTgefP0RP5BV79V6WtSv7m/rXS6/rxPc+0Q7Brna1adtOB517vu7KOKm7Mk7qfQ8+qvXr11dgE3BhTf+eYxKSdcWOo6YKFs6GOwuWObAlzP6u8wqrXrSrBBG3TTFlt/vNUabrDPCh2ufwsbwqbS46rx9rVq2oVSNjE1P1o28WmLJN7dLMXgoeBw4cOKgOLN4daQqW7rMEeyl4PGVXXSWGt9rojTo9zV3ddXDmzJpJz6RYUuO785/nny09fvLkSY5++zBHvxnP0a/v4+g3D5A55TYyP7mNzE9v5+g3D5D19f0c/eYBEmO7cfGFQ0onA5486SP69EwpLU0aBrJ69SoAPp3yCT1TEklNiuOakSPq7F5OePklzu+fyuB+Kbz1xisAZGUe4ZorhjOoZxzXXDGcrKyKJzOuDgTPjbMUkU4i8r2IbBCRdSJyl3G8lYjMFpEtxmdLt2seMObUrZ0Sydf9AJX1WdpNweMpu/LUPXWtELH6PTe7Dk5Jn2VmTqGmH8/TLl3D9Zd1m/VgVo7GJSTqTyvWaGZOoR45UaAtR7+jrcZ+rC2ve18Dmodo4363qjRopmelXaOtxn6sDeMu1rNSRumJvGJ97Ml/6j333qcn8opPKUtWrNYuXbrqibxi3bXvsHbs1El37Tuk2ScLtUmTJvr2u+97/V4u/+VXjY2L0027j+j2g9naf9C5umDpWr31zr/o3x55QndlnNS/PfKE3nbnvZqQlKq1/XuOTUjR1buOmypU0WcJBAOpxnZTXArBWOBfwHjj+HjgWWM7FlgNNKCWSiRLP1naTcHjKbsBAwfRwlgPxlcKEV/4XR1bs+vgNKwXWFp+/WUFkZGRRHeLolnjsxg5chSzZ0yjYb1AzqofxG9vj2X7G1ez7sXLiO/Snv/eeQGtmzbgwSuT2P7G1XTM38wvnz1H69538s9Jm3lxwpu07n3nKaXfhTewO7cVrXvfSeQ5fyIqqhtt27Zl2dKldO7chSU//eT1e7lx4wZ69epD06aNadigHv0GDGLWjK+ZPeMbRl07hqBAYdS1Y5g1Y6rHXp899WSpqvtVdaWxfRzYgGuw/KXAe4bZe8BlxvalwGRVzdNaKpEsHSztpuCpK/VFXbTRG3V6WzVV23VwioqKGNA7jajOwZw7eDAJyckATHzjNfr1SmHnjt9o2LAhAFKvMVp4+jjL4qytBLaIctnUb86mTRvZuWMHe3bv4nj28dKxkd68l3Fx8Sxa9ANHjmSQU7L20J49pB86RPsOrjly2ncIJuPw4XLvU01QjaFDbURkuVu5pcI6XVNHpgBLgPb6u+x6P9CuxHVOn1e3RkokSyd4ysto+rOCxxvcZmE3v6t7jzyxDk5gYCCLlqwgKyuL60ZdyaCzz6Vlq1b8vGINIkJw66Y8NP4+oHG59RefOAABQQSc1dpVd1BDXp7wOtddM5KszEyaNGlaugZQRf544l5Gx8Rw71//xqjLhtO4cRNi4xMICvJiKKjeIMp0M9lwEWkCfA7crarHKvnuyztRoyFAln6ytJuCp67UF3XRRm/U6S3VlKfXwWnRogUDBp7Nj4t+ICgoqHQdng7BISwx1EFacAIJOuuU64qythLYMuqUYxddfAkLFy/hzXfeo7i4mMjIqEq5PXUv/3jjOGb9sIQvZxhrD0VE0qZdOw4e2A/AwQP7ad22bYX3qrrw5IJlIlIPV6D8SFW/MA4fLJk60vg8ZBz32Ly6lg6WdlPw1JX6oi7a6Gu/zdqqmlM4VVXn4cOHycrKAiA3N5cF38+lS9dwCgsKSq/vFBZWqtopOrKRgOZdT2lHUdZWAlqcGiwPHXL9TUdERrJ921YuGDK0Tu5lCe+e3buY/s1XXHbVSIZceDFTJrkUU1MmfcjQ4ZdUer/MQvDcgmXieoR8G9igqi+4nZoKXG9sXw987XZ8lIg0qLUSydeZ78qy4b5Wk1iVe8TIUdreTYnhK4WIP9zz6iicKqtz6YrVmpCYrLHxCRoTG6cxsfHavn0HFRENCgrS4JBQPe/8Idqnb3+V+s01oEmoNogfpw2Tb9eGybdrvYjLVBq1L90vKSNGjtLomBiNjonRv/z1/jq7l/36D9Co7tEaG5egn3w9Q/dl5ena7ft0wKBztGt4hA4YdI6u+22/JibXPhsel5ii6/dlmypUnQ0fgOs1eg2wyijDgdbAXGCL8dnK7ZqHcGXBa6VEchQ8fgr3THdl8KVCxG7IKygyZdeh312m68xcNqGmzak1Mk9Uvf7QsHP6svqX2il44pNS9bOZi0zZxoQ0dhQ8Dhw4OHNhBwVPlcFSRN4BLgYO6e+rO34CdDdMWgBZqppczrU7gONAEVBo1f8xHDhw4F3YIFaaSvC8CwxzP6CqI1U12QiQnwNflHNdCc41bGsUKO0kvfMmd3J8NAkxUTz/3Jnltxnb6iwUZ7bOObNm0iMplpQyssiyKDq2k7wNH5G3/gMKD6445Vzh4dWkJceTmhTHKy+9CMB114ykd1oyvdOS6R7ZpXS98Zq0sSrJbEJkR87tm1J67PFHxjOwZwKD+6Vx47UjOGoktDwCO8zRZqZjE+gCrC3nuOAa8BlVwXU7gDY1TfDYRXrnDbsS+dyxnALt2jVc127YqpnHT2p8QqIuX7W29Lzd/K6JrdnF36qqMyunULNyCjXDkEWuWrdZDxmyyJ9XrCk9X5K8aZD0J5X6zbR+zHXaIPE2lYattX70aG2YfLvW7z5KpWErzTh6Qo/nFui55w3WX9dvPqU9/3f3X/SRfzzmlXs5e94CnTn/Z+0eE6v7svJ0X1aefvzFNN2VfkL3ZeXpn++6V/98170eSfDEJ6XoloM5pgoWnqKttkOHBgIHVXVLRbEYmCUiKyobiQ8gIreUjNo/nO5SDthFeucNu4AAISBAWLF8GRGRkURERtCwYQOuHjmK6dOmlp63m981sTW7+FtVdTaoF0iDeoGsMWSR3btF0dSQRc6aMa30fOayCWQum8DMV8Yw+Ow+HF3zAVkrXuexh+/l4esSyFw2gXcevYLrR19ObnEQR08Wk9ZnAJOmfEpGdj4Z2fmkH8/j0ylTuOCSK8nIzjfdRrN2AwYOIiy4HQEiND+rHs3PqselFw2nddOzaH5WPQb270f6wf0EeihBaIcHy9oGy9G4lsqtCP1VNRW4ELhdRAZVZKiqE1W1h6r2aNvGNRjWbtI7h7vuuauDuvSnRHKYeSSD3BLJodsyDksWL6Jtu3aER0SZrrMmdhXh/Xf/x5Chw6o2NAsbRMsaB0sRCQKuAD6pyEZV9xmfh4AvqaaAvbxhTf4svXO46567OqhLf0okh6MvH851V11CbNypksOvP/+ES6+82uttLA//euYpgoKCGDX6WlP2VcOsfsfa0bI2T5bnAxtVdU95J0WksYg0LdkGhgBrq0NgN+mdw1333NVBXfvzxxvHMXPBEj6fbkgOwyMBKCwsZMa0r7nk8hFeb2NZfPjBe8yY/i3vvPehR1cotcO64WaSO5OA/UABLp3lOOP4u8BtZWxDgOnGdjiueeRWA+uAh6qb4DmeW6BdunbVDZu3l3ZSr1i1ttzOebO2nrZzuK3NXVI2bvmtygRPXfuzc+9B3ZOZp0vWbNGIqG669rcDuiczTz/4dKr27jdQ92TmlRZvtHH9pu0aExtXmgz8cup0jY6O0R17DpYeS0lNq3WCJyEpVX9LzzVVsHCCx+cNqCxY5hbYS3rncPuGuzoLxdWlPyWSw5i4BJ381YzSwDhi9Bh9+t+vlBssPcVdnmQ2PDxCQzt21ITEJE1ITNJxN93isWC5I/2kqWLlYOnIHR048CHcM92VoXWT+h7nNiOZHdC3JytXLK/VC3JicppOm7fYlG3n1g0duaMDBw7OXFi9O9IMLD1FG9hPTeJwW5fb1/58P+c7BvWMp39qDBP+81ydcptVgdUIJpM7fp/g8WWfpd3UJA63dbl95U9Jn+TO9Bzt3KWr/vjLBt1+8LjGxCXovJ9WndZnWdcqMI/0WSan6u4jeaYKFu6ztPSTpd3UJA63dbl95U/rJvVp3aQ+29evIioqitT4aIJbNWH06NEsmju99Lw3uM2owDzxsCd4bvJfX8LSwdJuahKH27rcdvPH12qosrDDa7ilEzzlZer9WU3icFuX2xt12o27NrC6OscMLB0s7aYmcbity203f3ythjoN/h8rrZ3gsZuaxOG2Lrfd/PEUd6oHEjyJyal64Gi+qYKFEzw+b0BlwTK3wH5qEofbutx288cT3J4IlkkpqXrwWL6pYuVg6Sh4HDhwUCH69+7BiloqeJJT03T2D0tM2bZrWs9R8Dhw4ODMhR26LC09dAj8R33hcPs/t9388QZ3TWGHoUM+7weorM/S6uoLh9s+3Hbzx1PcnumzTNOM7EJTBQv3WVr6ydLq6guH2z7cdvPHW+sZ1QSCPZ4sLR0s7aaAcLity203fxwFj+dRZbAUkU4i8r2IbBCRdSJyl3G8lYjMFpEtxmfLCq4fJiKbRGSriIyvTuPKy9T7swLC4bYutzfqtBt3bXCmrMFTCNyrqjFAH1yrNMYC44G5qhoFzDX2T4GIBAKv4lrdMRYYbVxrCnZTQDjc1uW2mz+WUvCYfKq0+pNltTs5ga+BC4BNQLBxLBjYVI5tX+A7t/0HgAfMJnjsoIBwuP2D227+WEnBk5Kapsdyi0wVLJzgqW6g7ALsApoBWWXOZZZjfxXwltv+GGCC2WCZW+D/CgiH23+47eaPVRQ8Kalpeuxkkali5WBpWsEjIk2ABcBTqvqFiGSpagu385mq2rLMNSOAoap6k7E/BuilqneWU/8twC0AncLC0jZv22mqXQ4cOPAePKHgSU3roT8sXmbKtmnDAMsqeExlw0WkHvA58JGqfmEcPigiwcb5YOBQOZfuATq57XcE9pXHoaoTVbWHqvZo26at2fY7cODAD3BGTP4rrrTY28AGVX3B7dRU4Hpj+3pcfZllsQyIEpGuIlIfGGVc58CBgzMJYrKYqaoWI2xqAzNPlv1x9TWeJyKrjDIceAa4QES24Er4PAMgIiEiMh1AVQuBO4DvgA3AFFVd5wU/HDhwYGF4auhQbUfY1AZVTqShqouoOOYPLsd+HzDcbX86ML2mDXTgwIF/o0TB4yH0Araq6nYAEZkMXAqs9xhDBbDkrEMrV65IP6uelM3wtAHSfdEeL8FO/tjJF3D8cUfn2pKvXLniu7PqSRuT5g1FxH1+xomqOtFtPxTY7ba/B+hd2zaagSWDpaqeluERkeVWzZLVBHbyx06+gOOPp6GqwzxYXXnPqHUyKa+lteEOHDhwUAamR9h4Gk6wdODAgT/BZyNsLPkaXgEmVm3iV7CTP3byBRx/LAtVLRSRkhE2gcA7dTXCxpJr8Dhw4MCB1eC8hjtw4MCBCTjB0oEDBw5MwPLB0lfSJm9BRHaIyK+GEsrv1vsVkXdE5JCIrHU7ZmoiaCuiAn8eFZG9ZRRrlkdtJ+p2UDksHSx9KW3yMs5V1WQ/Hcv3LlB23FyVE0FbGO9yuj8A/zG+o2RDheYPqPFE3Q6qhqWDJW7SJlXNB0qkTQ58BFX9AThS5vClwHvG9nvAZXXZptqgAn/8Eqq6X1VXGtvHcc3HEIoffz9WgtWDZXnSplAftcVTUGCWiKww5vC0A9qr6n5w/cEC7XzcHk/gDhFZY7ym+91rq4h0AVKAJdjz+6lzWD1Y+kza5EX0V9VUXF0Lt4vIIF83yMFpeB2IAJKB/cC/fdqaasKYqPtz4G5VPebr9tgFVg+WPpM2eQvGrEyo6iHgS1xdDf4OMxNB+w1U9aCqFqlqMfAmfvQd1WKibgdVwOrB0laTB4tIYxFpWrINDAHWVn6VX8DMRNB+g5LAYuBy/OQ7quVE3Q6qgOUVPMawjRf5Xdr0lG9bVHOISDiup0lwSU0/9jd/RGQScA6uab8OAv8AvgKmAGG4FrQboap+kTSpwJ9zcL2CK7ADuLWkz8/KEJEBwELgV6DYOPwgrn5Lv/x+rATLB0sHDhw4sAKs/hruwIEDB5aAEywdOHDgwAScYOnAgQMHJuAESwcOHDgwASdYOnDgwIEJOMHSgQMHDkzACZYOHDhwYAL/D8qB+mYi+YbcAAAAAElFTkSuQmCC\n",
|
|
"text/plain": [
|
|
"<Figure size 432x288 with 2 Axes>"
|
|
]
|
|
},
|
|
"metadata": {
|
|
"needs_background": "light"
|
|
},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"import itertools\n",
|
|
"\n",
|
|
"plt.title(\"confusion matrix\")\n",
|
|
"plt.imshow(confusion,cmap='Blues')\n",
|
|
"\n",
|
|
"plt.colorbar()\n",
|
|
"for i,j in itertools.product(range(confusion.shape[0]),range(confusion.shape[1])):\n",
|
|
" plt.text(j,i,\"{:}\".format(confusion[i,j]),horizontalalignment=\"center\",color=\"black\" if i == j else \"black\")\n",
|
|
"plt.show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "543ede4b",
|
|
"metadata": {},
|
|
"source": [
|
|
"혼동행렬을 보면 별로인것 같습니다.\n",
|
|
"왼쪽이 predict이고 밑이 actual입니다."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "9e85493d",
|
|
"metadata": {},
|
|
"source": [
|
|
"$$Precision(class = a) = \\frac{TP(class = a)}{TP(class = a)+FP(class = a)}$$\n",
|
|
"\n",
|
|
"$$Recall(class = a) = \\frac{TP(class = a)}{TP(class = a)+FN(class = a)}$$\n",
|
|
"\n",
|
|
"$$F1Score(class = a) = \\frac{2}{\\frac{1}{Precision(class = a)}+\\frac{1}{Recall(class = a)} }$$"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "200c95b9",
|
|
"metadata": {},
|
|
"source": [
|
|
"F1Score는 다음과 같이 주어집니다."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "0b23e7d5",
|
|
"metadata": {},
|
|
"source": [
|
|
"다른 클래스에 대해서도 모두 해보자"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 75,
|
|
"id": "38b3eee6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def getF1Score(confusion,c):\n",
|
|
" TP = confusion[c,c]\n",
|
|
" FP = confusion[c].sum() - TP\n",
|
|
" FN = confusion[:,c].sum() - TP\n",
|
|
" precision = TP / (TP + FP)\n",
|
|
" recall = TP / (TP + FN)\n",
|
|
"\n",
|
|
" f1Score = (2*precision*recall)/(precision + recall)\n",
|
|
" return f1Score"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 77,
|
|
"id": "61fe2d6c",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"class 0 f1 score : nan\n",
|
|
"class 1 f1 score : nan\n",
|
|
"class 2 f1 score : nan\n",
|
|
"class 3 f1 score : nan\n",
|
|
"class 4 f1 score : 0.9583332538604736\n",
|
|
"class 5 f1 score : 0.9216590523719788\n",
|
|
"class 6 f1 score : 0.9232480525970459\n",
|
|
"class 7 f1 score : 0.9203747510910034\n",
|
|
"class 8 f1 score : 0.8780487179756165\n",
|
|
"class 9 f1 score : nan\n",
|
|
"class 10 f1 score : nan\n",
|
|
"class 11 f1 score : nan\n",
|
|
"class 12 f1 score : nan\n",
|
|
"class 13 f1 score : nan\n",
|
|
"class 14 f1 score : 0.9240506887435913\n",
|
|
"class 15 f1 score : 0.7439999580383301\n",
|
|
"class 16 f1 score : 0.885114848613739\n",
|
|
"class 17 f1 score : 0.9293712377548218\n",
|
|
"class 18 f1 score : 0.932692289352417\n",
|
|
"class 19 f1 score : nan\n",
|
|
"class 20 f1 score : nan\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for i in range(21):\n",
|
|
" f1 = getF1Score(confusion,i)\n",
|
|
" print(f\"class {i} f1 score : {f1}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0b9b55e7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.7.11"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|