monoid/ner-study
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix: print out stderr

Browse Source
This commit is contained in:
monoid 2022-02-22 17:45:53 +09:00
parent a7e447b6d6
commit 54e757c247
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
preprocessing.py
View File

@ -1,5 +1,6 @@
import sys
from read_data import TagIdConverter, make_long_namedEntity, readKoreanDataAll, Sentence from read_data import TagIdConverter, make_long_namedEntity, readKoreanDataAll, Sentence
from typing import Any, NamedTuple, List, Sequence, TypeVar from typing import Any, NamedTuple, List, Sequence, TypeVar
import json import json
@ -45,16 +46,16 @@ if __name__ == "__main__":
PRETAINED_MODEL_NAME = 'bert-base-multilingual-cased' PRETAINED_MODEL_NAME = 'bert-base-multilingual-cased'
rawTrain, rawDev, rawTest = readKoreanDataAll() rawTrain, rawDev, rawTest = readKoreanDataAll()
print("load tokenzier...") print("load tokenzier...",file=sys.stderr)
tokenizer = BertTokenizer.from_pretrained(PRETAINED_MODEL_NAME) tokenizer = BertTokenizer.from_pretrained(PRETAINED_MODEL_NAME)
converter = TagIdConverter() converter = TagIdConverter()
print("process train...") print("process train...",file=sys.stderr)
train = preprocessing(tokenizer,converter,rawTrain) train = preprocessing(tokenizer,converter,rawTrain)
saveObject(path.join(PRE_BASE_PATH,"train.json"),train) saveObject(path.join(PRE_BASE_PATH,"train.json"),train)
print("process dev...") print("process dev...",file=sys.stderr)
dev = preprocessing(tokenizer,converter,rawDev) dev = preprocessing(tokenizer,converter,rawDev)
saveObject(path.join(PRE_BASE_PATH,"dev.json"),dev) saveObject(path.join(PRE_BASE_PATH,"dev.json"),dev)
print("process test...") print("process test...",file=sys.stderr)
test = preprocessing(tokenizer,converter,rawTest) test = preprocessing(tokenizer,converter,rawTest)
saveObject(path.join(PRE_BASE_PATH,"test.json"),test) saveObject(path.join(PRE_BASE_PATH,"test.json"),test)