From 914527d9ada9a064a20e8b870be3057728333997 Mon Sep 17 00:00:00 2001 From: Igor Maslov Date: Fri, 10 Aug 2018 17:16:56 +0300 Subject: [PATCH 1/6] feat: allow config to be both a path to config or an already parsed dict in build_model_from_config (#362) * issue #361 fix --- CNAME | 1 + deeppavlov/core/commands/infer.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 CNAME diff --git a/CNAME b/CNAME new file mode 100644 index 0000000000..822ec1af2e --- /dev/null +++ b/CNAME @@ -0,0 +1 @@ +deeppavlov.ai \ No newline at end of file diff --git a/deeppavlov/core/commands/infer.py b/deeppavlov/core/commands/infer.py index b1a5e70ab8..12b4375c4a 100644 --- a/deeppavlov/core/commands/infer.py +++ b/deeppavlov/core/commands/infer.py @@ -13,6 +13,9 @@ See the License for the specific language governing permissions and limitations under the License. """ + +from pathlib import Path + from deeppavlov.core.commands.utils import set_deeppavlov_root, import_packages from deeppavlov.core.common.chainer import Chainer from deeppavlov.core.common.file import read_json @@ -25,7 +28,9 @@ log = get_logger(__name__) -def build_model_from_config(config, mode='infer', load_trained=False, as_component=False): +def build_model_from_config(config: [str, Path, dict], mode='infer', load_trained=False, as_component=False): + if isinstance(config, (str, Path)): + config = read_json(config) set_deeppavlov_root(config) import_packages(config.get('metadata', {}).get('imports', [])) From 5ab6b71fb9b7bc7b9ae459d24a268380117261a0 Mon Sep 17 00:00:00 2001 From: Aleksey Lymar Date: Fri, 10 Aug 2018 17:17:48 +0300 Subject: [PATCH 2/6] fix: fix various morphotagger bugs (#367) * fix: fix morphotagger predict configs * tests: add tests for more morphotagger configs * fix: make non-pymorphy morphotagger tag all tokens instead of only the first one * tests: use find_config() in run_model * fix: move model summary in morphotagger from stdout to logging * fix: fix training for non-pymorphy morphotagger configs --- deeppavlov/models/morpho_tagger/common.py | 2 +- deeppavlov/models/morpho_tagger/network.py | 19 +++++-------------- deeppavlov/models/morpho_tagger/tagger.py | 8 ++------ deeppavlov/run_model.py | 5 ++++- tests/test_quick_start.py | 9 ++++++++- 5 files changed, 20 insertions(+), 23 deletions(-) diff --git a/deeppavlov/models/morpho_tagger/common.py b/deeppavlov/models/morpho_tagger/common.py index 6274927c13..287950c90c 100644 --- a/deeppavlov/models/morpho_tagger/common.py +++ b/deeppavlov/models/morpho_tagger/common.py @@ -107,7 +107,7 @@ class TagOutputPrettifier(Component): """ def __init__(self, return_string: bool=True, begin: str="", - end: str ="", sep: str ="\n"): + end: str ="", sep: str ="\n", **kwargs): self.return_string = return_string self.begin = begin diff --git a/deeppavlov/models/morpho_tagger/network.py b/deeppavlov/models/morpho_tagger/network.py index f047f0a09f..debce29a6c 100644 --- a/deeppavlov/models/morpho_tagger/network.py +++ b/deeppavlov/models/morpho_tagger/network.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import inspect -import json from typing import List import keras.layers as kl @@ -110,7 +108,7 @@ def build(self): self.model_ = Model(inputs, outputs) self.model_.compile(**compile_args) if self.verbose > 0: - log.info(str(self.model_.summary())) + self.model_.summary(print_fn=log.info) return self def build_word_cnn(self, inputs): @@ -172,12 +170,10 @@ def build_basic_network(self, word_outputs): return pre_outputs, lstm_outputs def _transform_batch(self, data, labels=None, transform_to_one_hot=True): - if len(self.word_vectorizers) > 0: - data, additional_data = data[0], data[1:] + data, additional_data = data[0], data[1:] L = max(len(x) for x in data) X = np.array([self._make_sent_vector(x, L) for x in data]) - if len(self.word_vectorizers) > 0: - X = [X] + [np.array(x) for x in additional_data] + X = [X] + [np.array(x) for x in additional_data] if labels is not None: Y = np.array([self._make_tags_vector(y, L) for y in labels]) if transform_to_one_hot: @@ -197,7 +193,7 @@ def train_on_batch(self, data, labels): # TO_DO: add weights to deal with padded instances return self.model_.train_on_batch(X, Y) - def predict_on_batch(self, data: List, return_indexes=False): + def predict_on_batch(self, data: [list, tuple], return_indexes=False): """ Makes predictions on a single batch @@ -206,10 +202,7 @@ def predict_on_batch(self, data: List, return_indexes=False): answer: a batch of label sequences """ X = self._transform_batch(data) - if len(self.word_vectorizers) > 0: - objects_number, lengths = len(X[0]), [len(elem) for elem in data[0]] - else: - objects_number, lengths = len(X), [len(elem) for elem in data] + objects_number, lengths = len(X[0]), [len(elem) for elem in data[0]] Y = self.model_.predict_on_batch(X) labels = np.argmax(Y, axis=-1) answer: List[List[str]] = [None] * objects_number @@ -245,5 +238,3 @@ def save(self, outfile): def load(self, infile): self.model_.load_weights(infile) - - diff --git a/deeppavlov/models/morpho_tagger/tagger.py b/deeppavlov/models/morpho_tagger/tagger.py index 650b6fbd54..19bf5fdaac 100644 --- a/deeppavlov/models/morpho_tagger/tagger.py +++ b/deeppavlov/models/morpho_tagger/tagger.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import copy import inspect @@ -104,11 +103,8 @@ def train_on_batch(self, *args): Returns: """ - if len(args) > 2: - data, labels = [list(x) for x in args[:-1]], list(args[-1]) - else: - data, labels = args - self._net.train_on_batch(data, labels, **self.train_parameters) + *data, labels = args + self._net.train_on_batch(data, labels) def __call__(self, *x_batch, **kwargs): """ diff --git a/deeppavlov/run_model.py b/deeppavlov/run_model.py index 99a432106f..906b1cb522 100644 --- a/deeppavlov/run_model.py +++ b/deeppavlov/run_model.py @@ -14,13 +14,14 @@ limitations under the License. """ +from deeppavlov.deep import find_config from deeppavlov.core.commands.train import train_evaluate_model_from_config from deeppavlov.core.commands.infer import interact_model # PIPELINE_CONFIG_PATH = 'configs/intents/intents_dstc2.json' # PIPELINE_CONFIG_PATH = 'configs/intents/intents_snips.json' -PIPELINE_CONFIG_PATH = 'configs/ner/ner_dstc2.json' +# PIPELINE_CONFIG_PATH = 'configs/ner/ner_dstc2.json' # PIPELINE_CONFIG_PATH = 'configs/ner/ner_rus.json' # PIPELINE_CONFIG_PATH = 'configs/ner/slotfill_dstc2.json' # PIPELINE_CONFIG_PATH = 'configs/error_model/brillmoore_wikitypos_en.json' @@ -37,6 +38,8 @@ # PIPELINE_CONFIG_PATH = 'configs/odqa/en_odqa_infer_prod.json' # PIPELINE_CONFIG_PATH = 'configs/odqa/ru_odqa_infer_prod.json' # PIPELINE_CONFIG_PATH = 'configs/odqa/ranker_test.json' +# PIPELINE_CONFIG_PATH = find_config('morpho_ru_syntagrus_train') +PIPELINE_CONFIG_PATH = find_config('morpho_ru_syntagrus_train_pymorphy') if __name__ == '__main__': diff --git a/tests/test_quick_start.py b/tests/test_quick_start.py index 5fb8a78203..64ba7637e0 100644 --- a/tests/test_quick_start.py +++ b/tests/test_quick_start.py @@ -112,8 +112,15 @@ }, "morpho_tagger":{ ("morpho_tagger/UD2.0/hu/morpho_hu_train.json", "morpho_tagger_hu", ALL_MODES): [ONE_ARGUMENT_INFER_CHECK], + ("morpho_tagger/UD2.0/hu/morpho_hu_predict.json", "morpho_tagger_hu", ('IP',)): [ONE_ARGUMENT_INFER_CHECK], ("morpho_tagger/UD2.0/ru_syntagrus/morpho_ru_syntagrus_train_pymorphy.json", - "morpho_tagger_pymorphy", ALL_MODES): [ONE_ARGUMENT_INFER_CHECK] + "morpho_tagger_pymorphy", ALL_MODES): [ONE_ARGUMENT_INFER_CHECK], + ("morpho_tagger/UD2.0/ru_syntagrus/morpho_ru_syntagrus_train.json", + "morpho_tagger_pymorphy", ALL_MODES): [ONE_ARGUMENT_INFER_CHECK], + ("morpho_tagger/UD2.0/ru_syntagrus/morpho_ru_syntagrus_predict.json", + "morpho_tagger_pymorphy", ('IP',)): [ONE_ARGUMENT_INFER_CHECK], + ("morpho_tagger/UD2.0/ru_syntagrus/morpho_ru_syntagrus_predict_pymorphy.json", + "morpho_tagger_pymorphy", ('IP',)): [ONE_ARGUMENT_INFER_CHECK] } } From 0d57948815fed6c84aa09570159e0997e1a6cf0d Mon Sep 17 00:00:00 2001 From: Aleksey Lymar Date: Fri, 10 Aug 2018 17:18:36 +0300 Subject: [PATCH 3/6] tests: test a proper DeepPavlov installation with pip * tests: use library paths for deeppavlov and utils instead of relative to tests * feat: downloads are done relative to deeppavlov_root !broken commit! * feat: downloads are done relative to deeppavlov_root * tests: run tests with new downloads system * feat: allow deep_download to accept just a path to a config * tests: run tests on install instead of develop in Jenkins * tests: try and fix jenkins env activation * tests: try and fix jenkins packages installation order * tests: fix removing files in Jenkinsfile * tests: fix env activation in Tests stage on Jenkins * fix: fix the `No module named 'deeppavlov.models.evolution'` error when installing not for development * chore: do not explicitly install requirements in setup.py as this step is anyway ignored when installing from a wheel --- Jenkinsfile | 8 +-- deeppavlov/download.py | 92 +++++++++---------------- deeppavlov/models/evolution/__init__.py | 0 setup.py | 6 +- tests/test_quick_start.py | 14 ++-- 5 files changed, 44 insertions(+), 76 deletions(-) create mode 100644 deeppavlov/models/evolution/__init__.py diff --git a/Jenkinsfile b/Jenkinsfile index 20f5946d45..4912b09c51 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,11 +9,11 @@ node('gpu') { stage('Setup') { env.CUDA_VISIBLE_DEVICES=0 sh """ - virtualenv --python=python3 ".venv-$BUILD_NUMBER" - . .venv-$BUILD_NUMBER/bin/activate - sed -i "s/stream=True/stream=False/g" deeppavlov/core/data/utils.py - pip install -e .[tests] + virtualenv --python=python3 '.venv-$BUILD_NUMBER' + . '.venv-$BUILD_NUMBER/bin/activate' + pip install .[tests] pip install -r dp_requirements/tf-gpu.txt + rm -rf `find . -mindepth 1 -maxdepth 1 ! -name tests ! -name Jenkinsfile ! -name '.venv-$BUILD_NUMBER'` """ } stage('Tests') { diff --git a/deeppavlov/download.py b/deeppavlov/download.py index 407f514cb0..ee7f902f62 100644 --- a/deeppavlov/download.py +++ b/deeppavlov/download.py @@ -15,12 +15,12 @@ """ import argparse +from collections import defaultdict from pathlib import Path import sys -root_path = (Path(__file__) / ".." / "..").resolve() -sys.path.append(str(root_path)) - +import deeppavlov +from deeppavlov.core.commands.utils import get_deeppavlov_root, set_deeppavlov_root, expand_path from deeppavlov.core.common.file import read_json from deeppavlov.core.data.utils import download, download_decompress, get_all_elems_from_json from deeppavlov.core.common.log import get_logger @@ -35,73 +35,52 @@ parser.add_argument('-all', action='store_true', help="Download everything. Warning! There should be at least 10 GB space" " available on disk.") -parser.add_argument('-test', action='store_true', - help="Turn test mode") -def get_config_downloads(config_path, config_downloads=None): +def get_config_downloads(config_path): + dp_root_back = get_deeppavlov_root() config = read_json(config_path) + set_deeppavlov_root(config) - if config_downloads is None: - config_downloads = {} - + downloads = set() if 'metadata' in config and 'download' in config['metadata']: for resource in config['metadata']['download']: if isinstance(resource, str): - url = resource - sub_dir = '' - elif isinstance(resource, dict): - url = resource['url'] - sub_dir = resource['subdir'] if 'subdir' in resource else '' + resource = { + 'url': resource + } - if url in config_downloads: - config_downloads[url]['subdir'] = list(set(config_downloads[url]['subdir'] + - [sub_dir])) - else: - config_downloads[url] = {'url': url, 'subdir': [sub_dir]} + url = resource['url'] + dest = expand_path(resource.get('subdir', '')) - config_references = get_all_elems_from_json(config, 'config_path') - config_references = [root_path.joinpath(config_ref.split('../', 1)[1]) for config_ref in config_references] + downloads.add((url, dest)) - for config_ref in config_references: - config_downloads = get_config_downloads(config_ref, config_downloads) + config_references = [expand_path(config_ref) for config_ref in get_all_elems_from_json(config, 'config_path')] - return config_downloads + downloads |= {(url, dest) for config in config_references for url, dest in get_config_downloads(config)} + set_deeppavlov_root({'deeppavlov_root': dp_root_back}) -def get_configs_downloads(config_path=None, test=None): - all_downloads = {} + return downloads - if test: - configs_path = root_path / 'tests' / 'deeppavlov' / 'configs' - else: - configs_path = root_path / 'deeppavlov' / 'configs' + +def get_configs_downloads(config_path=None): + all_downloads = defaultdict(set) if config_path: configs = [config_path] else: - configs = list(configs_path.glob('**/*.json')) + configs = list(Path(deeppavlov.__path__[0], 'configs').glob('**/*.json')) for config_path in configs: - config_downloads = get_config_downloads(config_path) - for url in config_downloads: - if url in all_downloads: - all_downloads[url]['subdir'] = list(set(all_downloads[url]['subdir'] + - config_downloads[url]['subdir'])) - else: - all_downloads[url] = config_downloads[url] + for url, dest in get_config_downloads(config_path): + all_downloads[url].add(dest) return all_downloads -def download_resource(resource, download_path): - url = resource['url'] - sub_dirs = resource['subdir'] - dest_paths = [] - - for sub_dir in sub_dirs: - dest_path = download_path.joinpath(sub_dir) - dest_paths.append(dest_path) +def download_resource(url, dest_paths): + dest_paths = list(dest_paths) if url.endswith(('.tar.gz', '.gz', '.zip')): download_path = dest_paths[0].parent @@ -113,31 +92,22 @@ def download_resource(resource, download_path): def download_resources(args): - download_path = root_path / 'download' - - if args.test: - download_path = root_path / 'tests' / 'download' - test = True - else: - test = False - if not args.all and not args.config: log.error('You should provide either skill config path or -all flag') sys.exit(1) elif args.all: - downloads = get_configs_downloads(test=test) + downloads = get_configs_downloads() else: config_path = Path(args.config).resolve() downloads = get_configs_downloads(config_path=config_path) - download_path.mkdir(exist_ok=True) - - for url in downloads: - resource = downloads[url] - download_resource(resource, download_path) + for url, dest_paths in downloads.items(): + download_resource(url, dest_paths) -def deep_download(args=None): +def deep_download(args: [str, Path, list]=None): + if isinstance(args, (str, Path)): + args = ['-c', str(args)] # if args is a path to config args = parser.parse_args(args) log.info("Downloading...") download_resources(args) diff --git a/deeppavlov/models/evolution/__init__.py b/deeppavlov/models/evolution/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/setup.py b/setup.py index 23ec794f6e..db0f2ab4ee 100644 --- a/setup.py +++ b/setup.py @@ -14,20 +14,16 @@ import re import deeppavlov -from utils.pip_wrapper import install __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) def read_requirements(): - # # parses requirements from requirements.txt + """parses requirements from requirements.txt""" reqs_path = os.path.join(__location__, 'requirements.txt') with open(reqs_path, encoding='utf8') as f: reqs = [line.strip() for line in f if not line.strip().startswith('#')] - for req in reqs: - install(req) - names = [] links = [] for req in reqs: diff --git a/tests/test_quick_start.py b/tests/test_quick_start.py index 64ba7637e0..c31d3e4f4f 100644 --- a/tests/test_quick_start.py +++ b/tests/test_quick_start.py @@ -11,15 +11,17 @@ import requests from urllib.parse import urljoin +import deeppavlov from deeppavlov.download import deep_download from deeppavlov.core.data.utils import get_all_elems_from_json +import utils from utils.server_utils.server import get_server_params, SERVER_CONFIG_FILENAME cache_dir = None -tests_dir = Path(__file__, '..').resolve() +tests_dir = Path(__file__).parent test_configs_path = tests_dir / "deeppavlov" / "configs" -src_dir = tests_dir.parent / "deeppavlov" / "configs" +src_dir = Path(deeppavlov.__path__[0]) / "configs" test_src_dir = tests_dir / "test_configs" download_path = tests_dir / "download" @@ -238,7 +240,7 @@ def interact(conf_file, model_dir, qr_list=None): @staticmethod def interact_api(conf_file): - server_conf_file = Path(tests_dir, "..").resolve() / "utils" / "server_utils" / SERVER_CONFIG_FILENAME + server_conf_file = Path(utils.__path__[0]) / "server_utils" / SERVER_CONFIG_FILENAME server_params = get_server_params(server_conf_file, conf_file) model_args_names = server_params['model_args_names'] @@ -278,7 +280,7 @@ def test_interacting_pretrained_model(self, model, conf_file, model_dir, mode): if 'IP' in mode: config_file_path = str(test_configs_path.joinpath(conf_file)) self.install(config_file_path) - deep_download(['-test', '-c', config_file_path]) + deep_download(['-c', config_file_path]) self.interact(test_configs_path / conf_file, model_dir, PARAMS[model][(conf_file, model_dir, mode)]) else: @@ -301,7 +303,7 @@ def test_consecutive_training_and_interacting(self, model, conf_file, model_dir, if 'IP' not in mode: config_path = str(test_configs_path.joinpath(conf_file)) self.install(config_path) - deep_download(['-test', '-c', config_path]) + deep_download(['-c', config_path]) shutil.rmtree(str(model_path), ignore_errors=True) logfile = io.BytesIO(b'') @@ -324,7 +326,7 @@ def test_evolving(self, model, conf_file, model_dir, mode): if 'IP' not in mode and 'TI' not in mode: config_path = str(test_configs_path.joinpath(conf_file)) - deep_download(['-test', '-c', config_path]) + deep_download(['-c', config_path]) shutil.rmtree(str(model_path), ignore_errors=True) logfile = io.BytesIO(b'') From 04d0299433918f14a9233d7f87be68b859f5f4ea Mon Sep 17 00:00:00 2001 From: Aleksey Lymar Date: Fri, 10 Aug 2018 17:19:09 +0300 Subject: [PATCH 4/6] logs: replace prints to stdout with logging (#369) * logs: move keras model summary to log.info * logs: remove prints in favor of logging --- deeppavlov/core/models/keras_model.py | 3 +-- .../models/classifiers/keras_classification_model.py | 7 +++++-- deeppavlov/models/classifiers/utils.py | 3 +-- deeppavlov/models/preprocessors/squad_preprocessor.py | 2 +- deeppavlov/models/ranking/ranking_model.py | 5 +++-- deeppavlov/models/seq2seq_go_bot/bot.py | 2 +- deeppavlov/models/squad/utils.py | 1 - 7 files changed, 12 insertions(+), 11 deletions(-) diff --git a/deeppavlov/core/models/keras_model.py b/deeppavlov/core/models/keras_model.py index 5db706cfe4..5564b591cd 100644 --- a/deeppavlov/core/models/keras_model.py +++ b/deeppavlov/core/models/keras_model.py @@ -99,8 +99,7 @@ def init_model_from_scratch(self, model_name: str, optimizer_name: str, Returns: compiled model with given network and learning parameters """ - log.info("[initializing `{}` from scratch]".format(self.__class__.__name__)) - print(model_name) + log.info(f'[initializing `{self.__class__.__name__}` from scratch as {model_name}]') model_func = getattr(self, model_name, None) if callable(model_func): model = model_func(**self.opt) diff --git a/deeppavlov/models/classifiers/keras_classification_model.py b/deeppavlov/models/classifiers/keras_classification_model.py index 35c1977976..446088cf97 100644 --- a/deeppavlov/models/classifiers/keras_classification_model.py +++ b/deeppavlov/models/classifiers/keras_classification_model.py @@ -115,7 +115,7 @@ def __init__(self, text_size: int, "lear_rate": self.opt.get('lear_rate'), "lear_rate_decay": self.opt.get('lear_rate_decay')} - self.model = self.load(**params) + self.model: Model = self.load(**params) self._change_not_fixed_params(text_size=text_size, model_name=model_name, optimizer=optimizer, loss=loss, lear_rate=lear_rate, lear_rate_decay=lear_rate_decay, @@ -133,7 +133,10 @@ def __init__(self, text_size: int, if self.opt['fasttext_md5'] != current_fasttext_md5: raise ConfigError( "Given fasttext model does NOT match fasttext model used previously to train loaded model") - print("Model was successfully initialized!\nModel summary:\n{}".format(self.model.summary())) + + summary = ['Model was successfully initialized!', 'Model summary:'] + self.model.summary(print_fn=summary.append) + log.info('\n'.join(summary)) def _change_not_fixed_params(self, **kwargs) -> None: """ diff --git a/deeppavlov/models/classifiers/utils.py b/deeppavlov/models/classifiers/utils.py index 905dce3d9d..ed4364cf64 100644 --- a/deeppavlov/models/classifiers/utils.py +++ b/deeppavlov/models/classifiers/utils.py @@ -104,8 +104,7 @@ def log_metrics(names: [list, np.ndarray], values: [list, np.ndarray], Returns: None """ - sys.stdout.write("\r") # back to previous line - log.info("{} -->\t".format(mode)) + log.info("\r{} -->\t".format(mode)) if updates is not None: log.info("updates: {}\t".format(updates)) diff --git a/deeppavlov/models/preprocessors/squad_preprocessor.py b/deeppavlov/models/preprocessors/squad_preprocessor.py index d0b32e9ebf..c8647f211f 100644 --- a/deeppavlov/models/preprocessors/squad_preprocessor.py +++ b/deeppavlov/models/preprocessors/squad_preprocessor.py @@ -138,7 +138,7 @@ def convert_idx(text: str, tokens: List[str]) -> List[Tuple[int, int]]: for token in tokens: current = text.find(token, current) if current < 0: - print("Token {} cannot be found".format(token)) + logger.error("Token {} cannot be found".format(token)) raise Exception() spans.append((current, current + len(token))) current += len(token) diff --git a/deeppavlov/models/ranking/ranking_model.py b/deeppavlov/models/ranking/ranking_model.py index 07e78be8e9..b17894e4c4 100644 --- a/deeppavlov/models/ranking/ranking_model.py +++ b/deeppavlov/models/ranking/ranking_model.py @@ -231,7 +231,8 @@ def make_hard_triplets(self, x, y, net): if not no_samples: break if no_samples: - print("There is no negative examples with distances greater than positive examples distances.") + log.error("There are no negative examples with distances" + " greater than positive examples distances.") exit(0) else: if self.num_hardest_negatives is not None: @@ -285,7 +286,7 @@ def make_hard_triplets(self, x, y, net): rp = [el[1] for el in triplets] rn = [el[2] for el in triplets] ratio = sum(hrds) / len(hrds) - print("Ratio of semi-hard negative samples is %f" % ratio) + log.info("Ratio of semi-hard negative samples is %f" % ratio) return [(c, rp), (c, rn)] def get_semi_hard_negative_ind(self, i, j, k, distances, anchor_negative_dist, batch_size, num_samples): diff --git a/deeppavlov/models/seq2seq_go_bot/bot.py b/deeppavlov/models/seq2seq_go_bot/bot.py index aa5167eab3..216b71e11d 100644 --- a/deeppavlov/models/seq2seq_go_bot/bot.py +++ b/deeppavlov/models/seq2seq_go_bot/bot.py @@ -125,7 +125,7 @@ def _filter(tokens): preds = [list(_filter(self.tgt_vocab(utter_idxs))) for utter_idxs in pred_idxs] if self.debug: - print("Dialog prediction = \"{}\"".format(preds[-1])) + log.debug("Dialog prediction = \"{}\"".format(preds[-1])) return preds def save(self): diff --git a/deeppavlov/models/squad/utils.py b/deeppavlov/models/squad/utils.py index 301da548c7..f04ab2b88a 100644 --- a/deeppavlov/models/squad/utils.py +++ b/deeppavlov/models/squad/utils.py @@ -98,7 +98,6 @@ def __call__(self, inputs, seq_len, keep_prob=1.0, is_train=None, concat_layers= gru_fw, gru_bw = self.grus[layer] init_fw, init_bw = self.inits[layer] mask_fw, mask_bw = self.dropout_mask[layer] - print(outputs) with tf.variable_scope('fw_{}'.format(layer), reuse=tf.AUTO_REUSE): with tf.variable_scope('cudnn_gru', reuse=tf.AUTO_REUSE): out_fw, _ = tf.nn.dynamic_rnn(cell=gru_fw, inputs=outputs[-1] * mask_fw, time_major=True, From 652667380ad8498ef88c2955e4197dd378472a71 Mon Sep 17 00:00:00 2001 From: Aleksey Lymar Date: Fri, 10 Aug 2018 17:29:23 +0300 Subject: [PATCH 5/6] chore: Release 0.0.6.6 --- deeppavlov/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeppavlov/__init__.py b/deeppavlov/__init__.py index c1eae8f596..d14adda914 100644 --- a/deeppavlov/__init__.py +++ b/deeppavlov/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.0.6.5' +__version__ = '0.0.6.6' __author__ = 'Neural Networks and Deep Learning lab, MIPT' __description__ = 'An open source library for building end-to-end dialog systems and training chatbots.' __keywords__ = ['NLP', 'NER', 'SQUAD', 'Intents', 'Chatbot'] From 0d0cb8d6bcad905f72ab618b75a9376f4c5d8620 Mon Sep 17 00:00:00 2001 From: Aleksey Lymar Date: Fri, 10 Aug 2018 23:29:39 +0300 Subject: [PATCH 6/6] fix: remove squad model from odqa download list --- deeppavlov/configs/odqa/ru_odqa_infer_wiki.json | 3 +-- tests/test_configs/odqa/en_odqa_infer_wiki_test.json | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/deeppavlov/configs/odqa/ru_odqa_infer_wiki.json b/deeppavlov/configs/odqa/ru_odqa_infer_wiki.json index 05563db319..df32bcf7e1 100644 --- a/deeppavlov/configs/odqa/ru_odqa_infer_wiki.json +++ b/deeppavlov/configs/odqa/ru_odqa_infer_wiki.json @@ -54,8 +54,7 @@ }, "download": [ "http://files.deeppavlov.ai/datasets/wikipedia/ruwiki.tar.gz", - "http://files.deeppavlov.ai/deeppavlov_data/ru_odqa.tar.gz", - "http://files.deeppavlov.ai/deeppavlov_data/squad_model_ru.tar.gz" + "http://files.deeppavlov.ai/deeppavlov_data/ru_odqa.tar.gz" ] } } \ No newline at end of file diff --git a/tests/test_configs/odqa/en_odqa_infer_wiki_test.json b/tests/test_configs/odqa/en_odqa_infer_wiki_test.json index 63e0f16082..e0070edc6e 100644 --- a/tests/test_configs/odqa/en_odqa_infer_wiki_test.json +++ b/tests/test_configs/odqa/en_odqa_infer_wiki_test.json @@ -76,8 +76,7 @@ }, "download": [ "http://files.deeppavlov.ai/datasets/wikipedia/wiki_test.tar.gz", - "http://files.deeppavlov.ai/deeppavlov_data/odqa_test.tar.gz", - "http://files.deeppavlov.ai/deeppavlov_data/squad_model_1.1.tar.gz" + "http://files.deeppavlov.ai/deeppavlov_data/odqa_test.tar.gz" ] } } \ No newline at end of file