From 2e9e6c6c2056953b5f9abc31c94142594d210a1e Mon Sep 17 00:00:00 2001 From: yunfan Date: Mon, 14 Jan 2019 19:13:52 +0800 Subject: [PATCH 01/32] - fix trainer with validate_every > 0 - refine & fix Transformer Encoder - refine & speed up biaffine parser --- fastNLP/core/trainer.py | 2 +- fastNLP/models/biaffine_parser.py | 82 ++++++++++++------------- fastNLP/modules/aggregator/attention.py | 81 +++++++++++++++++------- fastNLP/modules/encoder/transformer.py | 43 +++++++++---- reproduction/Biaffine_parser/cfg.cfg | 11 ++-- reproduction/Biaffine_parser/run.py | 5 +- test/models/test_biaffine_parser.py | 3 +- 7 files changed, 138 insertions(+), 89 deletions(-) diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index 109315a3..6dcc9c78 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -281,7 +281,7 @@ def _train(self): self.callback_manager.after_batch() if ((self.validate_every > 0 and self.step % self.validate_every == 0) or - (self.validate_every < 0 and self.step % len(data_iterator)) == 0) \ + (self.validate_every < 0 and self.step % len(data_iterator) == 0)) \ and self.dev_data is not None: eval_res = self._do_validation(epoch=epoch, step=self.step) eval_str = "Evaluation at Epoch {}/{}. Step:{}/{}. ".format(epoch, self.n_epochs, self.step, diff --git a/fastNLP/models/biaffine_parser.py b/fastNLP/models/biaffine_parser.py index fb687301..b9b9dd56 100644 --- a/fastNLP/models/biaffine_parser.py +++ b/fastNLP/models/biaffine_parser.py @@ -6,6 +6,7 @@ from torch.nn import functional as F from fastNLP.modules.utils import initial_parameter from fastNLP.modules.encoder.variational_rnn import VarLSTM +from fastNLP.modules.encoder.transformer import TransformerEncoder from fastNLP.modules.dropout import TimestepDropout from fastNLP.models.base_model import BaseModel from fastNLP.modules.utils import seq_mask @@ -197,53 +198,49 @@ def __init__(self, pos_vocab_size, pos_emb_dim, num_label, - word_hid_dim=100, - pos_hid_dim=100, rnn_layers=1, rnn_hidden_size=200, arc_mlp_size=100, label_mlp_size=100, dropout=0.3, - use_var_lstm=False, + encoder='lstm', use_greedy_infer=False): super(BiaffineParser, self).__init__() rnn_out_size = 2 * rnn_hidden_size + word_hid_dim = pos_hid_dim = rnn_hidden_size self.word_embedding = nn.Embedding(num_embeddings=word_vocab_size, embedding_dim=word_emb_dim) self.pos_embedding = nn.Embedding(num_embeddings=pos_vocab_size, embedding_dim=pos_emb_dim) self.word_fc = nn.Linear(word_emb_dim, word_hid_dim) self.pos_fc = nn.Linear(pos_emb_dim, pos_hid_dim) self.word_norm = nn.LayerNorm(word_hid_dim) self.pos_norm = nn.LayerNorm(pos_hid_dim) - self.use_var_lstm = use_var_lstm - if use_var_lstm: - self.lstm = VarLSTM(input_size=word_hid_dim + pos_hid_dim, - hidden_size=rnn_hidden_size, - num_layers=rnn_layers, - bias=True, - batch_first=True, - input_dropout=dropout, - hidden_dropout=dropout, - bidirectional=True) + self.encoder_name = encoder + if encoder == 'var-lstm': + self.encoder = VarLSTM(input_size=word_hid_dim + pos_hid_dim, + hidden_size=rnn_hidden_size, + num_layers=rnn_layers, + bias=True, + batch_first=True, + input_dropout=dropout, + hidden_dropout=dropout, + bidirectional=True) + elif encoder == 'lstm': + self.encoder = nn.LSTM(input_size=word_hid_dim + pos_hid_dim, + hidden_size=rnn_hidden_size, + num_layers=rnn_layers, + bias=True, + batch_first=True, + dropout=dropout, + bidirectional=True) else: - self.lstm = nn.LSTM(input_size=word_hid_dim + pos_hid_dim, - hidden_size=rnn_hidden_size, - num_layers=rnn_layers, - bias=True, - batch_first=True, - dropout=dropout, - bidirectional=True) - - self.arc_head_mlp = nn.Sequential(nn.Linear(rnn_out_size, arc_mlp_size), - nn.LayerNorm(arc_mlp_size), + raise ValueError('unsupported encoder type: {}'.format(encoder)) + + self.mlp = nn.Sequential(nn.Linear(rnn_out_size, arc_mlp_size * 2 + label_mlp_size * 2), nn.ELU(), TimestepDropout(p=dropout),) - self.arc_dep_mlp = copy.deepcopy(self.arc_head_mlp) - self.label_head_mlp = nn.Sequential(nn.Linear(rnn_out_size, label_mlp_size), - nn.LayerNorm(label_mlp_size), - nn.ELU(), - TimestepDropout(p=dropout),) - self.label_dep_mlp = copy.deepcopy(self.label_head_mlp) + self.arc_mlp_size = arc_mlp_size + self.label_mlp_size = label_mlp_size self.arc_predictor = ArcBiaffine(arc_mlp_size, bias=True) self.label_predictor = LabelBilinear(label_mlp_size, label_mlp_size, num_label, bias=True) self.use_greedy_infer = use_greedy_infer @@ -286,24 +283,22 @@ def forward(self, word_seq, pos_seq, seq_lens, gold_heads=None): word, pos = self.word_fc(word), self.pos_fc(pos) word, pos = self.word_norm(word), self.pos_norm(pos) x = torch.cat([word, pos], dim=2) # -> [N,L,C] - del word, pos - # lstm, extract features + # encoder, extract features sort_lens, sort_idx = torch.sort(seq_lens, dim=0, descending=True) x = x[sort_idx] x = nn.utils.rnn.pack_padded_sequence(x, sort_lens, batch_first=True) - feat, _ = self.lstm(x) # -> [N,L,C] + feat, _ = self.encoder(x) # -> [N,L,C] feat, _ = nn.utils.rnn.pad_packed_sequence(feat, batch_first=True) _, unsort_idx = torch.sort(sort_idx, dim=0, descending=False) feat = feat[unsort_idx] # for arc biaffine # mlp, reduce dim - arc_dep = self.arc_dep_mlp(feat) - arc_head = self.arc_head_mlp(feat) - label_dep = self.label_dep_mlp(feat) - label_head = self.label_head_mlp(feat) - del feat + feat = self.mlp(feat) + arc_sz, label_sz = self.arc_mlp_size, self.label_mlp_size + arc_dep, arc_head = feat[:,:,:arc_sz], feat[:,:,arc_sz:2*arc_sz] + label_dep, label_head = feat[:,:,2*arc_sz:2*arc_sz+label_sz], feat[:,:,2*arc_sz+label_sz:] # biaffine arc classifier arc_pred = self.arc_predictor(arc_head, arc_dep) # [N, L, L] @@ -349,7 +344,7 @@ def loss(arc_pred, label_pred, arc_true, label_true, mask): batch_size, seq_len, _ = arc_pred.shape flip_mask = (mask == 0) _arc_pred = arc_pred.clone() - _arc_pred.masked_fill_(flip_mask.unsqueeze(1), -np.inf) + _arc_pred.masked_fill_(flip_mask.unsqueeze(1), -float('inf')) arc_logits = F.log_softmax(_arc_pred, dim=2) label_logits = F.log_softmax(label_pred, dim=2) batch_index = torch.arange(batch_size, device=arc_logits.device, dtype=torch.long).unsqueeze(1) @@ -357,12 +352,11 @@ def loss(arc_pred, label_pred, arc_true, label_true, mask): arc_loss = arc_logits[batch_index, child_index, arc_true] label_loss = label_logits[batch_index, child_index, label_true] - arc_loss = arc_loss[:, 1:] - label_loss = label_loss[:, 1:] - - float_mask = mask[:, 1:].float() - arc_nll = -(arc_loss*float_mask).mean() - label_nll = -(label_loss*float_mask).mean() + byte_mask = flip_mask.byte() + arc_loss.masked_fill_(byte_mask, 0) + label_loss.masked_fill_(byte_mask, 0) + arc_nll = -arc_loss.mean() + label_nll = -label_loss.mean() return arc_nll + label_nll def predict(self, word_seq, pos_seq, seq_lens): diff --git a/fastNLP/modules/aggregator/attention.py b/fastNLP/modules/aggregator/attention.py index 3fea1b10..233f2a1e 100644 --- a/fastNLP/modules/aggregator/attention.py +++ b/fastNLP/modules/aggregator/attention.py @@ -5,6 +5,7 @@ from torch import nn from fastNLP.modules.utils import mask_softmax +from fastNLP.modules.dropout import TimestepDropout class Attention(torch.nn.Module): @@ -23,47 +24,81 @@ def _atten_forward(self, query, memory): class DotAtte(nn.Module): - def __init__(self, key_size, value_size): + def __init__(self, key_size, value_size, dropout=0.1): super(DotAtte, self).__init__() self.key_size = key_size self.value_size = value_size self.scale = math.sqrt(key_size) + self.drop = nn.Dropout(dropout) + self.softmax = nn.Softmax(dim=2) - def forward(self, Q, K, V, seq_mask=None): + def forward(self, Q, K, V, mask_out=None): """ :param Q: [batch, seq_len, key_size] :param K: [batch, seq_len, key_size] :param V: [batch, seq_len, value_size] - :param seq_mask: [batch, seq_len] + :param mask_out: [batch, seq_len] """ output = torch.matmul(Q, K.transpose(1, 2)) / self.scale - if seq_mask is not None: - output.masked_fill_(seq_mask.lt(1), -float('inf')) - output = nn.functional.softmax(output, dim=2) + if mask_out is not None: + output.masked_fill_(mask_out, -float('inf')) + output = self.softmax(output) + output = self.drop(output) return torch.matmul(output, V) class MultiHeadAtte(nn.Module): - def __init__(self, input_size, output_size, key_size, value_size, num_atte): + def __init__(self, model_size, key_size, value_size, num_head, dropout=0.1): super(MultiHeadAtte, self).__init__() - self.in_linear = nn.ModuleList() - for i in range(num_atte * 3): - out_feat = key_size if (i % 3) != 2 else value_size - self.in_linear.append(nn.Linear(input_size, out_feat)) - self.attes = nn.ModuleList([DotAtte(key_size, value_size) for _ in range(num_atte)]) - self.out_linear = nn.Linear(value_size * num_atte, output_size) - - def forward(self, Q, K, V, seq_mask=None): - heads = [] - for i in range(len(self.attes)): - j = i * 3 - qi, ki, vi = self.in_linear[j](Q), self.in_linear[j+1](K), self.in_linear[j+2](V) - headi = self.attes[i](qi, ki, vi, seq_mask) - heads.append(headi) - output = torch.cat(heads, dim=2) - return self.out_linear(output) + self.input_size = model_size + self.key_size = key_size + self.value_size = value_size + self.num_head = num_head + + in_size = key_size * num_head + self.q_in = nn.Linear(model_size, in_size) + self.k_in = nn.Linear(model_size, in_size) + self.v_in = nn.Linear(model_size, in_size) + self.attention = DotAtte(key_size=key_size, value_size=value_size) + self.out = nn.Linear(value_size * num_head, model_size) + self.drop = TimestepDropout(dropout) + self.reset_parameters() + + def reset_parameters(self): + sqrt = math.sqrt + nn.init.normal_(self.q_in.weight, mean=0, std=sqrt(2.0 / (self.input_size + self.key_size))) + nn.init.normal_(self.k_in.weight, mean=0, std=sqrt(2.0 / (self.input_size + self.key_size))) + nn.init.normal_(self.v_in.weight, mean=0, std=sqrt(2.0 / (self.input_size + self.value_size))) + nn.init.xavier_normal_(self.out.weight) + + def forward(self, Q, K, V, atte_mask_out=None): + """ + :param Q: [batch, seq_len, model_size] + :param K: [batch, seq_len, model_size] + :param V: [batch, seq_len, model_size] + :param seq_mask: [batch, seq_len] + """ + batch, seq_len, _ = Q.size() + d_k, d_v, n_head = self.key_size, self.value_size, self.num_head + # input linear + q = self.q_in(Q).view(batch, seq_len, n_head, d_k) + k = self.k_in(K).view(batch, seq_len, n_head, d_k) + v = self.v_in(V).view(batch, seq_len, n_head, d_k) + + # transpose q, k and v to do batch attention + q = q.permute(2, 0, 1, 3).contiguous().view(-1, seq_len, d_k) + k = k.permute(2, 0, 1, 3).contiguous().view(-1, seq_len, d_k) + v = v.permute(2, 0, 1, 3).contiguous().view(-1, seq_len, d_v) + if atte_mask_out is not None: + atte_mask_out = atte_mask_out.repeat(n_head, 1, 1) + atte = self.attention(q, k, v, atte_mask_out).view(n_head, batch, seq_len, d_v) + + # concat all heads, do output linear + atte = atte.permute(1, 2, 0, 3).contiguous().view(batch, seq_len, -1) + output = self.drop(self.out(atte)) + return output class Bi_Attention(nn.Module): def __init__(self): diff --git a/fastNLP/modules/encoder/transformer.py b/fastNLP/modules/encoder/transformer.py index 615a6f34..ef9efabe 100644 --- a/fastNLP/modules/encoder/transformer.py +++ b/fastNLP/modules/encoder/transformer.py @@ -1,29 +1,48 @@ +import torch from torch import nn from ..aggregator.attention import MultiHeadAtte -from ..other_modules import LayerNormalization +from ..dropout import TimestepDropout class TransformerEncoder(nn.Module): class SubLayer(nn.Module): - def __init__(self, input_size, output_size, key_size, value_size, num_atte): + def __init__(self, model_size, inner_size, key_size, value_size, num_head, dropout=0.1): super(TransformerEncoder.SubLayer, self).__init__() - self.atte = MultiHeadAtte(input_size, output_size, key_size, value_size, num_atte) - self.norm1 = LayerNormalization(output_size) - self.ffn = nn.Sequential(nn.Linear(output_size, output_size), + self.atte = MultiHeadAtte(model_size, key_size, value_size, num_head, dropout) + self.norm1 = nn.LayerNorm(model_size) + self.ffn = nn.Sequential(nn.Linear(model_size, inner_size), nn.ReLU(), - nn.Linear(output_size, output_size)) - self.norm2 = LayerNormalization(output_size) + nn.Linear(inner_size, model_size), + TimestepDropout(dropout),) + self.norm2 = nn.LayerNorm(model_size) - def forward(self, input, seq_mask): - attention = self.atte(input) + def forward(self, input, seq_mask=None, atte_mask_out=None): + """ + + :param input: [batch, seq_len, model_size] + :param seq_mask: [batch, seq_len] + :return: [batch, seq_len, model_size] + """ + attention = self.atte(input, input, input, atte_mask_out) norm_atte = self.norm1(attention + input) + attention *= seq_mask output = self.ffn(norm_atte) - return self.norm2(output + norm_atte) + output = self.norm2(output + norm_atte) + output *= seq_mask + return output def __init__(self, num_layers, **kargs): super(TransformerEncoder, self).__init__() - self.layers = nn.Sequential(*[self.SubLayer(**kargs) for _ in range(num_layers)]) + self.layers = nn.ModuleList([self.SubLayer(**kargs) for _ in range(num_layers)]) def forward(self, x, seq_mask=None): - return self.layers(x, seq_mask) + output = x + if seq_mask is None: + atte_mask_out = None + else: + atte_mask_out = (seq_mask < 1)[:,None,:] + seq_mask = seq_mask[:,:,None] + for layer in self.layers: + output = layer(output, seq_mask, atte_mask_out) + return output diff --git a/reproduction/Biaffine_parser/cfg.cfg b/reproduction/Biaffine_parser/cfg.cfg index 9b00c209..ad06598f 100644 --- a/reproduction/Biaffine_parser/cfg.cfg +++ b/reproduction/Biaffine_parser/cfg.cfg @@ -2,7 +2,8 @@ n_epochs = 40 batch_size = 32 use_cuda = true -validate_every = 500 +use_tqdm=true +validate_every = -1 use_golden_train=true [test] @@ -19,15 +20,13 @@ word_vocab_size = -1 word_emb_dim = 100 pos_vocab_size = -1 pos_emb_dim = 100 -word_hid_dim = 100 -pos_hid_dim = 100 rnn_layers = 3 -rnn_hidden_size = 400 +rnn_hidden_size = 256 arc_mlp_size = 500 label_mlp_size = 100 num_label = -1 -dropout = 0.33 -use_var_lstm=true +dropout = 0.3 +encoder="transformer" use_greedy_infer=false [optim] diff --git a/reproduction/Biaffine_parser/run.py b/reproduction/Biaffine_parser/run.py index 656da201..e4928c63 100644 --- a/reproduction/Biaffine_parser/run.py +++ b/reproduction/Biaffine_parser/run.py @@ -141,7 +141,7 @@ def update_v(vocab, data, field): model_args['num_label'] = len(tag_v) model = BiaffineParser(**model_args.data) -model.reset_parameters() +print(model) word_idxp = IndexerProcessor(word_v, 'words', 'word_seq') pos_idxp = IndexerProcessor(pos_v, 'pos', 'pos_seq') @@ -209,7 +209,8 @@ def save_pipe(path): pipe = Pipeline(processors=[num_p, word_idxp, pos_idxp, seq_p, set_input_p]) pipe.add_processor(ModelProcessor(model=model, batch_size=32)) pipe.add_processor(label_toword_p) - torch.save(pipe, os.path.join(path, 'pipe.pkl')) + os.makedirs(path, exist_ok=True) + torch.save({'pipeline': pipe}, os.path.join(path, 'pipe.pkl')) def test(path): diff --git a/test/models/test_biaffine_parser.py b/test/models/test_biaffine_parser.py index 54935f76..d87000a0 100644 --- a/test/models/test_biaffine_parser.py +++ b/test/models/test_biaffine_parser.py @@ -77,9 +77,10 @@ def test_train(self): ds, v1, v2, v3 = init_data() model = BiaffineParser(word_vocab_size=len(v1), word_emb_dim=30, pos_vocab_size=len(v2), pos_emb_dim=30, - num_label=len(v3), use_var_lstm=True) + num_label=len(v3), encoder='var-lstm') trainer = fastNLP.Trainer(model=model, train_data=ds, dev_data=ds, loss=ParserLoss(), metrics=ParserMetric(), metric_key='UAS', + batch_size=1, validate_every=10, n_epochs=10, use_cuda=False, use_tqdm=False) trainer.train(load_best_model=False) From a6dbbe9812f301f1e3dfcc02d984ee53dad0df5d Mon Sep 17 00:00:00 2001 From: ChenXin Date: Tue, 15 Jan 2019 11:45:02 +0800 Subject: [PATCH 02/32] remove the gpu_id info when saving --- fastNLP/core/trainer.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index 6dcc9c78..9cc5431c 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -365,12 +365,23 @@ def _compute_loss(self, predict, truth): return self.losser(predict, truth) def _save_model(self, model, model_name, only_param=False): + """ 存储不含有显卡信息的state_dict或model + :param model: + :param model_name: + :param only_param: + :return: + """ if self.save_path is not None: - model_name = os.path.join(self.save_path, model_name) + model_path = os.path.join(self.save_path, model_name) if only_param: - torch.save(model.state_dict(), model_name) + state_dict = model.state_dict() + for key in state_dict: + state_dict[key] = state_dict[key].cpu() + torch.save(state_dict, model_path) else: - torch.save(model, model_name) + model.cpu() + torch.save(model, model_path) + model.cuda() def _load_model(self, model, model_name, only_param=False): # 返回bool值指示是否成功reload模型 From c4ba75d160c508123ce536df98a5ccbea2ed5ad9 Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Tue, 15 Jan 2019 14:30:37 +0800 Subject: [PATCH 03/32] code optimization * move used readers from reproduction to io/dataset_loader.py (API shall not call anything from reproduction/) --- fastNLP/api/api.py | 14 +- fastNLP/api/processor.py | 100 +++++ fastNLP/io/dataset_loader.py | 373 ++++++++++++++++++ reproduction/Biaffine_parser/main.py | 2 +- reproduction/Biaffine_parser/run.py | 7 +- reproduction/Biaffine_parser/util.py | 51 --- .../chinese_word_segment/cws_io/cws_reader.py | 194 --------- .../process/cws_processor.py | 103 ----- reproduction/pos_tag_model/pos_reader.py | 126 +----- reproduction/pos_tag_model/train_pos_tag.py | 5 +- 10 files changed, 489 insertions(+), 486 deletions(-) diff --git a/fastNLP/api/api.py b/fastNLP/api/api.py index 8368dcc9..b9bc7b70 100644 --- a/fastNLP/api/api.py +++ b/fastNLP/api/api.py @@ -9,9 +9,7 @@ from fastNLP.api.utils import load_url from fastNLP.api.processor import ModelProcessor -from reproduction.chinese_word_segment.cws_io.cws_reader import ConllCWSReader -from reproduction.pos_tag_model.pos_reader import ZhConllPOSReader -from reproduction.Biaffine_parser.util import ConllxDataLoader, add_seg_tag +from fastNLP.io.dataset_loader import ConllCWSReader, ZhConllPOSReader, ConllxDataLoader, add_seg_tag from fastNLP.core.instance import Instance from fastNLP.api.pipeline import Pipeline from fastNLP.core.metrics import SpanFPreRecMetric @@ -31,6 +29,16 @@ def __init__(self): self._dict = None def predict(self, *args, **kwargs): + """Do prediction for the given input. + """ + raise NotImplementedError + + def test(self, file_path): + """Test performance over the given data set. + + :param str file_path: + :return: a dictionary of metric values + """ raise NotImplementedError def load(self, path, device): diff --git a/fastNLP/api/processor.py b/fastNLP/api/processor.py index 7354fe0f..6867dae8 100644 --- a/fastNLP/api/processor.py +++ b/fastNLP/api/processor.py @@ -322,3 +322,103 @@ def __init__(self, *fields, flag=True): def process(self, dataset): dataset.set_input(*self.fields, flag=self.flag) return dataset + + +class VocabIndexerProcessor(Processor): + """ + 根据DataSet创建Vocabulary，并将其用数字index。新生成的index的field会被放在new_added_filed_name, 如果没有提供 + new_added_field_name, 则覆盖原有的field_name. + + """ + + def __init__(self, field_name, new_added_filed_name=None, min_freq=1, max_size=None, + verbose=0, is_input=True): + """ + + :param field_name: 从哪个field_name创建词表，以及对哪个field_name进行index操作 + :param new_added_filed_name: index时，生成的index field的名称，如果不传入，则覆盖field_name. + :param min_freq: 创建的Vocabulary允许的单词最少出现次数. + :param max_size: 创建的Vocabulary允许的最大的单词数量 + :param verbose: 0, 不输出任何信息；1，输出信息 + :param bool is_input: + """ + super(VocabIndexerProcessor, self).__init__(field_name, new_added_filed_name) + self.min_freq = min_freq + self.max_size = max_size + + self.verbose = verbose + self.is_input = is_input + + def construct_vocab(self, *datasets): + """ + 使用传入的DataSet创建vocabulary + + :param datasets: DataSet类型的数据，用于构建vocabulary + :return: + """ + self.vocab = Vocabulary(min_freq=self.min_freq, max_size=self.max_size) + for dataset in datasets: + assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) + dataset.apply(lambda ins: self.vocab.update(ins[self.field_name])) + self.vocab.build_vocab() + if self.verbose: + print("Vocabulary Constructed, has {} items.".format(len(self.vocab))) + + def process(self, *datasets, only_index_dataset=None): + """ + 若还未建立Vocabulary，则使用dataset中的DataSet建立vocabulary；若已经有了vocabulary则使用已有的vocabulary。得到vocabulary + 后，则会index datasets与only_index_dataset。 + + :param datasets: DataSet类型的数据 + :param only_index_dataset: DataSet, or list of DataSet. 该参数中的内容只会被用于index，不会被用于生成vocabulary。 + :return: + """ + if len(datasets) == 0 and not hasattr(self, 'vocab'): + raise RuntimeError("You have to construct vocabulary first. Or you have to pass datasets to construct it.") + if not hasattr(self, 'vocab'): + self.construct_vocab(*datasets) + else: + if self.verbose: + print("Using constructed vocabulary with {} items.".format(len(self.vocab))) + to_index_datasets = [] + if len(datasets) != 0: + for dataset in datasets: + assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) + to_index_datasets.append(dataset) + + if not (only_index_dataset is None): + if isinstance(only_index_dataset, list): + for dataset in only_index_dataset: + assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) + to_index_datasets.append(dataset) + elif isinstance(only_index_dataset, DataSet): + to_index_datasets.append(only_index_dataset) + else: + raise TypeError('Only DataSet or list of DataSet is allowed, not {}.'.format(type(only_index_dataset))) + + for dataset in to_index_datasets: + assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) + dataset.apply(lambda ins: [self.vocab.to_index(token) for token in ins[self.field_name]], + new_field_name=self.new_added_field_name, is_input=self.is_input) + # 只返回一个，infer时为了跟其他processor保持一致 + if len(to_index_datasets) == 1: + return to_index_datasets[0] + + def set_vocab(self, vocab): + assert isinstance(vocab, Vocabulary), "Only fastNLP.core.Vocabulary is allowed, not {}.".format(type(vocab)) + self.vocab = vocab + + def delete_vocab(self): + del self.vocab + + def get_vocab_size(self): + return len(self.vocab) + + def set_verbose(self, verbose): + """ + 设置processor verbose状态。 + + :param verbose: int, 0，不输出任何信息；1，输出vocab 信息。 + :return: + """ + self.verbose = verbose diff --git a/fastNLP/io/dataset_loader.py b/fastNLP/io/dataset_loader.py index 27d8a360..2d157da3 100644 --- a/fastNLP/io/dataset_loader.py +++ b/fastNLP/io/dataset_loader.py @@ -90,6 +90,7 @@ class NativeDataSetLoader(DataSetLoader): """A simple example of DataSetLoader """ + def __init__(self): super(NativeDataSetLoader, self).__init__() @@ -107,6 +108,7 @@ class RawDataSetLoader(DataSetLoader): """A simple example of raw data reader """ + def __init__(self): super(RawDataSetLoader, self).__init__() @@ -142,6 +144,7 @@ class POSDataSetLoader(DataSetLoader): In this example, there are two sentences "Tom and Jerry ." and "Hello world !". Each word has its own label. """ + def __init__(self): super(POSDataSetLoader, self).__init__() @@ -540,3 +543,373 @@ def convert(self, data): data_set.set_input("premise", "hypothesis", "premise_len", "hypothesis_len") data_set.set_target("truth") return data_set + + +class ConllCWSReader(object): + def __init__(self): + pass + + def load(self, path, cut_long_sent=False): + """ + 返回的DataSet只包含raw_sentence这个field，内容为str。 + 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 + 1 编者按 编者按 NN O 11 nmod:topic + 2 ： ： PU O 11 punct + 3 7月 7月 NT DATE 4 compound:nn + 4 12日 12日 NT DATE 11 nmod:tmod + 5 ， ， PU O 11 punct + + 1 这 这 DT O 3 det + 2 款 款 M O 1 mark:clf + 3 飞行 飞行 NN O 8 nsubj + 4 从 从 P O 5 case + 5 外型 外型 NN O 8 nmod:prep + """ + datalist = [] + with open(path, 'r', encoding='utf-8') as f: + sample = [] + for line in f: + if line.startswith('\n'): + datalist.append(sample) + sample = [] + elif line.startswith('#'): + continue + else: + sample.append(line.split('\t')) + if len(sample) > 0: + datalist.append(sample) + + ds = DataSet() + for sample in datalist: + # print(sample) + res = self.get_char_lst(sample) + if res is None: + continue + line = ' '.join(res) + if cut_long_sent: + sents = cut_long_sentence(line) + else: + sents = [line] + for raw_sentence in sents: + ds.append(Instance(raw_sentence=raw_sentence)) + + return ds + + def get_char_lst(self, sample): + if len(sample) == 0: + return None + text = [] + for w in sample: + t1, t2, t3, t4 = w[1], w[3], w[6], w[7] + if t3 == '_': + return None + text.append(t1) + return text + + +class POSCWSReader(DataSetLoader): + """ + 支持读取以下的情况, 即每一行是一个词, 用空行作为两句话的界限. + 迈 N + 向 N + 充 N + ... + 泽 I-PER + 民 I-PER + + （ N + 一 N + 九 N + ... + + + :param filepath: + :return: + """ + + def __init__(self, in_word_splitter=None): + super().__init__() + self.in_word_splitter = in_word_splitter + + def load(self, filepath, in_word_splitter=None, cut_long_sent=False): + if in_word_splitter is None: + in_word_splitter = self.in_word_splitter + dataset = DataSet() + with open(filepath, 'r') as f: + words = [] + for line in f: + line = line.strip() + if len(line) == 0: # new line + if len(words) == 0: # 不能接受空行 + continue + line = ' '.join(words) + if cut_long_sent: + sents = cut_long_sentence(line) + else: + sents = [line] + for sent in sents: + instance = Instance(raw_sentence=sent) + dataset.append(instance) + words = [] + else: + line = line.split()[0] + if in_word_splitter is None: + words.append(line) + else: + words.append(line.split(in_word_splitter)[0]) + return dataset + + +class NaiveCWSReader(DataSetLoader): + """ + 这个reader假设了分词数据集为以下形式, 即已经用空格分割好内容了 + 这是 fastNLP , 一个 非常 good 的 包 . + 或者,即每个part后面还有一个pos tag + 也/D 在/P 團員/Na 之中/Ng ，/COMMACATEGORY + """ + + def __init__(self, in_word_splitter=None): + super().__init__() + + self.in_word_splitter = in_word_splitter + + def load(self, filepath, in_word_splitter=None, cut_long_sent=False): + """ + 允许使用的情况有(默认以\t或空格作为seg) + 这是 fastNLP , 一个 非常 good 的 包 . + 和 + 也/D 在/P 團員/Na 之中/Ng ，/COMMACATEGORY + 如果splitter不为None则认为是第二种情况, 且我们会按splitter分割"也/D", 然后取第一部分. 例如"也/D".split('/')[0] + :param filepath: + :param in_word_splitter: + :return: + """ + if in_word_splitter == None: + in_word_splitter = self.in_word_splitter + dataset = DataSet() + with open(filepath, 'r') as f: + for line in f: + line = line.strip() + if len(line.replace(' ', '')) == 0: # 不能接受空行 + continue + + if not in_word_splitter is None: + words = [] + for part in line.split(): + word = part.split(in_word_splitter)[0] + words.append(word) + line = ' '.join(words) + if cut_long_sent: + sents = cut_long_sentence(line) + else: + sents = [line] + for sent in sents: + instance = Instance(raw_sentence=sent) + dataset.append(instance) + + return dataset + + +def cut_long_sentence(sent, max_sample_length=200): + """ + 将长于max_sample_length的sentence截成多段，只会在有空格的地方发生截断。所以截取的句子可能长于或者短于max_sample_length + + :param sent: str. + :param max_sample_length: int. + :return: list of str. + """ + sent_no_space = sent.replace(' ', '') + cutted_sentence = [] + if len(sent_no_space) > max_sample_length: + parts = sent.strip().split() + new_line = '' + length = 0 + for part in parts: + length += len(part) + new_line += part + ' ' + if length > max_sample_length: + new_line = new_line[:-1] + cutted_sentence.append(new_line) + length = 0 + new_line = '' + if new_line != '': + cutted_sentence.append(new_line[:-1]) + else: + cutted_sentence.append(sent) + return cutted_sentence + + +class ZhConllPOSReader(object): + # 中文colln格式reader + def __init__(self): + pass + + def load(self, path): + """ + 返回的DataSet, 包含以下的field + words：list of str, + tag: list of str, 被加入了BMES tag, 比如原来的序列为['VP', 'NN', 'NN', ..]，会被认为是["S-VP", "B-NN", "M-NN",..] + 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 + 1 编者按 编者按 NN O 11 nmod:topic + 2 ： ： PU O 11 punct + 3 7月 7月 NT DATE 4 compound:nn + 4 12日 12日 NT DATE 11 nmod:tmod + 5 ， ， PU O 11 punct + + 1 这 这 DT O 3 det + 2 款 款 M O 1 mark:clf + 3 飞行 飞行 NN O 8 nsubj + 4 从 从 P O 5 case + 5 外型 外型 NN O 8 nmod:prep + """ + datalist = [] + with open(path, 'r', encoding='utf-8') as f: + sample = [] + for line in f: + if line.startswith('\n'): + datalist.append(sample) + sample = [] + elif line.startswith('#'): + continue + else: + sample.append(line.split('\t')) + if len(sample) > 0: + datalist.append(sample) + + ds = DataSet() + for sample in datalist: + # print(sample) + res = self.get_one(sample) + if res is None: + continue + char_seq = [] + pos_seq = [] + for word, tag in zip(res[0], res[1]): + char_seq.extend(list(word)) + if len(word) == 1: + pos_seq.append('S-{}'.format(tag)) + elif len(word) > 1: + pos_seq.append('B-{}'.format(tag)) + for _ in range(len(word) - 2): + pos_seq.append('M-{}'.format(tag)) + pos_seq.append('E-{}'.format(tag)) + else: + raise ValueError("Zero length of word detected.") + + ds.append(Instance(words=char_seq, + tag=pos_seq)) + + return ds + + def get_one(self, sample): + if len(sample) == 0: + return None + text = [] + pos_tags = [] + for w in sample: + t1, t2, t3, t4 = w[1], w[3], w[6], w[7] + if t3 == '_': + return None + text.append(t1) + pos_tags.append(t2) + return text, pos_tags + + +class ConllPOSReader(object): + # 返回的Dataset包含words(list of list, 里层的list是character), tag两个field(list of str, str是标有BIO的tag)。 + def __init__(self): + pass + + def load(self, path): + datalist = [] + with open(path, 'r', encoding='utf-8') as f: + sample = [] + for line in f: + if line.startswith('\n'): + datalist.append(sample) + sample = [] + elif line.startswith('#'): + continue + else: + sample.append(line.split('\t')) + if len(sample) > 0: + datalist.append(sample) + + ds = DataSet() + for sample in datalist: + # print(sample) + res = self.get_one(sample) + if res is None: + continue + char_seq = [] + pos_seq = [] + for word, tag in zip(res[0], res[1]): + if len(word) == 1: + char_seq.append(word) + pos_seq.append('S-{}'.format(tag)) + elif len(word) > 1: + pos_seq.append('B-{}'.format(tag)) + for _ in range(len(word) - 2): + pos_seq.append('M-{}'.format(tag)) + pos_seq.append('E-{}'.format(tag)) + char_seq.extend(list(word)) + else: + raise ValueError("Zero length of word detected.") + + ds.append(Instance(words=char_seq, + tag=pos_seq)) + + return ds + + +class ConllxDataLoader(object): + def load(self, path): + datalist = [] + with open(path, 'r', encoding='utf-8') as f: + sample = [] + for line in f: + if line.startswith('\n'): + datalist.append(sample) + sample = [] + elif line.startswith('#'): + continue + else: + sample.append(line.split('\t')) + if len(sample) > 0: + datalist.append(sample) + + data = [self.get_one(sample) for sample in datalist] + return list(filter(lambda x: x is not None, data)) + + def get_one(self, sample): + sample = list(map(list, zip(*sample))) + if len(sample) == 0: + return None + for w in sample[7]: + if w == '_': + print('Error Sample {}'.format(sample)) + return None + # return word_seq, pos_seq, head_seq, head_tag_seq + return sample[1], sample[3], list(map(int, sample[6])), sample[7] + + +def add_seg_tag(data): + """ + + :param data: list of ([word], [pos], [heads], [head_tags]) + :return: list of ([word], [pos]) + """ + + _processed = [] + for word_list, pos_list, _, _ in data: + new_sample = [] + for word, pos in zip(word_list, pos_list): + if len(word) == 1: + new_sample.append((word, 'S-' + pos)) + else: + new_sample.append((word[0], 'B-' + pos)) + for c in word[1:-1]: + new_sample.append((c, 'M-' + pos)) + new_sample.append((word[-1], 'E-' + pos)) + _processed.append(list(map(list, zip(*new_sample)))) + return _processed diff --git a/reproduction/Biaffine_parser/main.py b/reproduction/Biaffine_parser/main.py index 9028ff80..f4fd5836 100644 --- a/reproduction/Biaffine_parser/main.py +++ b/reproduction/Biaffine_parser/main.py @@ -5,7 +5,7 @@ import torch import argparse -from reproduction.Biaffine_parser.util import ConllxDataLoader, add_seg_tag +from fastNLP.io.dataset_loader import ConllxDataLoader, add_seg_tag from fastNLP.core.dataset import DataSet from fastNLP.core.instance import Instance diff --git a/reproduction/Biaffine_parser/run.py b/reproduction/Biaffine_parser/run.py index e4928c63..ded7487d 100644 --- a/reproduction/Biaffine_parser/run.py +++ b/reproduction/Biaffine_parser/run.py @@ -4,20 +4,15 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) import fastNLP -import torch from fastNLP.core.trainer import Trainer from fastNLP.core.instance import Instance from fastNLP.api.pipeline import Pipeline from fastNLP.models.biaffine_parser import BiaffineParser, ParserMetric, ParserLoss -from fastNLP.core.vocabulary import Vocabulary -from fastNLP.core.dataset import DataSet from fastNLP.core.tester import Tester from fastNLP.io.config_io import ConfigLoader, ConfigSection from fastNLP.io.model_io import ModelLoader -from fastNLP.io.embed_loader import EmbedLoader -from fastNLP.io.model_io import ModelSaver -from reproduction.Biaffine_parser.util import ConllxDataLoader, MyDataloader +from fastNLP.io.dataset_loader import ConllxDataLoader from fastNLP.api.processor import * BOS = '' diff --git a/reproduction/Biaffine_parser/util.py b/reproduction/Biaffine_parser/util.py index 793b1fb2..aa40e4e9 100644 --- a/reproduction/Biaffine_parser/util.py +++ b/reproduction/Biaffine_parser/util.py @@ -1,34 +1,3 @@ -class ConllxDataLoader(object): - def load(self, path): - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - sample = [] - for line in f: - if line.startswith('\n'): - datalist.append(sample) - sample = [] - elif line.startswith('#'): - continue - else: - sample.append(line.split('\t')) - if len(sample) > 0: - datalist.append(sample) - - data = [self.get_one(sample) for sample in datalist] - return list(filter(lambda x: x is not None, data)) - - def get_one(self, sample): - sample = list(map(list, zip(*sample))) - if len(sample) == 0: - return None - for w in sample[7]: - if w == '_': - print('Error Sample {}'.format(sample)) - return None - # return word_seq, pos_seq, head_seq, head_tag_seq - return sample[1], sample[3], list(map(int, sample[6])), sample[7] - - class MyDataloader: def load(self, data_path): with open(data_path, "r", encoding="utf-8") as f: @@ -56,23 +25,3 @@ def parse(self, lines): return data -def add_seg_tag(data): - """ - - :param data: list of ([word], [pos], [heads], [head_tags]) - :return: list of ([word], [pos]) - """ - - _processed = [] - for word_list, pos_list, _, _ in data: - new_sample = [] - for word, pos in zip(word_list, pos_list): - if len(word) == 1: - new_sample.append((word, 'S-' + pos)) - else: - new_sample.append((word[0], 'B-' + pos)) - for c in word[1:-1]: - new_sample.append((c, 'M-' + pos)) - new_sample.append((word[-1], 'E-' + pos)) - _processed.append(list(map(list, zip(*new_sample)))) - return _processed \ No newline at end of file diff --git a/reproduction/chinese_word_segment/cws_io/cws_reader.py b/reproduction/chinese_word_segment/cws_io/cws_reader.py index 34bcf7dd..b28b04f6 100644 --- a/reproduction/chinese_word_segment/cws_io/cws_reader.py +++ b/reproduction/chinese_word_segment/cws_io/cws_reader.py @@ -1,197 +1,3 @@ -from fastNLP.core.dataset import DataSet -from fastNLP.core.instance import Instance -from fastNLP.io.dataset_loader import DataSetLoader - - -def cut_long_sentence(sent, max_sample_length=200): - """ - 将长于max_sample_length的sentence截成多段，只会在有空格的地方发生截断。所以截取的句子可能长于或者短于max_sample_length - - :param sent: str. - :param max_sample_length: int. - :return: list of str. - """ - sent_no_space = sent.replace(' ', '') - cutted_sentence = [] - if len(sent_no_space) > max_sample_length: - parts = sent.strip().split() - new_line = '' - length = 0 - for part in parts: - length += len(part) - new_line += part + ' ' - if length > max_sample_length: - new_line = new_line[:-1] - cutted_sentence.append(new_line) - length = 0 - new_line = '' - if new_line != '': - cutted_sentence.append(new_line[:-1]) - else: - cutted_sentence.append(sent) - return cutted_sentence - -class NaiveCWSReader(DataSetLoader): - """ - 这个reader假设了分词数据集为以下形式, 即已经用空格分割好内容了 - 这是 fastNLP , 一个 非常 good 的 包 . - 或者,即每个part后面还有一个pos tag - 也/D 在/P 團員/Na 之中/Ng ，/COMMACATEGORY - """ - def __init__(self, in_word_splitter=None): - super().__init__() - - self.in_word_splitter = in_word_splitter - - def load(self, filepath, in_word_splitter=None, cut_long_sent=False): - """ - 允许使用的情况有(默认以\t或空格作为seg) - 这是 fastNLP , 一个 非常 good 的 包 . - 和 - 也/D 在/P 團員/Na 之中/Ng ，/COMMACATEGORY - 如果splitter不为None则认为是第二种情况, 且我们会按splitter分割"也/D", 然后取第一部分. 例如"也/D".split('/')[0] - :param filepath: - :param in_word_splitter: - :return: - """ - if in_word_splitter == None: - in_word_splitter = self.in_word_splitter - dataset = DataSet() - with open(filepath, 'r') as f: - for line in f: - line = line.strip() - if len(line.replace(' ', ''))==0: # 不能接受空行 - continue - - if not in_word_splitter is None: - words = [] - for part in line.split(): - word = part.split(in_word_splitter)[0] - words.append(word) - line = ' '.join(words) - if cut_long_sent: - sents = cut_long_sentence(line) - else: - sents = [line] - for sent in sents: - instance = Instance(raw_sentence=sent) - dataset.append(instance) - - return dataset - - -class POSCWSReader(DataSetLoader): - """ - 支持读取以下的情况, 即每一行是一个词, 用空行作为两句话的界限. - 迈 N - 向 N - 充 N - ... - 泽 I-PER - 民 I-PER - - （ N - 一 N - 九 N - ... - - - :param filepath: - :return: - """ - def __init__(self, in_word_splitter=None): - super().__init__() - self.in_word_splitter = in_word_splitter - - def load(self, filepath, in_word_splitter=None, cut_long_sent=False): - if in_word_splitter is None: - in_word_splitter = self.in_word_splitter - dataset = DataSet() - with open(filepath, 'r') as f: - words = [] - for line in f: - line = line.strip() - if len(line) == 0: # new line - if len(words)==0: # 不能接受空行 - continue - line = ' '.join(words) - if cut_long_sent: - sents = cut_long_sentence(line) - else: - sents = [line] - for sent in sents: - instance = Instance(raw_sentence=sent) - dataset.append(instance) - words = [] - else: - line = line.split()[0] - if in_word_splitter is None: - words.append(line) - else: - words.append(line.split(in_word_splitter)[0]) - return dataset - - -class ConllCWSReader(object): - def __init__(self): - pass - - def load(self, path, cut_long_sent=False): - """ - 返回的DataSet只包含raw_sentence这个field，内容为str。 - 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 - 1 编者按 编者按 NN O 11 nmod:topic - 2 ： ： PU O 11 punct - 3 7月 7月 NT DATE 4 compound:nn - 4 12日 12日 NT DATE 11 nmod:tmod - 5 ， ， PU O 11 punct - - 1 这 这 DT O 3 det - 2 款 款 M O 1 mark:clf - 3 飞行 飞行 NN O 8 nsubj - 4 从 从 P O 5 case - 5 外型 外型 NN O 8 nmod:prep - """ - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - sample = [] - for line in f: - if line.startswith('\n'): - datalist.append(sample) - sample = [] - elif line.startswith('#'): - continue - else: - sample.append(line.split('\t')) - if len(sample) > 0: - datalist.append(sample) - - ds = DataSet() - for sample in datalist: - # print(sample) - res = self.get_char_lst(sample) - if res is None: - continue - line = ' '.join(res) - if cut_long_sent: - sents = cut_long_sentence(line) - else: - sents = [line] - for raw_sentence in sents: - ds.append(Instance(raw_sentence=raw_sentence)) - - return ds - - def get_char_lst(self, sample): - if len(sample)==0: - return None - text = [] - for w in sample: - t1, t2, t3, t4 = w[1], w[3], w[6], w[7] - if t3 == '_': - return None - text.append(t1) - return text diff --git a/reproduction/chinese_word_segment/process/cws_processor.py b/reproduction/chinese_word_segment/process/cws_processor.py index 9e57d35a..be6ca6b1 100644 --- a/reproduction/chinese_word_segment/process/cws_processor.py +++ b/reproduction/chinese_word_segment/process/cws_processor.py @@ -226,109 +226,6 @@ def _generate_bigram(self, characters): return bigrams -# 这里需要建立vocabulary了，但是遇到了以下的问题 -# (1) 如果使用Processor的方式的话，但是在这种情况返回的不是dataset。所以建立vocabulary的工作用另外的方式实现，不借用 -# Processor了 -# TODO 如何将建立vocab和index这两步统一了？ - -class VocabIndexerProcessor(Processor): - """ - 根据DataSet创建Vocabulary，并将其用数字index。新生成的index的field会被放在new_added_filed_name, 如果没有提供 - new_added_field_name, 则覆盖原有的field_name. - - """ - def __init__(self, field_name, new_added_filed_name=None, min_freq=1, max_size=None, - verbose=0, is_input=True): - """ - - :param field_name: 从哪个field_name创建词表，以及对哪个field_name进行index操作 - :param new_added_filed_name: index时，生成的index field的名称，如果不传入，则覆盖field_name. - :param min_freq: 创建的Vocabulary允许的单词最少出现次数. - :param max_size: 创建的Vocabulary允许的最大的单词数量 - :param verbose: 0, 不输出任何信息；1，输出信息 - :param bool is_input: - """ - super(VocabIndexerProcessor, self).__init__(field_name, new_added_filed_name) - self.min_freq = min_freq - self.max_size = max_size - - self.verbose =verbose - self.is_input = is_input - - def construct_vocab(self, *datasets): - """ - 使用传入的DataSet创建vocabulary - - :param datasets: DataSet类型的数据，用于构建vocabulary - :return: - """ - self.vocab = Vocabulary(min_freq=self.min_freq, max_size=self.max_size) - for dataset in datasets: - assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - dataset.apply(lambda ins: self.vocab.update(ins[self.field_name])) - self.vocab.build_vocab() - if self.verbose: - print("Vocabulary Constructed, has {} items.".format(len(self.vocab))) - - def process(self, *datasets, only_index_dataset=None): - """ - 若还未建立Vocabulary，则使用dataset中的DataSet建立vocabulary；若已经有了vocabulary则使用已有的vocabulary。得到vocabulary - 后，则会index datasets与only_index_dataset。 - - :param datasets: DataSet类型的数据 - :param only_index_dataset: DataSet, or list of DataSet. 该参数中的内容只会被用于index，不会被用于生成vocabulary。 - :return: - """ - if len(datasets)==0 and not hasattr(self,'vocab'): - raise RuntimeError("You have to construct vocabulary first. Or you have to pass datasets to construct it.") - if not hasattr(self, 'vocab'): - self.construct_vocab(*datasets) - else: - if self.verbose: - print("Using constructed vocabulary with {} items.".format(len(self.vocab))) - to_index_datasets = [] - if len(datasets)!=0: - for dataset in datasets: - assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) - to_index_datasets.append(dataset) - - if not (only_index_dataset is None): - if isinstance(only_index_dataset, list): - for dataset in only_index_dataset: - assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) - to_index_datasets.append(dataset) - elif isinstance(only_index_dataset, DataSet): - to_index_datasets.append(only_index_dataset) - else: - raise TypeError('Only DataSet or list of DataSet is allowed, not {}.'.format(type(only_index_dataset))) - - for dataset in to_index_datasets: - assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) - dataset.apply(lambda ins: [self.vocab.to_index(token) for token in ins[self.field_name]], - new_field_name=self.new_added_field_name, is_input=self.is_input) - # 只返回一个，infer时为了跟其他processor保持一致 - if len(to_index_datasets) == 1: - return to_index_datasets[0] - - def set_vocab(self, vocab): - assert isinstance(vocab, Vocabulary), "Only fastNLP.core.Vocabulary is allowed, not {}.".format(type(vocab)) - self.vocab = vocab - - def delete_vocab(self): - del self.vocab - - def get_vocab_size(self): - return len(self.vocab) - - def set_verbose(self, verbose): - """ - 设置processor verbose状态。 - - :param verbose: int, 0，不输出任何信息；1，输出vocab 信息。 - :return: - """ - self.verbose = verbose - class VocabProcessor(Processor): def __init__(self, field_name, min_freq=1, max_size=None): diff --git a/reproduction/pos_tag_model/pos_reader.py b/reproduction/pos_tag_model/pos_reader.py index c0a8c4cd..4ff58f4b 100644 --- a/reproduction/pos_tag_model/pos_reader.py +++ b/reproduction/pos_tag_model/pos_reader.py @@ -1,6 +1,5 @@ +from fastNLP.io.dataset_loader import ZhConllPOSReader -from fastNLP.core.dataset import DataSet -from fastNLP.core.instance import Instance def cut_long_sentence(sent, max_sample_length=200): sent_no_space = sent.replace(' ', '') @@ -24,129 +23,6 @@ def cut_long_sentence(sent, max_sample_length=200): return cutted_sentence -class ConllPOSReader(object): - # 返回的Dataset包含words(list of list, 里层的list是character), tag两个field(list of str, str是标有BIO的tag)。 - def __init__(self): - pass - - def load(self, path): - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - sample = [] - for line in f: - if line.startswith('\n'): - datalist.append(sample) - sample = [] - elif line.startswith('#'): - continue - else: - sample.append(line.split('\t')) - if len(sample) > 0: - datalist.append(sample) - - ds = DataSet() - for sample in datalist: - # print(sample) - res = self.get_one(sample) - if res is None: - continue - char_seq = [] - pos_seq = [] - for word, tag in zip(res[0], res[1]): - if len(word)==1: - char_seq.append(word) - pos_seq.append('S-{}'.format(tag)) - elif len(word)>1: - pos_seq.append('B-{}'.format(tag)) - for _ in range(len(word)-2): - pos_seq.append('M-{}'.format(tag)) - pos_seq.append('E-{}'.format(tag)) - char_seq.extend(list(word)) - else: - raise ValueError("Zero length of word detected.") - - ds.append(Instance(words=char_seq, - tag=pos_seq)) - - return ds - - - -class ZhConllPOSReader(object): - # 中文colln格式reader - def __init__(self): - pass - - def load(self, path): - """ - 返回的DataSet, 包含以下的field - words：list of str, - tag: list of str, 被加入了BMES tag, 比如原来的序列为['VP', 'NN', 'NN', ..]，会被认为是["S-VP", "B-NN", "M-NN",..] - 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 - 1 编者按 编者按 NN O 11 nmod:topic - 2 ： ： PU O 11 punct - 3 7月 7月 NT DATE 4 compound:nn - 4 12日 12日 NT DATE 11 nmod:tmod - 5 ， ， PU O 11 punct - - 1 这 这 DT O 3 det - 2 款 款 M O 1 mark:clf - 3 飞行 飞行 NN O 8 nsubj - 4 从 从 P O 5 case - 5 外型 外型 NN O 8 nmod:prep - """ - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - sample = [] - for line in f: - if line.startswith('\n'): - datalist.append(sample) - sample = [] - elif line.startswith('#'): - continue - else: - sample.append(line.split('\t')) - if len(sample) > 0: - datalist.append(sample) - - ds = DataSet() - for sample in datalist: - # print(sample) - res = self.get_one(sample) - if res is None: - continue - char_seq = [] - pos_seq = [] - for word, tag in zip(res[0], res[1]): - char_seq.extend(list(word)) - if len(word)==1: - pos_seq.append('S-{}'.format(tag)) - elif len(word)>1: - pos_seq.append('B-{}'.format(tag)) - for _ in range(len(word)-2): - pos_seq.append('M-{}'.format(tag)) - pos_seq.append('E-{}'.format(tag)) - else: - raise ValueError("Zero length of word detected.") - - ds.append(Instance(words=char_seq, - tag=pos_seq)) - - return ds - - def get_one(self, sample): - if len(sample)==0: - return None - text = [] - pos_tags = [] - for w in sample: - t1, t2, t3, t4 = w[1], w[3], w[6], w[7] - if t3 == '_': - return None - text.append(t1) - pos_tags.append(t2) - return text, pos_tags - if __name__ == '__main__': reader = ZhConllPOSReader() d = reader.load('/home/hyan/train.conllx') diff --git a/reproduction/pos_tag_model/train_pos_tag.py b/reproduction/pos_tag_model/train_pos_tag.py index adc9359c..09a9ba02 100644 --- a/reproduction/pos_tag_model/train_pos_tag.py +++ b/reproduction/pos_tag_model/train_pos_tag.py @@ -10,13 +10,12 @@ from fastNLP.api.pipeline import Pipeline -from fastNLP.api.processor import SeqLenProcessor +from fastNLP.api.processor import SeqLenProcessor, VocabIndexerProcessor from fastNLP.core.metrics import SpanFPreRecMetric from fastNLP.core.trainer import Trainer from fastNLP.io.config_io import ConfigLoader, ConfigSection from fastNLP.models.sequence_modeling import AdvSeqLabel -from reproduction.chinese_word_segment.process.cws_processor import VocabIndexerProcessor -from reproduction.pos_tag_model.pos_reader import ZhConllPOSReader +from fastNLP.io.dataset_loader import ZhConllPOSReader from fastNLP.api.processor import ModelProcessor, Index2WordProcessor cfgfile = './pos_tag.cfg' From 1fdaf236d2c049270d0f3f47272e48abb0a73a7a Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Tue, 15 Jan 2019 14:56:01 +0800 Subject: [PATCH 04/32] =?UTF-8?q?Updates:=20*=20=E6=94=B9=E5=90=8D:=20chin?= =?UTF-8?q?ese=5Fword=5Fsegment=20--->=20Chinese=5Fword=5Fsegmentation=20*?= =?UTF-8?q?=20=E6=94=B9=E5=90=8D:=20pos=5Ftag=5Fmodel=20--->=20POS=5Ftaggi?= =?UTF-8?q?ng=20*=20=E6=B7=BB=E5=8A=A04=E4=B8=AA=E5=AF=B9Batch=E7=9A=84?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=20*=20=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E7=9A=84chinese=5Fword=5Fsegment/run.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cws.cfg | 0 .../cws_io/__init__.py | 0 .../cws_io/cws_reader.py | 0 .../models/__init__.py | 0 .../models/cws_model.py | 8 +- .../process/__init__.py | 0 .../process/cws_processor.py | 2 +- .../process/span_converter.py | 0 .../utils.py | 0 .../pos_processor.py | 0 .../pos_reader.py | 0 .../pos_tag.cfg | 0 .../train_pos_tag.py | 0 .../{pos_tag_model => POS_tagging}/utils.py | 0 reproduction/chinese_word_segment/run.py | 151 ------------------ test/core/test_batch.py | 45 ++++++ 16 files changed, 50 insertions(+), 156 deletions(-) rename reproduction/{chinese_word_segment => Chinese_word_segmentation}/cws.cfg (100%) rename reproduction/{chinese_word_segment => Chinese_word_segmentation}/cws_io/__init__.py (100%) rename reproduction/{chinese_word_segment => Chinese_word_segmentation}/cws_io/cws_reader.py (100%) rename reproduction/{chinese_word_segment => Chinese_word_segmentation}/models/__init__.py (100%) rename reproduction/{chinese_word_segment => Chinese_word_segmentation}/models/cws_model.py (98%) rename reproduction/{chinese_word_segment => Chinese_word_segmentation}/process/__init__.py (100%) rename reproduction/{chinese_word_segment => Chinese_word_segmentation}/process/cws_processor.py (99%) rename reproduction/{chinese_word_segment => Chinese_word_segmentation}/process/span_converter.py (100%) rename reproduction/{chinese_word_segment => Chinese_word_segmentation}/utils.py (100%) rename reproduction/{pos_tag_model => POS_tagging}/pos_processor.py (100%) rename reproduction/{pos_tag_model => POS_tagging}/pos_reader.py (100%) rename reproduction/{pos_tag_model => POS_tagging}/pos_tag.cfg (100%) rename reproduction/{pos_tag_model => POS_tagging}/train_pos_tag.py (100%) rename reproduction/{pos_tag_model => POS_tagging}/utils.py (100%) delete mode 100644 reproduction/chinese_word_segment/run.py diff --git a/reproduction/chinese_word_segment/cws.cfg b/reproduction/Chinese_word_segmentation/cws.cfg similarity index 100% rename from reproduction/chinese_word_segment/cws.cfg rename to reproduction/Chinese_word_segmentation/cws.cfg diff --git a/reproduction/chinese_word_segment/cws_io/__init__.py b/reproduction/Chinese_word_segmentation/cws_io/__init__.py similarity index 100% rename from reproduction/chinese_word_segment/cws_io/__init__.py rename to reproduction/Chinese_word_segmentation/cws_io/__init__.py diff --git a/reproduction/chinese_word_segment/cws_io/cws_reader.py b/reproduction/Chinese_word_segmentation/cws_io/cws_reader.py similarity index 100% rename from reproduction/chinese_word_segment/cws_io/cws_reader.py rename to reproduction/Chinese_word_segmentation/cws_io/cws_reader.py diff --git a/reproduction/chinese_word_segment/models/__init__.py b/reproduction/Chinese_word_segmentation/models/__init__.py similarity index 100% rename from reproduction/chinese_word_segment/models/__init__.py rename to reproduction/Chinese_word_segmentation/models/__init__.py diff --git a/reproduction/chinese_word_segment/models/cws_model.py b/reproduction/Chinese_word_segmentation/models/cws_model.py similarity index 98% rename from reproduction/chinese_word_segment/models/cws_model.py rename to reproduction/Chinese_word_segmentation/models/cws_model.py index c6cf6746..daefc380 100644 --- a/reproduction/chinese_word_segment/models/cws_model.py +++ b/reproduction/Chinese_word_segmentation/models/cws_model.py @@ -1,11 +1,11 @@ -from torch import nn import torch -import torch.nn.functional as F +from torch import nn -from fastNLP.modules.decoder.MLP import MLP from fastNLP.models.base_model import BaseModel -from reproduction.chinese_word_segment.utils import seq_lens_to_mask +from fastNLP.modules.decoder.MLP import MLP +from reproduction.Chinese_word_segmentation.utils import seq_lens_to_mask + class CWSBiLSTMEncoder(BaseModel): def __init__(self, vocab_num, embed_dim=100, bigram_vocab_num=None, bigram_embed_dim=100, num_bigram_per_char=None, diff --git a/reproduction/chinese_word_segment/process/__init__.py b/reproduction/Chinese_word_segmentation/process/__init__.py similarity index 100% rename from reproduction/chinese_word_segment/process/__init__.py rename to reproduction/Chinese_word_segmentation/process/__init__.py diff --git a/reproduction/chinese_word_segment/process/cws_processor.py b/reproduction/Chinese_word_segmentation/process/cws_processor.py similarity index 99% rename from reproduction/chinese_word_segment/process/cws_processor.py rename to reproduction/Chinese_word_segmentation/process/cws_processor.py index be6ca6b1..614d9ef5 100644 --- a/reproduction/chinese_word_segment/process/cws_processor.py +++ b/reproduction/Chinese_word_segmentation/process/cws_processor.py @@ -4,7 +4,7 @@ from fastNLP.api.processor import Processor from fastNLP.core.dataset import DataSet from fastNLP.core.vocabulary import Vocabulary -from reproduction.chinese_word_segment.process.span_converter import SpanConverter +from reproduction.Chinese_word_segmentation.process.span_converter import SpanConverter _SPECIAL_TAG_PATTERN = '<[a-zA-Z]+>' diff --git a/reproduction/chinese_word_segment/process/span_converter.py b/reproduction/Chinese_word_segmentation/process/span_converter.py similarity index 100% rename from reproduction/chinese_word_segment/process/span_converter.py rename to reproduction/Chinese_word_segmentation/process/span_converter.py diff --git a/reproduction/chinese_word_segment/utils.py b/reproduction/Chinese_word_segmentation/utils.py similarity index 100% rename from reproduction/chinese_word_segment/utils.py rename to reproduction/Chinese_word_segmentation/utils.py diff --git a/reproduction/pos_tag_model/pos_processor.py b/reproduction/POS_tagging/pos_processor.py similarity index 100% rename from reproduction/pos_tag_model/pos_processor.py rename to reproduction/POS_tagging/pos_processor.py diff --git a/reproduction/pos_tag_model/pos_reader.py b/reproduction/POS_tagging/pos_reader.py similarity index 100% rename from reproduction/pos_tag_model/pos_reader.py rename to reproduction/POS_tagging/pos_reader.py diff --git a/reproduction/pos_tag_model/pos_tag.cfg b/reproduction/POS_tagging/pos_tag.cfg similarity index 100% rename from reproduction/pos_tag_model/pos_tag.cfg rename to reproduction/POS_tagging/pos_tag.cfg diff --git a/reproduction/pos_tag_model/train_pos_tag.py b/reproduction/POS_tagging/train_pos_tag.py similarity index 100% rename from reproduction/pos_tag_model/train_pos_tag.py rename to reproduction/POS_tagging/train_pos_tag.py diff --git a/reproduction/pos_tag_model/utils.py b/reproduction/POS_tagging/utils.py similarity index 100% rename from reproduction/pos_tag_model/utils.py rename to reproduction/POS_tagging/utils.py diff --git a/reproduction/chinese_word_segment/run.py b/reproduction/chinese_word_segment/run.py deleted file mode 100644 index e7804bae..00000000 --- a/reproduction/chinese_word_segment/run.py +++ /dev/null @@ -1,151 +0,0 @@ -import os -import sys - -sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) - -from fastNLP.io.config_io import ConfigLoader, ConfigSection -from fastNLP.core.trainer import SeqLabelTrainer -from fastNLP.io.dataset_loader import BaseLoader, TokenizeDataSetLoader -from fastNLP.core.utils import load_pickle -from fastNLP.io.model_io import ModelLoader, ModelSaver -from fastNLP.core.tester import SeqLabelTester -from fastNLP.models.sequence_modeling import AdvSeqLabel -from fastNLP.core.predictor import SeqLabelInfer -from fastNLP.core.utils import save_pickle -from fastNLP.core.metrics import SeqLabelEvaluator - -# not in the file's dir -if len(os.path.dirname(__file__)) != 0: - os.chdir(os.path.dirname(__file__)) -datadir = "/home/zyfeng/data/" -cfgfile = './cws.cfg' - -cws_data_path = os.path.join(datadir, "pku_training.utf8") -pickle_path = "save" -data_infer_path = os.path.join(datadir, "infer.utf8") - - -def infer(): - # Config Loader - test_args = ConfigSection() - ConfigLoader().load_config(cfgfile, {"POS_test": test_args}) - - # fetch dictionary size and number of labels from pickle files - word2index = load_pickle(pickle_path, "word2id.pkl") - test_args["vocab_size"] = len(word2index) - index2label = load_pickle(pickle_path, "label2id.pkl") - test_args["num_classes"] = len(index2label) - - # Define the same model - model = AdvSeqLabel(test_args) - - try: - ModelLoader.load_pytorch(model, "./save/trained_model.pkl") - print('model loaded!') - except Exception as e: - print('cannot load model!') - raise - - # Data Loader - infer_data = SeqLabelDataSet(load_func=BaseLoader.load_lines) - infer_data.load(data_infer_path, vocabs={"word_vocab": word2index}, infer=True) - print('data loaded') - - # Inference interface - infer = SeqLabelInfer(pickle_path) - results = infer.predict(model, infer_data) - - print(results) - print("Inference finished!") - - -def train(): - # Config Loader - train_args = ConfigSection() - test_args = ConfigSection() - ConfigLoader().load_config(cfgfile, {"train": train_args, "test": test_args}) - - print("loading data set...") - data = SeqLabelDataSet(load_func=TokenizeDataSetLoader.load) - data.load(cws_data_path) - data_train, data_dev = data.split(ratio=0.3) - train_args["vocab_size"] = len(data.word_vocab) - train_args["num_classes"] = len(data.label_vocab) - print("vocab size={}, num_classes={}".format(len(data.word_vocab), len(data.label_vocab))) - - change_field_is_target(data_dev, "truth", True) - save_pickle(data_dev, "./save/", "data_dev.pkl") - save_pickle(data.word_vocab, "./save/", "word2id.pkl") - save_pickle(data.label_vocab, "./save/", "label2id.pkl") - - # Trainer - trainer = SeqLabelTrainer(epochs=train_args["epochs"], batch_size=train_args["batch_size"], - validate=train_args["validate"], - use_cuda=train_args["use_cuda"], pickle_path=train_args["pickle_path"], - save_best_dev=True, print_every_step=10, model_name="trained_model.pkl", - evaluator=SeqLabelEvaluator()) - - # Model - model = AdvSeqLabel(train_args) - try: - ModelLoader.load_pytorch(model, "./save/saved_model.pkl") - print('model parameter loaded!') - except Exception as e: - print("No saved model. Continue.") - pass - - # Start training - trainer.train(model, data_train, data_dev) - print("Training finished!") - - # Saver - saver = ModelSaver("./save/trained_model.pkl") - saver.save_pytorch(model) - print("Model saved!") - - -def predict(): - # Config Loader - test_args = ConfigSection() - ConfigLoader().load_config(cfgfile, {"POS_test": test_args}) - - # fetch dictionary size and number of labels from pickle files - word2index = load_pickle(pickle_path, "word2id.pkl") - test_args["vocab_size"] = len(word2index) - index2label = load_pickle(pickle_path, "label2id.pkl") - test_args["num_classes"] = len(index2label) - - # load dev data - dev_data = load_pickle(pickle_path, "data_dev.pkl") - - # Define the same model - model = AdvSeqLabel(test_args) - - # Dump trained parameters into the model - ModelLoader.load_pytorch(model, "./save/trained_model.pkl") - print("model loaded!") - - # Tester - test_args["evaluator"] = SeqLabelEvaluator() - tester = SeqLabelTester(**test_args.data) - - # Start testing - tester.test(model, dev_data) - - -if __name__ == "__main__": - - import argparse - - parser = argparse.ArgumentParser(description='Run a chinese word segmentation model') - parser.add_argument('--mode', help='set the model\'s model', choices=['train', 'test', 'infer']) - args = parser.parse_args() - if args.mode == 'train': - train() - elif args.mode == 'test': - predict() - elif args.mode == 'infer': - infer() - else: - print('no mode specified for model!') - parser.print_help() diff --git a/test/core/test_batch.py b/test/core/test_batch.py index 08d803f1..1c4b22f8 100644 --- a/test/core/test_batch.py +++ b/test/core/test_batch.py @@ -1,6 +1,7 @@ import unittest import numpy as np +import torch from fastNLP.core.batch import Batch from fastNLP.core.dataset import DataSet @@ -31,3 +32,47 @@ def test_dataset_batching(self): self.assertEqual(len(y["y"]), 4) self.assertListEqual(list(x["x"][-1]), [1, 2, 3, 4]) self.assertListEqual(list(y["y"][-1]), [5, 6]) + + def test_list_padding(self): + ds = DataSet({"x": [[1], [1, 2], [1, 2, 3], [1, 2, 3, 4]] * 10, + "y": [[4, 3, 2, 1], [3, 2, 1], [2, 1], [1]] * 10}) + ds.set_input("x") + ds.set_target("y") + iter = Batch(ds, batch_size=4, sampler=SequentialSampler(), as_numpy=True) + for x, y in iter: + self.assertEqual(x["x"].shape, (4, 4)) + self.assertEqual(y["y"].shape, (4, 4)) + + def test_numpy_padding(self): + ds = DataSet({"x": np.array([[1], [1, 2], [1, 2, 3], [1, 2, 3, 4]] * 10), + "y": np.array([[4, 3, 2, 1], [3, 2, 1], [2, 1], [1]] * 10)}) + ds.set_input("x") + ds.set_target("y") + iter = Batch(ds, batch_size=4, sampler=SequentialSampler(), as_numpy=True) + for x, y in iter: + self.assertEqual(x["x"].shape, (4, 4)) + self.assertEqual(y["y"].shape, (4, 4)) + + def test_list_to_tensor(self): + ds = DataSet({"x": [[1], [1, 2], [1, 2, 3], [1, 2, 3, 4]] * 10, + "y": [[4, 3, 2, 1], [3, 2, 1], [2, 1], [1]] * 10}) + ds.set_input("x") + ds.set_target("y") + iter = Batch(ds, batch_size=4, sampler=SequentialSampler(), as_numpy=False) + for x, y in iter: + self.assertTrue(isinstance(x["x"], torch.Tensor)) + self.assertEqual(tuple(x["x"].shape), (4, 4)) + self.assertTrue(isinstance(y["y"], torch.Tensor)) + self.assertEqual(tuple(y["y"].shape), (4, 4)) + + def test_numpy_to_tensor(self): + ds = DataSet({"x": np.array([[1], [1, 2], [1, 2, 3], [1, 2, 3, 4]] * 10), + "y": np.array([[4, 3, 2, 1], [3, 2, 1], [2, 1], [1]] * 10)}) + ds.set_input("x") + ds.set_target("y") + iter = Batch(ds, batch_size=4, sampler=SequentialSampler(), as_numpy=False) + for x, y in iter: + self.assertTrue(isinstance(x["x"], torch.Tensor)) + self.assertEqual(tuple(x["x"].shape), (4, 4)) + self.assertTrue(isinstance(y["y"], torch.Tensor)) + self.assertEqual(tuple(y["y"].shape), (4, 4)) From 6a0a1ed4ad8349258e46cf9a641ff70e55a2c19c Mon Sep 17 00:00:00 2001 From: yh Date: Tue, 15 Jan 2019 14:58:43 +0800 Subject: [PATCH 05/32] =?UTF-8?q?train=E5=A2=9E=E5=8A=A0=E6=B3=A8=E9=87=8A?= =?UTF-8?q?=EF=BC=9Battention=E5=A2=9E=E5=8A=A0=E6=B3=A8=E9=87=8A=EF=BC=9B?= =?UTF-8?q?=E6=96=B0=E5=A2=9Etransformer=E5=88=86=E8=AF=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/trainer.py | 4 +- fastNLP/modules/aggregator/attention.py | 15 ++++ .../models/cws_transformer.py | 70 +++++++++++++++++++ 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 reproduction/chinese_word_segment/models/cws_transformer.py diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index 109315a3..add86156 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -51,7 +51,9 @@ def __init__(self, train_data, model, loss=None, metrics=None, n_epochs=3, batch :param Optimizer optimizer: an optimizer object :param int check_code_level: level of FastNLP code checker. -1: don't check, 0: ignore. 1: warning. 2: strict.\\ `ignore` will not check unused field; `warning` when warn if some field are not used; `strict` means - it will raise error if some field are not used. + it will raise error if some field are not used. 检查的原理是通过使用很小的batch(默认两个sample)来检查代码是否能够 + 运行，但是这个过程理论上不会修改任何参数，只是会检查能否运行。但如果(1)模型中存在将batch_size写为某个固定值的情况，；(2) + 模型中存在累加前向计算次数的，可能会多计算几次。建议将check_code_level设置为-1 :param str metric_key: a single indicator used to decide the best model based on metric results. It must be one of the keys returned by the FIRST metric in `metrics`. If the overall result gets better if the indicator gets smaller, add "-" in front of the string. For example:: diff --git a/fastNLP/modules/aggregator/attention.py b/fastNLP/modules/aggregator/attention.py index 3fea1b10..9f7d72dc 100644 --- a/fastNLP/modules/aggregator/attention.py +++ b/fastNLP/modules/aggregator/attention.py @@ -46,6 +46,21 @@ def forward(self, Q, K, V, seq_mask=None): class MultiHeadAtte(nn.Module): def __init__(self, input_size, output_size, key_size, value_size, num_atte): + """ + 实现的是以下内容 + QW1: (batch_size, seq_len, input_size) * (input_size, key_size) + KW2: (batch_size, seq_len, input_size) * (input_size, key_size) + VW3: (batch_size, seq_len, input_size) * (input_size, value_size) + + softmax(QK^T/sqrt(scale))*V: (batch_size, seq_len, value_size) 多个head(num_atten指定)的结果为 + (batch_size, seq_len, value_size*num_atte) + 最终结果将上式过一个（value_size*num_atte, output_size)的线性层，output为(batch_size, seq_len, output_size) + :param input_size: int, 输入的维度 + :param output_size: int, 输出特征的维度 + :param key_size: int, query和key映射到该维度 + :param value_size: int, value映射到该维度 + :param num_atte: + """ super(MultiHeadAtte, self).__init__() self.in_linear = nn.ModuleList() for i in range(num_atte * 3): diff --git a/reproduction/chinese_word_segment/models/cws_transformer.py b/reproduction/chinese_word_segment/models/cws_transformer.py new file mode 100644 index 00000000..3fcf91b5 --- /dev/null +++ b/reproduction/chinese_word_segment/models/cws_transformer.py @@ -0,0 +1,70 @@ + + + +""" +使用transformer作为分词的encoder端 + +""" + +from torch import nn +import torch +from fastNLP.modules.encoder.transformer import TransformerEncoder +from fastNLP.modules.decoder.CRF import ConditionalRandomField,seq_len_to_byte_mask +from fastNLP.modules.decoder.CRF import allowed_transitions + +class TransformerCWS(nn.Module): + def __init__(self, vocab_num, embed_dim=100, bigram_vocab_num=None, bigram_embed_dim=100, num_bigram_per_char=None, + hidden_size=200, embed_drop_p=0.3, num_layers=1, num_heads=8, tag_size=4): + super().__init__() + + self.embedding = nn.Embedding(vocab_num, embed_dim) + input_size = embed_dim + if bigram_vocab_num: + self.bigram_embedding = nn.Embedding(bigram_vocab_num, bigram_embed_dim) + input_size += num_bigram_per_char*bigram_embed_dim + + self.drop = nn.Dropout(embed_drop_p, inplace=True) + + self.fc1 = nn.Linear(input_size, hidden_size) + + value_size = hidden_size//num_heads + self.transformer = TransformerEncoder(num_layers, input_size=input_size, output_size=hidden_size, + key_size=value_size, value_size=value_size, num_atte=num_heads) + + self.fc2 = nn.Linear(hidden_size, tag_size) + + allowed_trans = allowed_transitions({0:'b', 1:'m', 2:'e', 3:'s'}, encoding_type='bmes') + self.crf = ConditionalRandomField(num_tags=tag_size, include_start_end_trans=False, + allowed_transitions=allowed_trans) + + def forward(self, chars, target, seq_lens, bigrams=None): + seq_lens = seq_lens + masks = seq_len_to_byte_mask(seq_lens) + x = self.embedding(chars) + batch_size = x.size(0) + length = x.size(1) + if hasattr(self, 'bigram_embedding'): + bigrams = self.bigram_embedding(bigrams) # batch_size x seq_lens x per_char x embed_size + x = torch.cat([x, bigrams.view(batch_size, length, -1)], dim=-1) + self.drop(x) + x = self.fc1(x) + feats = self.transformer(x, masks) + feats = self.fc2(feats) + losses = self.crf(feats, target, masks.float()) + + pred_dict = {} + pred_dict['seq_lens'] = seq_lens + pred_dict['loss'] = torch.mean(losses) + + return pred_dict + + +if __name__ == '__main__': + transformer = TransformerCWS(10, embed_dim=100, bigram_vocab_num=10, bigram_embed_dim=100, num_bigram_per_char=8, + hidden_size=200, embed_drop_p=0.3, num_layers=1, num_heads=8, tag_size=4) + chars = torch.randint(10, size=(4, 7)).long() + bigrams = torch.randint(10, size=(4, 56)).long() + seq_lens = torch.ones(4).long()*7 + target = torch.randint(4, size=(4, 7)) + + print(transformer(chars, target, seq_lens, bigrams)) \ No newline at end of file From d80d944e4077f879aff18aaaaf104238c6253e6e Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Tue, 15 Jan 2019 17:08:53 +0800 Subject: [PATCH 06/32] =?UTF-8?q?*=20=E6=B7=BB=E5=8A=A0callbacks=EF=BC=9AE?= =?UTF-8?q?arlyStopCallback=20*=20=E5=B0=86dataset.py=E4=B8=AD=E7=9A=84ass?= =?UTF-8?q?ert=E6=94=B9=E4=B8=BAraise=20error=20*=20=E7=BB=99trainer?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0try-except,=E6=8D=95=E6=8D=89EarlyStopError?= =?UTF-8?q?=20*=20=E4=BC=98=E5=8C=96trainer=E4=BB=A3=E7=A0=81=20*=20?= =?UTF-8?q?=E7=BB=99callbacks=E6=B7=BB=E5=8A=A0=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/callback.py | 40 ++++++++++++++++++--- fastNLP/core/dataset.py | 5 ++- fastNLP/core/trainer.py | 29 ++++++++------- test/core/test_batch.py | 13 +++++++ test/core/test_callbacks.py | 70 +++++++++++++++++++++++++++---------- 5 files changed, 119 insertions(+), 38 deletions(-) diff --git a/fastNLP/core/callback.py b/fastNLP/core/callback.py index ce9627ea..e6760a28 100644 --- a/fastNLP/core/callback.py +++ b/fastNLP/core/callback.py @@ -69,16 +69,16 @@ def after_train(self, model): """ pass - def on_exception(self, exception, model, indices): + def on_exception(self, exception, model): """ 当训练过程出现异常，会触发该方法 :param exception: 某种类型的Exception，比如KeyboardInterrupt等 :param model: 传入Trainer的模型 - :param indices: 当前batch的index :return: """ pass + def transfer(func): """装饰器，将对CallbackManager的调用转发到各个Callback子类. @@ -206,10 +206,10 @@ def after_epoch(self, cur_epoch, n_epoch, optimizer): def after_train(self, model): print("after_train") + class GradientClipCallback(Callback): def __init__(self, parameters=None, clip_value=1, clip_type='norm'): - """ - 每次backward前，将parameter的gradient clip到某个范围。 + """每次backward前，将parameter的gradient clip到某个范围。 :param parameters: None, torch.Tensor或List[torch.Tensor], 一般通过model.parameters()获得。如果为None则默认对Trainer 的model中所有参数进行clip @@ -235,6 +235,38 @@ def after_backward(self, model): self.clip_fun(model.parameters(), self.clip_value) +class EarlyStopError(BaseException): + def __init__(self, msg): + super(EarlyStopError, self).__init__(msg) + + +class EarlyStopCallback(Callback): + def __init__(self, patience): + """ + + :param int patience: 停止之前等待的epoch数 + """ + super(EarlyStopCallback, self).__init__() + self.trainer = None # override by CallbackManager + self.patience = patience + self.wait = 0 + self.epoch = 0 + + def after_valid(self, eval_result, metric_key, optimizer): + self.epoch += 1 + if not self.trainer._better_eval_result(eval_result): + # current result is getting worse + if self.wait == self.patience: + raise EarlyStopError("Early stopping raised.") + else: + self.wait += 1 + else: + self.wait = 0 + + def on_exception(self, exception, model): + if isinstance(exception, EarlyStopError): + print("Early Stopping triggered in epoch {}!".format(self.epoch)) + if __name__ == "__main__": manager = CallbackManager(env={"n_epoch": 3}, callbacks=[DummyCallback(), DummyCallback()]) diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index 64aa2934..59bffad2 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -146,7 +146,10 @@ def append(self, ins): for name, field in ins.fields.items(): self.field_arrays[name] = FieldArray(name, [field]) else: - assert len(self.field_arrays) == len(ins.fields) + if len(self.field_arrays) != len(ins.fields): + raise ValueError( + "DataSet object has {} fields, but attempt to append an Instance object with {} fields." + .format(len(self.field_arrays), len(ins.fields))) for name, field in ins.fields.items(): assert name in self.field_arrays self.field_arrays[name].append(field) diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index ccb3d18e..fcafeb32 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -181,7 +181,6 @@ def train(self, load_best_model=True): if torch.cuda.is_available() and self.use_cuda: self.model = self.model.cuda() self._model_device = self.model.parameters().__next__().device - self._mode(self.model, is_test=False) self.start_time = str(datetime.now().strftime('%Y-%m-%d %H-%M-%S')) @@ -200,9 +199,12 @@ def pass_func(*args, **kwargs): path = os.path.join(self.save_path, 'tensorboard_logs_{}'.format(self.start_time)) self._summary_writer = SummaryWriter(path) - self.callback_manager.before_train() - self._train() - self.callback_manager.after_train(self.model) + try: + self.callback_manager.before_train() + self._train() + self.callback_manager.after_train(self.model) + except BaseException as e: + self.callback_manager.on_exception(e, self.model) if self.dev_data is not None: print("\nIn Epoch:{}/Step:{}, got best dev performance:".format(self.best_dev_epoch, self.best_dev_step) + @@ -231,10 +233,11 @@ def _train(self): inner_tqdm = tqdm self.step = 0 start = time.time() - data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False) - total_steps = data_iterator.num_batches * self.n_epochs + total_steps = (len(self.train_data) // self.batch_size + int( + len(self.train_data) % self.batch_size != 0)) * self.n_epochs with inner_tqdm(total=total_steps, postfix='loss:{0:<6.5f}', leave=False, dynamic_ncols=True) as pbar: avg_loss = 0 + data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False) for epoch in range(1, self.n_epochs+1): pbar.set_description_str(desc="Epoch {}/{}".format(epoch, self.n_epochs)) # early stopping @@ -291,17 +294,13 @@ def _train(self): self.tester._format_eval_results(eval_res) pbar.write(eval_str) - # if self.validate_every < 0 and self.dev_data: - # eval_res = self._do_validation(epoch=epoch, step=self.step) - # eval_str = "Epoch {}/{}. Step:{}/{}. ".format(epoch, self.n_epochs, self.step, total_steps) + \ - # self.tester._format_eval_results(eval_res) - # pbar.write(eval_str) - if epoch != self.n_epochs: - data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, - as_numpy=False) + # ================= mini-batch end ==================== # + # lr decay; early stopping self.callback_manager.after_epoch(epoch, self.n_epochs, self.optimizer) + # =============== epochs end =================== # pbar.close() + # ============ tqdm end ============== # def _do_validation(self, epoch, step): res = self.tester.test() @@ -314,7 +313,7 @@ def _do_validation(self, epoch, step): self._save_model(self.model, "best_" + "_".join([self.model.__class__.__name__, self.metric_key, self.start_time])) else: - self._best_model_states = {name:param.cpu().clone() for name, param in self.model.named_parameters()} + self._best_model_states = {name: param.cpu().clone() for name, param in self.model.named_parameters()} self.best_dev_perf = res self.best_dev_epoch = epoch self.best_dev_step = step diff --git a/test/core/test_batch.py b/test/core/test_batch.py index 1c4b22f8..77aebea5 100644 --- a/test/core/test_batch.py +++ b/test/core/test_batch.py @@ -6,6 +6,7 @@ from fastNLP.core.batch import Batch from fastNLP.core.dataset import DataSet from fastNLP.core.dataset import construct_dataset +from fastNLP.core.instance import Instance from fastNLP.core.sampler import SequentialSampler @@ -76,3 +77,15 @@ def test_numpy_to_tensor(self): self.assertEqual(tuple(x["x"].shape), (4, 4)) self.assertTrue(isinstance(y["y"], torch.Tensor)) self.assertEqual(tuple(y["y"].shape), (4, 4)) + + def test_list_of_list_to_tensor(self): + ds = DataSet([Instance(x=[1, 2], y=[3, 4]) for _ in range(2)] + + [Instance(x=[1, 2, 3, 4], y=[3, 4, 5, 6]) for _ in range(2)]) + ds.set_input("x") + ds.set_target("y") + iter = Batch(ds, batch_size=4, sampler=SequentialSampler(), as_numpy=False) + for x, y in iter: + self.assertTrue(isinstance(x["x"], torch.Tensor)) + self.assertEqual(tuple(x["x"].shape), (4, 4)) + self.assertTrue(isinstance(y["y"], torch.Tensor)) + self.assertEqual(tuple(y["y"].shape), (4, 4)) diff --git a/test/core/test_callbacks.py b/test/core/test_callbacks.py index 20822cde..e5c4dc6b 100644 --- a/test/core/test_callbacks.py +++ b/test/core/test_callbacks.py @@ -2,39 +2,43 @@ import numpy as np -from fastNLP.core.callback import EchoCallback +from fastNLP.core.callback import EchoCallback, EarlyStopCallback, GradientClipCallback from fastNLP.core.dataset import DataSet from fastNLP.core.instance import Instance from fastNLP.core.losses import BCELoss +from fastNLP.core.metrics import AccuracyMetric from fastNLP.core.optimizer import SGD from fastNLP.core.trainer import Trainer from fastNLP.models.base_model import NaiveClassifier -class TestCallback(unittest.TestCase): - def test_case(self): - def prepare_fake_dataset(): - mean = np.array([-3, -3]) - cov = np.array([[1, 0], [0, 1]]) - class_A = np.random.multivariate_normal(mean, cov, size=(1000,)) +def prepare_env(): + def prepare_fake_dataset(): + mean = np.array([-3, -3]) + cov = np.array([[1, 0], [0, 1]]) + class_A = np.random.multivariate_normal(mean, cov, size=(1000,)) - mean = np.array([3, 3]) - cov = np.array([[1, 0], [0, 1]]) - class_B = np.random.multivariate_normal(mean, cov, size=(1000,)) + mean = np.array([3, 3]) + cov = np.array([[1, 0], [0, 1]]) + class_B = np.random.multivariate_normal(mean, cov, size=(1000,)) - data_set = DataSet([Instance(x=[float(item[0]), float(item[1])], y=[0.0]) for item in class_A] + - [Instance(x=[float(item[0]), float(item[1])], y=[1.0]) for item in class_B]) - return data_set + data_set = DataSet([Instance(x=[float(item[0]), float(item[1])], y=[0.0]) for item in class_A] + + [Instance(x=[float(item[0]), float(item[1])], y=[1.0]) for item in class_B]) + return data_set - data_set = prepare_fake_dataset() - data_set.set_input("x") - data_set.set_target("y") + data_set = prepare_fake_dataset() + data_set.set_input("x") + data_set.set_target("y") + model = NaiveClassifier(2, 1) + return data_set, model - model = NaiveClassifier(2, 1) +class TestCallback(unittest.TestCase): + def test_echo_callback(self): + data_set, model = prepare_env() trainer = Trainer(data_set, model, loss=BCELoss(pred="predict", target="y"), - n_epochs=1, + n_epochs=2, batch_size=32, print_every=50, optimizer=SGD(lr=0.1), @@ -42,3 +46,33 @@ def prepare_fake_dataset(): use_tqdm=False, callbacks=[EchoCallback()]) trainer.train() + + def test_gradient_clip(self): + data_set, model = prepare_env() + trainer = Trainer(data_set, model, + loss=BCELoss(pred="predict", target="y"), + n_epochs=30, + batch_size=32, + print_every=50, + optimizer=SGD(lr=0.1), + check_code_level=2, + use_tqdm=False, + dev_data=data_set, + metrics=AccuracyMetric(pred="predict", target="y"), + callbacks=[GradientClipCallback(model.parameters(), clip_value=2)]) + trainer.train() + + def test_early_stop(self): + data_set, model = prepare_env() + trainer = Trainer(data_set, model, + loss=BCELoss(pred="predict", target="y"), + n_epochs=50, + batch_size=32, + print_every=50, + optimizer=SGD(lr=0.01), + check_code_level=2, + use_tqdm=False, + dev_data=data_set, + metrics=AccuracyMetric(pred="predict", target="y"), + callbacks=[EarlyStopCallback(5)]) + trainer.train() From 8091a734ee7c325cebab09792602fb3ddc373158 Mon Sep 17 00:00:00 2001 From: yh Date: Tue, 15 Jan 2019 22:21:55 +0800 Subject: [PATCH 07/32] =?UTF-8?q?1.=20=E5=B0=86pad=E7=9A=84=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=E4=BB=8EFieldArray=E4=B8=AD=E5=89=A5=E7=A6=BB?= =?UTF-8?q?=E5=87=BA=E6=9D=A5=EF=BC=8C=E4=BD=BF=E7=94=A8Padder=E5=AE=8C?= =?UTF-8?q?=E6=88=90=E5=90=84=E7=A7=8Dpadding=E6=93=8D=E4=BD=9C=E3=80=82?= =?UTF-8?q?=202.=20FieldArray=E9=BB=98=E8=AE=A4=E4=BD=BF=E7=94=A8AutoPadde?= =?UTF-8?q?r,=20AutoPadder=E7=9A=84=E8=A1=8C=E4=B8=BA=E4=B8=8E=E4=B9=8B?= =?UTF-8?q?=E5=89=8D=E4=B8=8D=E4=BD=BF=E7=94=A8padder=E6=98=AF=E4=B8=80?= =?UTF-8?q?=E8=87=B4=E7=9A=84=E7=9A=84=203.=20=E4=B8=BA=E4=BA=86=E8=A7=A3?= =?UTF-8?q?=E5=86=B3=E4=BA=8C=E7=BB=B4padding=E7=9A=84=E9=97=AE=E9=A2=98?= =?UTF-8?q?=EF=BC=8C=E5=BC=95=E5=85=A5=E4=BA=86EngChar2dPadder=E7=94=A8?= =?UTF-8?q?=E4=BA=8E=E5=AF=B9character=E8=BF=9B=E8=A1=8Cpadding=204.=20?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=80=E4=BB=BDpadding=E7=9A=84tutorial?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/batch.py | 13 +- fastNLP/core/dataset.py | 50 ++- fastNLP/core/fieldarray.py | 213 +++++++++- .../Chinese_word_segmentation/__init__.py | 0 .../models/cws_transformer.py | 58 ++- reproduction/__init__.py | 0 test/core/test_fieldarray.py | 61 +++ tutorials/fastNLP_padding_tutorial.ipynb | 370 ++++++++++++++++++ 8 files changed, 727 insertions(+), 38 deletions(-) create mode 100644 reproduction/Chinese_word_segmentation/__init__.py rename reproduction/{chinese_word_segment => Chinese_word_segmentation}/models/cws_transformer.py (61%) create mode 100644 reproduction/__init__.py create mode 100644 tutorials/fastNLP_padding_tutorial.ipynb diff --git a/fastNLP/core/batch.py b/fastNLP/core/batch.py index 9ba8dca8..d4fcbf23 100644 --- a/fastNLP/core/batch.py +++ b/fastNLP/core/batch.py @@ -48,7 +48,7 @@ def __next__(self): for field_name, field in self.dataset.get_all_fields().items(): if field.is_target or field.is_input: batch = field.get(indices) - if not self.as_numpy: + if not self.as_numpy and field.padder is not None: batch = to_tensor(batch, field.dtype) if field.is_target: batch_y[field_name] = batch @@ -67,8 +67,11 @@ def get_batch_indices(self): def to_tensor(batch, dtype): - if dtype in (int, np.int8, np.int16, np.int32, np.int64): - batch = torch.LongTensor(batch) - if dtype in (float, np.float32, np.float64): - batch = torch.FloatTensor(batch) + try: + if dtype in (int, np.int8, np.int16, np.int32, np.int64): + batch = torch.LongTensor(batch) + if dtype in (float, np.float32, np.float64): + batch = torch.FloatTensor(batch) + except: + pass return batch diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index 64aa2934..e2457e10 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -3,6 +3,7 @@ import numpy as np from fastNLP.core.fieldarray import FieldArray +from fastNLP.core.fieldarray import AutoPadder from fastNLP.core.instance import Instance from fastNLP.core.utils import get_func_signature from fastNLP.io.base_loader import DataLoaderRegister @@ -88,11 +89,8 @@ def __getitem__(self, idx): raise RuntimeError(f"Start index {idx.start} out of range 0-{len(self)-1}") data_set = DataSet() for field in self.field_arrays.values(): - data_set.add_field(name=field.name, - fields=field.content[idx], - padding_val=field.padding_val, - is_input=field.is_input, - is_target=field.is_target) + data_set.add_field(name=field.name, fields=field.content[idx], padder=field.padder, + is_input=field.is_input, is_target=field.is_target) return data_set else: raise KeyError("Unrecognized type {} for idx in __getitem__ method".format(type(idx))) @@ -151,12 +149,12 @@ def append(self, ins): assert name in self.field_arrays self.field_arrays[name].append(field) - def add_field(self, name, fields, padding_val=0, is_input=False, is_target=False): + def add_field(self, name, fields, padder=AutoPadder(pad_val=0), is_input=False, is_target=False): """Add a new field to the DataSet. :param str name: the name of the field. :param fields: a list of int, float, or other objects. - :param int padding_val: integer for padding. + :param int padder: PadBase对象，如何对该Field进行padding。大部分情况使用默认值即可 :param bool is_input: whether this field is model input. :param bool is_target: whether this field is label or target. """ @@ -164,8 +162,8 @@ def add_field(self, name, fields, padding_val=0, is_input=False, is_target=False if len(self) != len(fields): raise RuntimeError(f"The field to append must have the same size as dataset. " f"Dataset size {len(self)} != field size {len(fields)}") - self.field_arrays[name] = FieldArray(name, fields, padding_val=padding_val, is_target=is_target, - is_input=is_input) + self.field_arrays[name] = FieldArray(name, fields, is_target=is_target, is_input=is_input, + padder=padder) def delete_field(self, name): """Delete a field based on the field name. @@ -229,6 +227,25 @@ def set_input(self, *field_name, flag=True): else: raise KeyError("{} is not a valid field name.".format(name)) + def set_padder(self, field_name, padder): + """ + 为field_name设置padder + :param field_name: str, 设置field的padding方式为padder + :param padder: PadderBase类型或None. 设置为None即删除padder。即对该field不进行padding操作. + :return: + """ + self.field_arrays[field_name].set_padder(padder) + + def set_pad_val(self, field_name, pad_val): + """ + 为某个 + + :param field_name: str，修改该field的pad_val + :param pad_val: int，该field的padder会以pad_val作为padding index + :return: + """ + self.field_arrays[field_name].set_pad_val(pad_val) + def get_input_name(self): """Get all field names with `is_input` as True. @@ -270,12 +287,9 @@ def apply(self, func, new_field_name=None, **kwargs): extra_param['is_input'] = old_field.is_input if 'is_target' not in extra_param: extra_param['is_target'] = old_field.is_target - self.add_field(name=new_field_name, - fields=results, - padding_val=old_field.padding_val, - **extra_param) + self.add_field(name=new_field_name, fields=results) else: - self.add_field(name=new_field_name, fields=results, **extra_param) + self.add_field(name=new_field_name, fields=results) else: return results @@ -314,8 +328,16 @@ def split(self, dev_ratio): for field_name in self.field_arrays: train_set.field_arrays[field_name].is_input = self.field_arrays[field_name].is_input train_set.field_arrays[field_name].is_target = self.field_arrays[field_name].is_target + train_set.field_arrays[field_name].padder = self.field_arrays[field_name].padder + train_set.field_arrays[field_name].dtype = self.field_arrays[field_name].dtype + train_set.field_arrays[field_name].pytype = self.field_arrays[field_name].pytype + train_set.field_arrays[field_name].is_2d_list = self.field_arrays[field_name].is_2d_list dev_set.field_arrays[field_name].is_input = self.field_arrays[field_name].is_input dev_set.field_arrays[field_name].is_target = self.field_arrays[field_name].is_target + dev_set.field_arrays[field_name].padder = self.field_arrays[field_name].padder + dev_set.field_arrays[field_name].dtype = self.field_arrays[field_name].dtype + dev_set.field_arrays[field_name].pytype = self.field_arrays[field_name].pytype + dev_set.field_arrays[field_name].is_2d_list = self.field_arrays[field_name].is_2d_list return train_set, dev_set diff --git a/fastNLP/core/fieldarray.py b/fastNLP/core/fieldarray.py index c1a2db1c..afb81697 100644 --- a/fastNLP/core/fieldarray.py +++ b/fastNLP/core/fieldarray.py @@ -1,19 +1,105 @@ import numpy as np +class PadderBase: + """ + 所有padder都需要继承这个类，并覆盖__call__()方法。 + 用于对batch进行padding操作。传入的element是inplace的，即直接修改element可能导致数据变化，建议inplace修改之前deepcopy一份。 + """ + def __init__(self, pad_val=0, **kwargs): + self.pad_val = pad_val + + def set_pad_val(self, pad_val): + self.pad_val = pad_val + + def __call__(self, contents, field_name, field_ele_dtype): + """ + 传入的是List内容。假设有以下的DataSet。 + from fastNLP import DataSet + from fastNLP import Instance + dataset = DataSet() + dataset.append(Instance(word='this is a demo', length=4, + chars=[['t', 'h', 'i', 's'], ['i', 's'], ['a'], ['d', 'e', 'm', 'o']])) + dataset.append(Instance(word='another one', length=2, + chars=[['a', 'n', 'o', 't', 'h', 'e', 'r'], ['o', 'n', 'e']])) + # 如果batch_size=2, 下面只是用str的方式看起来更直观一点，但实际上可能word和chars在pad时都已经为index了。 + word这个field的pad_func会接收到的内容会是 + [ + 'this is a demo', + 'another one' + ] + length这个field的pad_func会接收到的内容会是 + [4, 2] + chars这个field的pad_func会接收到的内容会是 + [ + [['t', 'h', 'i', 's'], ['i', 's'], ['a'], ['d', 'e', 'm', 'o']], + [['a', 'n', 'o', 't', 'h', 'e', 'r'], ['o', 'n', 'e']] + ] + 即把每个instance中某个field的内容合成一个List传入 + :param contents: List[element]。传入的element是inplace的，即直接修改element可能导致数据变化，建议inplace修改之前 + deepcopy一份。 + :param field_name: str, field的名称，帮助定位错误 + :param field_ele_dtype: np.int64, np.float64, np.str. 该field的内层list元素的类型。辅助判断是否pad，大多数情况用不上 + :return: List[padded_element]或np.array([padded_element]) + """ + raise NotImplementedError + + +class AutoPadder(PadderBase): + """ + 根据contents的数据自动判定是否需要做padding。 + (1) 如果元素类型(元素类型是指field中最里层List的元素的数据类型, 可以通过FieldArray.dtype查看，比如['This', 'is', ...]的元素类 + 型为np.str, [[1,2], ...]的元素类型为np.int64)的数据不为(np.int64, np.float64)则不会进行padding + (2) 如果元素类型为(np.int64, np.float64), + (2.1) 如果该field的内容只有一个，比如为sequence_length, 则不进行padding + (2.2) 如果该field的内容为List, 那么会将Batch中的List pad为一样长。若该List下还有里层的List需要padding，请使用其它padder。 + 如果某个instance中field为[1, 2, 3]，则可以pad； 若为[[1,2], [3,4, ...]]则不能进行pad + """ + def __init__(self, pad_val=0): + """ + :param pad_val: int, padding的位置使用该index + """ + super().__init__(pad_val=pad_val) + + def _is_two_dimension(self, contents): + """ + 判断contents是不是只有两个维度。[[1,2], [3]]是两个维度. [[[1,2], [3, 4, 5]], [[4,5]]]有三个维度 + :param contents: + :return: + """ + value = contents[0] + if isinstance(value , (np.ndarray, list)): + value = value[0] + if isinstance(value, (np.ndarray, list)): + return False + return True + return False + + def __call__(self, contents, field_name, field_ele_dtype): + if not is_iterable(contents[0]): + array = np.array([content for content in contents], dtype=field_ele_dtype) + elif field_ele_dtype in (np.int64, np.float64) and self._is_two_dimension(contents): + max_len = max([len(content) for content in contents]) + array = np.full((len(contents), max_len), self.pad_val, dtype=field_ele_dtype) + for i, content in enumerate(contents): + array[i][:len(content)] = content + else: # should only be str + array = np.array([content for content in contents]) + return array + + class FieldArray(object): """``FieldArray`` is the collection of ``Instance``s of the same field. It is the basic element of ``DataSet`` class. :param str name: the name of the FieldArray :param list content: a list of int, float, str or np.ndarray, or a list of list of one, or a np.ndarray. - :param int padding_val: the integer for padding. Default: 0. :param bool is_target: If True, this FieldArray is used to compute loss. :param bool is_input: If True, this FieldArray is used to the model input. - + :param padder: PadderBase类型。大多数情况下都不需要设置该值，除非需要在多个维度上进行padding(比如英文中对character进行padding) """ - def __init__(self, name, content, padding_val=0, is_target=None, is_input=None): + def __init__(self, name, content, is_target=None, is_input=None, padder=AutoPadder(pad_val=0)): self.name = name if isinstance(content, list): content = content @@ -22,7 +108,7 @@ def __init__(self, name, content, padding_val=0, is_target=None, is_input=None): else: raise TypeError("content in FieldArray can only be list or numpy.ndarray, got {}.".format(type(content))) self.content = content - self.padding_val = padding_val + self.set_padder(padder) self._is_target = None self._is_input = None @@ -149,28 +235,44 @@ def __setitem__(self, idx, val): assert isinstance(idx, int) self.content[idx] = val - def get(self, indices): + def get(self, indices, pad=True): """Fetch instances based on indices. :param indices: an int, or a list of int. + :param pad: bool, 是否对返回的结果进行padding。 :return: """ if isinstance(indices, int): return self.content[indices] if self.is_input is False and self.is_target is False: raise RuntimeError("Please specify either is_input or is_target is True for {}".format(self.name)) - batch_size = len(indices) - - if not is_iterable(self.content[0]): - array = np.array([self.content[i] for i in indices], dtype=self.dtype) - elif self.dtype in (np.int64, np.float64): - max_len = max([len(self.content[i]) for i in indices]) - array = np.full((batch_size, max_len), self.padding_val, dtype=self.dtype) - for i, idx in enumerate(indices): - array[i][:len(self.content[idx])] = self.content[idx] - else: # should only be str - array = np.array([self.content[i] for i in indices]) - return array + + contents = [self.content[i] for i in indices] + if self.padder is None or pad is False: + return np.array(contents) + else: + return self.padder(contents, field_name=self.name, field_ele_dtype=self.dtype) + + def set_padder(self, padder): + """ + 设置padding方式 + + :param padder: PadderBase类型或None. 设置为None即删除padder. + :return: + """ + if padder is not None: + assert isinstance(padder, PadderBase), "padder must be of type PadderBase." + self.padder = padder + + def set_pad_val(self, pad_val): + """ + 修改padder的pad_val. + :param pad_val: int。 + :return: + """ + if self.padder is not None: + self.padder.set_pad_val(pad_val) + def __len__(self): """Returns the size of FieldArray. @@ -186,3 +288,80 @@ def is_iterable(content): except TypeError: return False return True + + +class EngChar2DPadder(PadderBase): + """ + 用于为英语执行character级别的2D padding操作。对应的field内容应该为[['T', 'h', 'i', 's'], ['a'], ['d', 'e', 'm', 'o']](这里为 + 了更直观，把它们写为str，但实际使用时它们应该是character的index)。 + padded过后的batch内容，形状为(batch_size, max_sentence_length, max_word_length). max_sentence_length最大句子长度。 + max_word_length最长的word的长度 + + """ + def __init__(self, pad_val=0, pad_length=0): + """ + :param pad_val: int, padding的位置使用该index + :param pad_length: int, 如果为0则取一个batch中最大的单词长度作为padding长度。如果为大于0的数，则将所有单词的长度都pad或截 + 取到该长度. + """ + super().__init__(pad_val=pad_val) + + self.pad_length = pad_length + + def _exactly_three_dims(self, contents, field_name): + """ + 检查传入的contents是否刚好是3维，如果不是3维就报错。理论上，第一个维度是batch，第二个维度是word，第三个维度是character + :param contents: + :param field_name: str + :return: + """ + if not isinstance(contents, list): + raise TypeError("contents should be a list, not {}.".format(type(contents))) + value = contents[0] + try: + value = value[0] + except: + raise ValueError("Field:{} only has one dimension.".format(field_name)) + try: + value = value[1] + except: + raise ValueError("Field:{} only has two dimensions.".format(field_name)) + + if is_iterable(value): + raise ValueError("Field:{} has more than 3 dimension.".format(field_name)) + + def __call__(self, contents, field_name, field_ele_dtype): + """ + 期望输入类似于 + [ + [[0, 2], [2, 3, 4], ..], + [[9, 8, 2, 4], [1, 2,], ...], + .... + ] + + :param contents: + :param field_name: + :param field_ele_dtype + :return: + """ + if field_ele_dtype not in (np.int64, np.float64): + raise TypeError('dtype of Field:{} should be np.int64 or np.float64 to do 2D padding, get {}.'.format( + field_name, field_ele_dtype + )) + self._exactly_three_dims(contents, field_name) + if self.pad_length < 1: + max_char_length = max(max([[len(char_lst) for char_lst in word_lst] for word_lst in contents])) + else: + max_char_length = self.pad_length + max_sent_length = max(len(word_lst) for word_lst in contents) + batch_size = len(contents) + dtype = type(contents[0][0][0]) + + padded_array = np.full((batch_size, max_sent_length, max_char_length), fill_value=self.pad_val, + dtype=dtype) + for b_idx, word_lst in enumerate(contents): + for c_idx, char_lst in enumerate(word_lst): + chars = char_lst[:max_char_length] + padded_array[b_idx, c_idx, :len(chars)] = chars + + return padded_array \ No newline at end of file diff --git a/reproduction/Chinese_word_segmentation/__init__.py b/reproduction/Chinese_word_segmentation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/reproduction/chinese_word_segment/models/cws_transformer.py b/reproduction/Chinese_word_segmentation/models/cws_transformer.py similarity index 61% rename from reproduction/chinese_word_segment/models/cws_transformer.py rename to reproduction/Chinese_word_segmentation/models/cws_transformer.py index 64f9b09f..736edade 100644 --- a/reproduction/chinese_word_segment/models/cws_transformer.py +++ b/reproduction/Chinese_word_segmentation/models/cws_transformer.py @@ -39,7 +39,6 @@ def __init__(self, vocab_num, embed_dim=100, bigram_vocab_num=None, bigram_embed allowed_transitions=allowed_trans) def forward(self, chars, target, seq_lens, bigrams=None): - seq_lens = seq_lens masks = seq_len_to_byte_mask(seq_lens).float() x = self.embedding(chars) batch_size = x.size(0) @@ -59,8 +58,59 @@ def forward(self, chars, target, seq_lens, bigrams=None): return pred_dict + def predict(self, chars, seq_lens, bigrams=None): + masks = seq_len_to_byte_mask(seq_lens).float() + + x = self.embedding(chars) + batch_size = x.size(0) + length = x.size(1) + if hasattr(self, 'bigram_embedding'): + bigrams = self.bigram_embedding(bigrams) # batch_size x seq_lens x per_char x embed_size + x = torch.cat([x, bigrams.view(batch_size, length, -1)], dim=-1) + self.drop(x) + x = self.fc1(x) + feats = self.transformer(x, masks) + feats = self.fc2(feats) + + probs = self.crf.viterbi_decode(feats, masks, get_score=False) + + return {'pred': probs, 'seq_lens':seq_lens} + + +class NoamOpt(torch.optim.Optimizer): + "Optim wrapper that implements rate." + + def __init__(self, model_size, factor, warmup, optimizer): + super().__init__([torch.nn.Parameter(torch.ones(1))], {}) + + self.optimizer = optimizer + self._step = 0 + self.warmup = warmup + self.factor = factor + self.model_size = model_size + self._rate = 0 + + def step(self, **kwargs): + "Update parameters and rate" + self._step += 1 + rate = self.rate() + for p in self.optimizer.param_groups: + p['lr'] = rate + self._rate = rate + self.optimizer.step() + + def rate(self, step=None): + "Implement `lrate` above" + if step is None: + step = self._step + return self.factor * \ + (self.model_size ** (-0.5) * + min(step ** (-0.5), step * self.warmup ** (-1.5))) + if __name__ == '__main__': + + transformer = TransformerCWS(10, embed_dim=100, bigram_vocab_num=10, bigram_embed_dim=100, num_bigram_per_char=8, hidden_size=200, embed_drop_p=0.3, num_layers=1, num_heads=8, tag_size=4) chars = torch.randint(10, size=(4, 7)).long() @@ -68,4 +118,8 @@ def forward(self, chars, target, seq_lens, bigrams=None): seq_lens = torch.ones(4).long()*7 target = torch.randint(4, size=(4, 7)) - print(transformer(chars, target, seq_lens, bigrams)) \ No newline at end of file + print(transformer(chars, target, seq_lens, bigrams)) + + optimizer = torch.optim.Adam(transformer.parameters()) + + opt = NoamOpt(10 ,1, 400, optimizer) \ No newline at end of file diff --git a/reproduction/__init__.py b/reproduction/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/core/test_fieldarray.py b/test/core/test_fieldarray.py index 1204cda5..8b9e8754 100644 --- a/test/core/test_fieldarray.py +++ b/test/core/test_fieldarray.py @@ -97,3 +97,64 @@ def test_append(self): fa.append([1.2, 2.3, 3.4, 4.5, 5.6]) self.assertEqual(len(fa), 3) self.assertEqual(fa[2], [1.2, 2.3, 3.4, 4.5, 5.6]) + + +class TestPadder(unittest.TestCase): + + def test01(self): + """ + 测试AutoPadder能否正常工作 + :return: + """ + from fastNLP.core.fieldarray import AutoPadder + padder = AutoPadder() + content = ['This is a str', 'this is another str'] + self.assertListEqual(content, padder(content, None, np.str).tolist()) + + content = [1, 2] + self.assertListEqual(content, padder(content, None, np.int64).tolist()) + + content = [[1,2], [3], [4]] + self.assertListEqual([[1,2], [3, 0], [4, 0]], + padder(content, None, np.int64).tolist()) + + contents = [ + [[1, 2, 3], [4, 5], [7,8,9,10]], + [[1]] + ] + print(padder(contents, None, np.int64)) + + def test02(self): + """ + 测试EngChar2DPadder能不能正确使用 + :return: + """ + from fastNLP.core.fieldarray import EngChar2DPadder + padder = EngChar2DPadder(pad_length=0) + + contents = [1, 2] + # 不能是1维 + with self.assertRaises(ValueError): + padder(contents, None, np.int64) + contents = [[1, 2]] + # 不能是2维 + with self.assertRaises(ValueError): + padder(contents, None, np.int64) + contents = [[[[1, 2]]]] + # 不能是3维以上 + with self.assertRaises(ValueError): + padder(contents, None, np.int64) + + contents = [ + [[1, 2, 3], [4, 5], [7,8,9,10]], + [[1]] + ] + self.assertListEqual([[[1, 2, 3, 0], [4, 5, 0, 0], [7, 8, 9, 10]], [[1, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]], + padder(contents, None, np.int64).tolist()) + + padder = EngChar2DPadder(pad_length=5, pad_val=-100) + self.assertListEqual( + [[[1, 2, 3, -100, -100], [4, 5, -100, -100, -100], [7, 8, 9, 10, -100]], + [[1, -100, -100, -100, -100], [-100, -100, -100, -100, -100], [-100, -100, -100, -100, -100]]], + padder(contents, None, np.int64).tolist() + ) \ No newline at end of file diff --git a/tutorials/fastNLP_padding_tutorial.ipynb b/tutorials/fastNLP_padding_tutorial.ipynb new file mode 100644 index 00000000..7dc50206 --- /dev/null +++ b/tutorials/fastNLP_padding_tutorial.ipynb @@ -0,0 +1,370 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/yh/miniconda2/envs/python3/lib/python3.6/site-packages/tqdm/autonotebook/__init__.py:14: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", + " \" (e.g. in jupyter console)\", TqdmExperimentalWarning)\n" + ] + }, + { + "data": { + "text/plain": [ + "DataSet({'raw_sent': this is a bad idea . type=str,\n", + "'label': 0 type=int,\n", + "'word_str_lst': ['this', 'is', 'a', 'bad', 'idea', '.'] type=list,\n", + "'words': [4, 2, 5, 6, 7, 3] type=list},\n", + "{'raw_sent': it is great . type=str,\n", + "'label': 1 type=int,\n", + "'word_str_lst': ['it', 'is', 'great', '.'] type=list,\n", + "'words': [8, 2, 9, 3] type=list})" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 假设有以下的DataSet, 这里只是为了举例所以只选择了两个sample\n", + "import sys\n", + "import os\n", + "sys.path.append('/Users/yh/Desktop/fastNLP/fastNLP')\n", + "\n", + "from fastNLP import DataSet\n", + "from fastNLP import Instance\n", + "from fastNLP import Vocabulary\n", + "\n", + "dataset = DataSet()\n", + "dataset.append(Instance(raw_sent='This is a bad idea .', label=0))\n", + "dataset.append(Instance(raw_sent='It is great .', label=1))\n", + "\n", + "# 按照fastNLP_10min_tutorial.ipynb的步骤，对数据进行一些处理。这里为了演示padding操作，把field的名称做了一些改变\n", + "dataset.apply(lambda x:x['raw_sent'].lower(), new_field_name='raw_sent')\n", + "dataset.apply(lambda x:x['raw_sent'].split(), new_field_name='word_str_lst')\n", + "\n", + "# 建立Vocabulary\n", + "word_vocab = Vocabulary()\n", + "dataset.apply(lambda x:word_vocab.update(x['word_str_lst']))\n", + "dataset.apply(lambda x:[word_vocab.to_index(word) for word in x['word_str_lst']], new_field_name='words')\n", + "\n", + "# 检查以下是否得到我们想要的结果了\n", + "dataset[:2]" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "batch_x has: {'word_str_lst': array([list(['this', 'is', 'a', 'bad', 'idea', '.']),\n", + " list(['it', 'is', 'great', '.'])], dtype=object), 'words': tensor([[4, 2, 5, 6, 7, 3],\n", + " [8, 2, 9, 3, 0, 0]])}\n", + "batch_y has: {'label': tensor([0, 1])}\n" + ] + }, + { + "data": { + "text/plain": [ + "'\"\\n结果中\\n Batch会对元素类型(元素即最内层的数据，raw_sent为str，word_str_lst为str，words为int, label为int)为int或者float的数据进行默认\\n padding，而非int或float的则不进行padding。但若每个Instance中该field为二维数据，也不进行padding。因为二维数据的padding涉及到\\n 两个维度的padding，不容易自动判断padding的形式。\\n'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 将field设置为input或者target\n", + "dataset.set_input('word_str_lst')\n", + "dataset.set_input('words')\n", + "dataset.set_target('label')\n", + "\n", + "# 使用Batch取出batch数据\n", + "from fastNLP.core.batch import Batch\n", + "from fastNLP.core.sampler import RandomSampler\n", + "\n", + "batch_iterator = Batch(dataset=dataset, batch_size=2, sampler=RandomSampler())\n", + "for batch_x, batch_y in batch_iterator:\n", + " print(\"batch_x has: \", batch_x)\n", + " print(\"batch_y has: \", batch_y)\n", + "\"\"\"\"\n", + "结果中\n", + " Batch会对元素类型(元素即最内层的数据，raw_sent为str，word_str_lst为str，words为int, label为int)为int或者float的数据进行默认\n", + " padding，而非int或float的则不进行padding。但若每个Instance中该field为二维数据，也不进行padding。因为二维数据的padding涉及到\n", + " 两个维度的padding，不容易自动判断padding的形式。\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "batch_x has: {'word_str_lst': array([list(['it', 'is', 'great', '.']),\n", + " list(['this', 'is', 'a', 'bad', 'idea', '.'])], dtype=object), 'words': tensor([[ 8, 2, 9, 3, -100, -100],\n", + " [ 4, 2, 5, 6, 7, 3]])}\n", + "batch_y has: {'label': tensor([1, 0])}\n" + ] + } + ], + "source": [ + "# 所有的pad_val都默认为0，如果需要修改某一个field的默认pad值，可以通过DataSet.set_pad_val(field_name, pad_val)进行修改\n", + "# 若需要将word的padding修改为-100\n", + "dataset.set_pad_val('words', pad_val=-100)\n", + "batch_iterator = Batch(dataset=dataset, batch_size=2, sampler=RandomSampler())\n", + "for batch_x, batch_y in batch_iterator:\n", + " print(\"batch_x has: \", batch_x)\n", + " print(\"batch_y has: \", batch_y)\n", + "# pad的值修改为-100了" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DataSet({'raw_sent': this is a bad idea . type=str,\n", + "'label': 0 type=int,\n", + "'word_str_lst': ['this', 'is', 'a', 'bad', 'idea', '.'] type=list,\n", + "'words': [4, 2, 5, 6, 7, 3] type=list,\n", + "'char_str_lst': [['t', 'h', 'i', 's'], ['i', 's'], ['a'], ['b', 'a', 'd'], ['i', 'd', 'e', 'a'], ['.']] type=list,\n", + "'chars': [[4, 9, 2, 5], [2, 5], [3], [10, 3, 6], [2, 6, 7, 3], [8]] type=list},\n", + "{'raw_sent': it is great . type=str,\n", + "'label': 1 type=int,\n", + "'word_str_lst': ['it', 'is', 'great', '.'] type=list,\n", + "'words': [8, 2, 9, 3] type=list,\n", + "'char_str_lst': [['i', 't'], ['i', 's'], ['g', 'r', 'e', 'a', 't'], ['.']] type=list,\n", + "'chars': [[2, 4], [2, 5], [11, 12, 7, 3, 4], [8]] type=list})" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 若需要使用二维padding或指定padding方式，可以通过设置该field的padder实现，下面以英文的character padding为例。在某些场景下，可能想要\n", + "# 使用英文word的character作为特征，character的padding为二维padding，fastNLP默认只会进行一维padding。\n", + "\n", + "dataset.apply(lambda x: [[c for c in word] for word in x['word_str_lst']], new_field_name='char_str_lst')\n", + "char_vocab = Vocabulary()\n", + "dataset.apply(lambda x:[char_vocab.update(chars) for chars in x['char_str_lst']])\n", + "dataset.apply(lambda x:[[char_vocab.to_index(c) for c in chars] for chars in x['char_str_lst']],new_field_name='chars')\n", + "dataset[:2]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "batch_x has: {'word_str_lst': array([list(['this', 'is', 'a', 'bad', 'idea', '.']),\n", + " list(['it', 'is', 'great', '.'])], dtype=object), 'words': tensor([[ 4, 2, 5, 6, 7, 3],\n", + " [ 8, 2, 9, 3, -100, -100]]), 'chars': array([list([[4, 9, 2, 5], [2, 5], [3], [10, 3, 6], [2, 6, 7, 3], [8]]),\n", + " list([[2, 4], [2, 5], [11, 12, 7, 3, 4], [8]])], dtype=object)}\n", + "batch_y has: {'label': tensor([0, 1])}\n" + ] + }, + { + "data": { + "text/plain": [ + "'\\n 其它field与之前的是相同的。chars因为存在两个维度需要padding，不能自动决定padding方式，所以直接输出了原始形式。\\n'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 如果不针对二维的character指定padding方法\n", + "dataset.set_input('chars')\n", + "batch_iterator = Batch(dataset=dataset, batch_size=2, sampler=RandomSampler())\n", + "for batch_x, batch_y in batch_iterator:\n", + " print(\"batch_x has: \", batch_x)\n", + " print(\"batch_y has: \", batch_y)\n", + " \n", + "\"\"\"\n", + " 其它field与之前的是相同的。chars因为存在两个维度需要padding，不能自动决定padding方式，所以直接输出了原始形式。\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "batch_x has: {'word_str_lst': array([list(['this', 'is', 'a', 'bad', 'idea', '.']),\n", + " list(['it', 'is', 'great', '.'])], dtype=object), 'words': tensor([[ 4, 2, 5, 6, 7, 3],\n", + " [ 8, 2, 9, 3, -100, -100]]), 'chars': tensor([[[ 4, 9, 2, 5],\n", + " [ 2, 5, 0, 0],\n", + " [ 3, 0, 0, 0],\n", + " [10, 3, 6, 0],\n", + " [ 2, 6, 7, 3],\n", + " [ 8, 0, 0, 0]],\n", + "\n", + " [[ 2, 4, 0, 0],\n", + " [ 2, 5, 0, 0],\n", + " [11, 12, 7, 3],\n", + " [ 8, 0, 0, 0],\n", + " [ 0, 0, 0, 0],\n", + " [ 0, 0, 0, 0]]])}\n", + "batch_y has: {'label': tensor([0, 1])}\n" + ] + }, + { + "data": { + "text/plain": [ + "'\\n chars被正确padding了\\n'" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 若要使用二维padding，需要手动设置padding方式\n", + "from fastNLP.core.fieldarray import EngChar2DPadder\n", + "dataset.set_padder('chars', EngChar2DPadder())\n", + "batch_iterator = Batch(dataset=dataset, batch_size=2, sampler=RandomSampler())\n", + "for batch_x, batch_y in batch_iterator:\n", + " print(\"batch_x has: \", batch_x)\n", + " print(\"batch_y has: \", batch_y)\n", + " \n", + "\"\"\"\n", + " chars被正确padding了\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "batch_x has: {'raw_sent': ['this is a bad idea .', 'it is great . '], 'word_str_lst': array([list(['this', 'is', 'a', 'bad', 'idea', '.']),\n", + " list(['it', 'is', 'great', '.'])], dtype=object), 'words': tensor([[ 4, 2, 5, 6, 7, 3],\n", + " [ 8, 2, 9, 3, -100, -100]]), 'chars': tensor([[[ 4, 9, 2, 5],\n", + " [ 2, 5, 0, 0],\n", + " [ 3, 0, 0, 0],\n", + " [10, 3, 6, 0],\n", + " [ 2, 6, 7, 3],\n", + " [ 8, 0, 0, 0]],\n", + "\n", + " [[ 2, 4, 0, 0],\n", + " [ 2, 5, 0, 0],\n", + " [11, 12, 7, 3],\n", + " [ 8, 0, 0, 0],\n", + " [ 0, 0, 0, 0],\n", + " [ 0, 0, 0, 0]]])}\n", + "batch_y has: {'label': tensor([0, 1])}\n" + ] + }, + { + "data": { + "text/plain": [ + "'\\n raw_sent正确输出，对应内容也进行了pad。\\n'" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 如果AutoPad与EngChar2DPadder不能满足需要，可以自己实现Padder对象。这里举一个例子，比如需要把raw_sentence pad到一样长\n", + "from fastNLP.core.fieldarray import PadderBase\n", + "\n", + "class PadStr(PadderBase):\n", + " def __init__(self, pad_val=' '):\n", + " super().__init__(pad_val=pad_val) #让父类管理pad_val的值，这样可以通过DataSet.set_pad_val()修改到该值\n", + " \n", + " def __call__(self, contents, field_name, field_ele_dtype):\n", + " \"\"\"\n", + " 如果以上面的例子举例，在raw_sent这个field进行pad时，传入的\n", + " contents:\n", + " [\n", + " 'This is a bad idea .',\n", + " 'It is great .'\n", + " ]\n", + " field_name: 'raw_sent'，当前field的名称，主要用于帮助debug。\n", + " field_ele_dtype: np.str. 这个参数基本都用不上，是该field中内部元素的类型\n", + " \"\"\"\n", + " max_len = max([len(str_) for str_ in contents])\n", + " pad_strs = []\n", + " for content in contents:\n", + " pad_strs.append(content + (max_len-len(content))*self.pad_val)\n", + " return pad_strs\n", + "\n", + "dataset.set_input('raw_sent')\n", + "dataset.set_padder('raw_sent', PadStr())\n", + "batch_iterator = Batch(dataset=dataset, batch_size=2, sampler=RandomSampler())\n", + "for batch_x, batch_y in batch_iterator:\n", + " print(\"batch_x has: \", batch_x)\n", + " print(\"batch_y has: \", batch_y)\n", + "\n", + "\"\"\"\n", + " raw_sent正确输出，对应内容也进行了pad。\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 3e33a23042de762b6b7265b5c16b8a67b57eb444 Mon Sep 17 00:00:00 2001 From: yh Date: Tue, 15 Jan 2019 22:23:19 +0800 Subject: [PATCH 08/32] =?UTF-8?q?=E4=BF=AE=E6=94=B9Padder=E7=9A=84?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/core/test_fieldarray.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/core/test_fieldarray.py b/test/core/test_fieldarray.py index 8b9e8754..82285462 100644 --- a/test/core/test_fieldarray.py +++ b/test/core/test_fieldarray.py @@ -118,11 +118,12 @@ def test01(self): self.assertListEqual([[1,2], [3, 0], [4, 0]], padder(content, None, np.int64).tolist()) - contents = [ + content = [ [[1, 2, 3], [4, 5], [7,8,9,10]], [[1]] ] - print(padder(contents, None, np.int64)) + self.assertListEqual(content, + padder(content, None, np.int64).tolist()) def test02(self): """ From e4f997d52a733c67d62392056eb01924519c2837 Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Thu, 17 Jan 2019 12:25:37 +0800 Subject: [PATCH 09/32] =?UTF-8?q?refactor=20type=20system=20in=20FieldArra?= =?UTF-8?q?y:=20*=20=E9=87=8D=E6=9E=84dtype=E7=9A=84=E6=A3=80=E6=B5=8B?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=EF=BC=8C=E5=9C=A8FieldArray=E7=9A=84?= =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E5=92=8Cappend=E4=B8=A4=E5=A4=84?= =?UTF-8?q?=EF=BC=8C=E8=BE=BE=E5=88=B0=E6=9B=B4=E5=A5=BD=E7=9A=84=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E5=A4=8D=E7=94=A8=20*=20=E7=B1=BB=E5=9E=8B=E6=A3=80?= =?UTF-8?q?=E6=B5=8B=E7=9A=84=E8=B4=A3=E4=BB=BB=E5=AE=8C=E5=85=A8=E8=90=BD?= =?UTF-8?q?=E5=9C=A8FieldArray=EF=BC=8CDataSet=E4=B8=8E=E4=B9=8B=E9=85=8D?= =?UTF-8?q?=E5=90=88=20=E6=B5=8B=E8=AF=95=EF=BC=9A=20*=20=E6=95=B4?= =?UTF-8?q?=E7=90=86dtype=E7=9B=B8=E5=85=B3=E7=9A=84=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=20*=20=E7=BB=99=E6=89=80=E6=9C=89tutorial?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=B5=8B=E8=AF=95=20=E5=85=B6=E4=BB=96?= =?UTF-8?q?=EF=BC=9A=20*=20=E5=AE=8C=E5=96=84=E4=B8=80=E4=B8=AA=E5=AE=8C?= =?UTF-8?q?=E6=95=B4=E7=9A=84Conll=20dataset=20loader=20*=20=E5=8D=87?= =?UTF-8?q?=E7=BA=A7POS=20tag=20model=E8=AE=AD=E7=BB=83=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/dataset.py | 16 +- fastNLP/core/fieldarray.py | 214 +++++++---- fastNLP/core/instance.py | 6 +- fastNLP/io/dataset_loader.py | 22 +- reproduction/POS_tagging/pos_tag.cfg | 2 +- reproduction/POS_tagging/train_pos_tag.py | 90 ++++- test/core/test_batch.py | 9 + test/core/test_dataset.py | 20 +- test/core/test_fieldarray.py | 10 +- test/models/test_biaffine_parser.py | 15 +- test/test_tutorial.py | 91 ----- test/test_tutorials.py | 432 ++++++++++++++++++++++ 12 files changed, 725 insertions(+), 202 deletions(-) delete mode 100644 test/test_tutorial.py create mode 100644 test/test_tutorials.py diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index 2dba3267..f4e64c5d 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -2,8 +2,8 @@ import numpy as np -from fastNLP.core.fieldarray import FieldArray from fastNLP.core.fieldarray import AutoPadder +from fastNLP.core.fieldarray import FieldArray from fastNLP.core.instance import Instance from fastNLP.core.utils import get_func_signature from fastNLP.io.base_loader import DataLoaderRegister @@ -142,7 +142,8 @@ def append(self, ins): if len(self.field_arrays) == 0: # DataSet has no field yet for name, field in ins.fields.items(): - self.field_arrays[name] = FieldArray(name, [field]) + field = field.tolist() if isinstance(field, np.ndarray) else field + self.field_arrays[name] = FieldArray(name, [field]) # 第一个样本，必须用list包装起来 else: if len(self.field_arrays) != len(ins.fields): raise ValueError( @@ -290,9 +291,11 @@ def apply(self, func, new_field_name=None, **kwargs): extra_param['is_input'] = old_field.is_input if 'is_target' not in extra_param: extra_param['is_target'] = old_field.is_target - self.add_field(name=new_field_name, fields=results) + self.add_field(name=new_field_name, fields=results, is_input=extra_param["is_input"], + is_target=extra_param["is_target"]) else: - self.add_field(name=new_field_name, fields=results) + self.add_field(name=new_field_name, fields=results, is_input=extra_param.get("is_input", None), + is_target=extra_param.get("is_target", None)) else: return results @@ -334,13 +337,14 @@ def split(self, dev_ratio): train_set.field_arrays[field_name].padder = self.field_arrays[field_name].padder train_set.field_arrays[field_name].dtype = self.field_arrays[field_name].dtype train_set.field_arrays[field_name].pytype = self.field_arrays[field_name].pytype - train_set.field_arrays[field_name].is_2d_list = self.field_arrays[field_name].is_2d_list + train_set.field_arrays[field_name].content_dim = self.field_arrays[field_name].content_dim + dev_set.field_arrays[field_name].is_input = self.field_arrays[field_name].is_input dev_set.field_arrays[field_name].is_target = self.field_arrays[field_name].is_target dev_set.field_arrays[field_name].padder = self.field_arrays[field_name].padder dev_set.field_arrays[field_name].dtype = self.field_arrays[field_name].dtype dev_set.field_arrays[field_name].pytype = self.field_arrays[field_name].pytype - dev_set.field_arrays[field_name].is_2d_list = self.field_arrays[field_name].is_2d_list + dev_set.field_arrays[field_name].content_dim = self.field_arrays[field_name].content_dim return train_set, dev_set diff --git a/fastNLP/core/fieldarray.py b/fastNLP/core/fieldarray.py index afb81697..4cde86ab 100644 --- a/fastNLP/core/fieldarray.py +++ b/fastNLP/core/fieldarray.py @@ -100,6 +100,22 @@ class FieldArray(object): """ def __init__(self, name, content, is_target=None, is_input=None, padder=AutoPadder(pad_val=0)): + """DataSet在初始化时会有两类方法对FieldArray操作： + 1） 如果DataSet使用dict初始化，那么在add_field中会构造FieldArray： + 1.1) 二维list DataSet({"x": [[1, 2], [3, 4]]}) + 1.2) 二维array DataSet({"x": np.array([[1, 2], [3, 4]])}) + 1.3) 三维list DataSet({"x": [[[1, 2], [3, 4]], [[1, 2], [3, 4]]]}) + 2） 如果DataSet使用list of Instance 初始化,那么在append中会先对第一个样本初始化FieldArray； + 然后后面的样本使用FieldArray.append进行添加。 + 2.1) 一维list DataSet([Instance(x=[1, 2, 3, 4])]) + 2.2) 一维array DataSet([Instance(x=np.array([1, 2, 3, 4]))]) + 2.3) 二维list DataSet([Instance(x=[[1, 2], [3, 4]])]) + 2.4) 二维array DataSet([Instance(x=np.array([[1, 2], [3, 4]]))]) + + 注意：np.array必须仅在最外层，即np.array([np.array, np.array]) 和 list of np.array不考虑 + 类型检查(dtype check)发生在当该field被设置为is_input或者is_target时。 + + """ self.name = name if isinstance(content, list): content = content @@ -107,31 +123,39 @@ def __init__(self, name, content, is_target=None, is_input=None, padder=AutoPadd content = content.tolist() # convert np.ndarray into 2-D list else: raise TypeError("content in FieldArray can only be list or numpy.ndarray, got {}.".format(type(content))) - self.content = content + if len(content) == 0: + raise RuntimeError("Cannot initialize FieldArray with empty list.") + + self.content = content # 1维 或 2维 或 3维 list, 形状可能不对齐 + self.content_dim = None # 表示content是多少维的list self.set_padder(padder) - self._is_target = None - self._is_input = None + self.BASIC_TYPES = (int, float, str) # content中可接受的Python基本类型，这里没有np.array - self.BASIC_TYPES = (int, float, str, np.ndarray) - self.is_2d_list = False - self.pytype = None # int, float, str, or np.ndarray - self.dtype = None # np.int64, np.float64, np.str + self.pytype = None + self.dtype = None + self._is_input = None + self._is_target = None - if is_input is not None: + if is_input is not None or is_target is not None: self.is_input = is_input - if is_target is not None: self.is_target = is_target + def _set_dtype(self): + self.pytype = self._type_detection(self.content) + self.dtype = self._map_to_np_type(self.pytype) + @property def is_input(self): return self._is_input @is_input.setter def is_input(self, value): + """ + 当 field_array.is_input = True / False 时被调用 + """ if value is True: - self.pytype = self._type_detection(self.content) - self.dtype = self._map_to_np_type(self.pytype) + self._set_dtype() self._is_input = value @property @@ -140,46 +164,99 @@ def is_target(self): @is_target.setter def is_target(self, value): + """ + 当 field_array.is_target = True / False 时被调用 + """ if value is True: - self.pytype = self._type_detection(self.content) - self.dtype = self._map_to_np_type(self.pytype) + self._set_dtype() self._is_target = value def _type_detection(self, content): - """ - - :param content: a list of int, float, str or np.ndarray, or a list of list of one. - :return type: one of int, float, str, np.ndarray + """当该field被设置为is_input或者is_target时被调用 """ - if isinstance(content, list) and len(content) > 0 and isinstance(content[0], list): - # content is a 2-D list - if not all(isinstance(_, list) for _ in content): # strict check 2-D list - raise TypeError("Please provide 2-D list.") - type_set = set([self._type_detection(x) for x in content]) - if len(type_set) == 2 and int in type_set and float in type_set: - type_set = {float} - elif len(type_set) > 1: - raise TypeError("Cannot create FieldArray with more than one type. Provided {}".format(type_set)) - self.is_2d_list = True + if len(content) == 0: + raise RuntimeError("Empty list in Field {}.".format(self.name)) + + type_set = set([type(item) for item in content]) + + if list in type_set: + if len(type_set) > 1: + # list 跟 非list 混在一起 + raise RuntimeError("Mixed data types in Field {}: {}".format(self.name, type_set)) + # >1维list + inner_type_set = set() + for l in content: + [inner_type_set.add(type(obj)) for obj in l] + if list not in inner_type_set: + # 二维list + self.content_dim = 2 + return self._basic_type_detection(inner_type_set) + else: + if len(inner_type_set) == 1: + # >2维list + inner_inner_type_set = set() + for _2d_list in content: + for _1d_list in _2d_list: + [inner_inner_type_set.add(type(obj)) for obj in _1d_list] + if list in inner_inner_type_set: + raise RuntimeError("FieldArray cannot handle 4-D or more-D list.") + # 3维list + self.content_dim = 3 + return self._basic_type_detection(inner_inner_type_set) + else: + # list 跟 非list 混在一起 + raise RuntimeError("Mixed data types in Field {}: {}".format(self.name, inner_type_set)) + else: + # 一维list + for content_type in type_set: + if content_type not in self.BASIC_TYPES: + raise RuntimeError("Unexpected data type in Field '{}'. Expect one of {}. Got {}.".format( + self.name, self.BASIC_TYPES, content_type)) + self.content_dim = 1 + return self._basic_type_detection(type_set) + + def _basic_type_detection(self, type_set): + """ + :param type_set: a set of Python types + :return: one of self.BASIC_TYPES + """ + if len(type_set) == 1: return type_set.pop() - - elif isinstance(content, list): - # content is a 1-D list - if len(content) == 0: - # the old error is not informative enough. - raise RuntimeError("Cannot create FieldArray with an empty list. Or one element in the list is empty.") - type_set = set([type(item) for item in content]) - - if len(type_set) == 1 and tuple(type_set)[0] in self.BASIC_TYPES: - return type_set.pop() - elif len(type_set) == 2 and float in type_set and int in type_set: + elif len(type_set) == 2: + # 有多个basic type; 可能需要up-cast + if float in type_set and int in type_set: # up-cast int to float return float else: - raise TypeError("Cannot create FieldArray with type {}".format(*type_set)) + # str 跟 int 或者 float 混在一起 + raise RuntimeError("Mixed data types in Field {}: {}".format(self.name, type_set)) else: - raise TypeError("Cannot create FieldArray with type {}".format(type(content))) + # str, int, float混在一起 + raise RuntimeError("Mixed data types in Field {}: {}".format(self.name, type_set)) + + def _1d_list_check(self, val): + """如果不是1D list就报错 + """ + type_set = set((type(obj) for obj in val)) + if any(obj not in self.BASIC_TYPES for obj in type_set): + raise ValueError("Mixed data types in Field {}: {}".format(self.name, type_set)) + self._basic_type_detection(type_set) + # otherwise: _basic_type_detection will raise error + return True + + def _2d_list_check(self, val): + """如果不是2D list 就报错 + """ + type_set = set(type(obj) for obj in val) + if list(type_set) != [list]: + raise ValueError("Mixed data types in Field {}: {}".format(self.name, type_set)) + inner_type_set = set() + for l in val: + for obj in l: + inner_type_set.add(type(obj)) + self._basic_type_detection(inner_type_set) + return True @staticmethod def _map_to_np_type(basic_type): @@ -194,38 +271,39 @@ def append(self, val): :param val: int, float, str, or a list of one. """ - if self.is_target is True or self.is_input is True: - # only check type when used as target or input + if isinstance(val, list): + pass + elif isinstance(val, tuple): # 确保最外层是list + val = list(val) + elif isinstance(val, np.ndarray): + val = val.tolist() + elif any((isinstance(val, t) for t in self.BASIC_TYPES)): + pass + else: + raise RuntimeError( + "Unexpected data type {}. Should be list, np.array, or {}".format(type(val), self.BASIC_TYPES)) - val_type = type(val) - if val_type == list: # shape check - if self.is_2d_list is False: - raise RuntimeError("Cannot append a list into a 1-D FieldArray. Please provide an element.") + if self.is_input is True or self.is_target is True: + if type(val) == list: if len(val) == 0: - raise RuntimeError("Cannot append an empty list.") - val_list_type = set([type(_) for _ in val]) # type check - if len(val_list_type) == 2 and int in val_list_type and float in val_list_type: - # up-cast int to float - val_type = float - elif len(val_list_type) == 1: - val_type = val_list_type.pop() + raise ValueError("Cannot append an empty list.") + if self.content_dim == 2 and self._1d_list_check(val): + # 1维list检查 + pass + elif self.content_dim == 3 and self._2d_list_check(val): + # 2维list检查 + pass else: - raise TypeError("Cannot append a list of {}".format(val_list_type)) - else: - if self.is_2d_list is True: - raise RuntimeError("Cannot append a non-list into a 2-D list. Please provide a list.") - - if val_type == float and self.pytype == int: - # up-cast - self.pytype = float - self.dtype = self._map_to_np_type(self.pytype) - elif val_type == int and self.pytype == float: - pass - elif val_type == self.pytype: - pass + raise RuntimeError( + "Dimension not matched: expect dim={}, got {}.".format(self.content_dim - 1, val)) + elif type(val) in self.BASIC_TYPES and self.content_dim == 1: + # scalar检查 + if type(val) == float and self.pytype == int: + self.pytype = float + self.dtype = self._map_to_np_type(self.pytype) else: - raise TypeError("Cannot append type {} into type {}".format(val_type, self.pytype)) - + raise RuntimeError( + "Unexpected data type {}. Should be list, np.array, or {}".format(type(val), self.BASIC_TYPES)) self.content.append(val) def __getitem__(self, indices): diff --git a/fastNLP/core/instance.py b/fastNLP/core/instance.py index a102b51c..5ac52e3f 100644 --- a/fastNLP/core/instance.py +++ b/fastNLP/core/instance.py @@ -11,6 +11,10 @@ class Instance(object): """ def __init__(self, **fields): + """ + + :param fields: 可能是一维或者二维的 list or np.array + """ self.fields = fields def add_field(self, field_name, field): @@ -32,5 +36,5 @@ def __setitem__(self, name, field): def __repr__(self): s = '\'' return "{" + ",\n".join( - "\'" + field_name + "\': " + str(self.fields[field_name]) +\ + "\'" + field_name + "\': " + str(self.fields[field_name]) + \ f" type={(str(type(self.fields[field_name]))).split(s)[1]}" for field_name in self.fields) + "}" diff --git a/fastNLP/io/dataset_loader.py b/fastNLP/io/dataset_loader.py index 2d157da3..fb781c3e 100644 --- a/fastNLP/io/dataset_loader.py +++ b/fastNLP/io/dataset_loader.py @@ -858,9 +858,22 @@ def load(self, path): ds.append(Instance(words=char_seq, tag=pos_seq)) - return ds + def get_one(self, sample): + if len(sample) == 0: + return None + text = [] + pos_tags = [] + for w in sample: + t1, t2, t3, t4 = w[1], w[3], w[6], w[7] + if t3 == '_': + return None + text.append(t1) + pos_tags.append(t2) + return text, pos_tags + + class ConllxDataLoader(object): def load(self, path): @@ -879,7 +892,12 @@ def load(self, path): datalist.append(sample) data = [self.get_one(sample) for sample in datalist] - return list(filter(lambda x: x is not None, data)) + data_list = list(filter(lambda x: x is not None, data)) + + ds = DataSet() + for example in data_list: + ds.append(Instance(words=example[0], tag=example[1])) + return ds def get_one(self, sample): sample = list(map(list, zip(*sample))) diff --git a/reproduction/POS_tagging/pos_tag.cfg b/reproduction/POS_tagging/pos_tag.cfg index c9ee8320..f8224234 100644 --- a/reproduction/POS_tagging/pos_tag.cfg +++ b/reproduction/POS_tagging/pos_tag.cfg @@ -10,7 +10,7 @@ eval_sort_key = 'accuracy' [model] rnn_hidden_units = 300 -word_emb_dim = 100 +word_emb_dim = 300 dropout = 0.5 use_crf = true print_every_step = 10 diff --git a/reproduction/POS_tagging/train_pos_tag.py b/reproduction/POS_tagging/train_pos_tag.py index 09a9ba02..e817db44 100644 --- a/reproduction/POS_tagging/train_pos_tag.py +++ b/reproduction/POS_tagging/train_pos_tag.py @@ -8,16 +8,16 @@ # in order to run fastNLP without installation sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) - from fastNLP.api.pipeline import Pipeline -from fastNLP.api.processor import SeqLenProcessor, VocabIndexerProcessor +from fastNLP.api.processor import SeqLenProcessor, VocabIndexerProcessor, SetInputProcessor, IndexerProcessor from fastNLP.core.metrics import SpanFPreRecMetric from fastNLP.core.trainer import Trainer from fastNLP.io.config_io import ConfigLoader, ConfigSection from fastNLP.models.sequence_modeling import AdvSeqLabel -from fastNLP.io.dataset_loader import ZhConllPOSReader +from fastNLP.io.dataset_loader import ZhConllPOSReader, ConllxDataLoader from fastNLP.api.processor import ModelProcessor, Index2WordProcessor + cfgfile = './pos_tag.cfg' pickle_path = "save" @@ -35,7 +35,7 @@ def load_tencent_embed(embed_path, word2id): return embedding_tensor -def train(checkpoint=None): +def train(train_data_path, dev_data_path, checkpoint=None): # load config train_param = ConfigSection() model_param = ConfigSection() @@ -43,24 +43,36 @@ def train(checkpoint=None): print("config loaded") # Data Loader - dataset = ZhConllPOSReader().load("/home/hyan/train.conllx") + print("loading training set...") + dataset = ConllxDataLoader().load(train_data_path) + print("loading dev set...") + dev_data = ConllxDataLoader().load(dev_data_path) print(dataset) - print("dataset transformed") + print("================= dataset ready =====================") dataset.rename_field("tag", "truth") + dev_data.rename_field("tag", "truth") vocab_proc = VocabIndexerProcessor("words", new_added_filed_name="word_seq") tag_proc = VocabIndexerProcessor("truth") seq_len_proc = SeqLenProcessor(field_name="word_seq", new_added_field_name="word_seq_origin_len", is_input=True) + set_input_proc = SetInputProcessor("word_seq", "word_seq_origin_len", "truth") vocab_proc(dataset) tag_proc(dataset) seq_len_proc(dataset) + # index dev set + word_vocab, tag_vocab = vocab_proc.vocab, tag_proc.vocab + dev_data.apply(lambda ins: [word_vocab.to_index(w) for w in ins["words"]], new_field_name="word_seq") + dev_data.apply(lambda ins: [tag_vocab.to_index(w) for w in ins["truth"]], new_field_name="truth") + dev_data.apply(lambda ins: len(ins["word_seq"]), new_field_name="word_seq_origin_len") + + # set input & target dataset.set_input("word_seq", "word_seq_origin_len", "truth") + dev_data.set_input("word_seq", "word_seq_origin_len", "truth") dataset.set_target("truth", "word_seq_origin_len") - - print("processors defined") + dev_data.set_target("truth", "word_seq_origin_len") # dataset.set_is_target(tag_ids=True) model_param["vocab_size"] = vocab_proc.get_vocab_size() @@ -71,7 +83,7 @@ def train(checkpoint=None): if checkpoint is None: # pre_trained = load_tencent_embed("/home/zyfeng/data/char_tencent_embedding.pkl", vocab_proc.vocab.word2idx) pre_trained = None - model = AdvSeqLabel(model_param, id2words=tag_proc.vocab.idx2word, emb=pre_trained) + model = AdvSeqLabel(model_param, id2words=None, emb=pre_trained) print(model) else: model = torch.load(checkpoint) @@ -80,33 +92,71 @@ def train(checkpoint=None): trainer = Trainer(dataset, model, loss=None, metrics=SpanFPreRecMetric(tag_proc.vocab, pred="predict", target="truth", seq_lens="word_seq_origin_len"), - dev_data=dataset, metric_key="f", - use_tqdm=True, use_cuda=True, print_every=5, n_epochs=6, save_path="./save") + dev_data=dev_data, metric_key="f", + use_tqdm=True, use_cuda=True, print_every=5, n_epochs=6, save_path="./save_0") trainer.train(load_best_model=True) # save model & pipeline model_proc = ModelProcessor(model, seq_len_field_name="word_seq_origin_len") id2tag = Index2WordProcessor(tag_proc.vocab, "predict", "tag") - pp = Pipeline([vocab_proc, seq_len_proc, model_proc, id2tag]) + pp = Pipeline([vocab_proc, seq_len_proc, set_input_proc, model_proc, id2tag]) save_dict = {"pipeline": pp, "model": model, "tag_vocab": tag_proc.vocab} torch.save(save_dict, "model_pp.pkl") print("pipeline saved") - torch.save(model, "./save/best_model.pkl") + +def run_test(test_path): + test_data = ZhConllPOSReader().load(test_path) + + with open("model_pp.pkl", "rb") as f: + save_dict = torch.load(f) + tag_vocab = save_dict["tag_vocab"] + pipeline = save_dict["pipeline"] + index_tag = IndexerProcessor(vocab=tag_vocab, field_name="tag", new_added_field_name="truth", is_input=False) + pipeline.pipeline = [index_tag] + pipeline.pipeline + + pipeline(test_data) + test_data.set_target("truth") + prediction = test_data.field_arrays["predict"].content + truth = test_data.field_arrays["truth"].content + seq_len = test_data.field_arrays["word_seq_origin_len"].content + + # padding by hand + max_length = max([len(seq) for seq in prediction]) + for idx in range(len(prediction)): + prediction[idx] = list(prediction[idx]) + ([0] * (max_length - len(prediction[idx]))) + truth[idx] = list(truth[idx]) + ([0] * (max_length - len(truth[idx]))) + evaluator = SpanFPreRecMetric(tag_vocab=tag_vocab, pred="predict", target="truth", + seq_lens="word_seq_origin_len") + evaluator({"predict": torch.Tensor(prediction), "word_seq_origin_len": torch.Tensor(seq_len)}, + {"truth": torch.Tensor(truth)}) + test_result = evaluator.get_metric() + f1 = round(test_result['f'] * 100, 2) + pre = round(test_result['pre'] * 100, 2) + rec = round(test_result['rec'] * 100, 2) + + return {"F1": f1, "precision": pre, "recall": rec} if __name__ == "__main__": parser = argparse.ArgumentParser() + parser.add_argument("--train", type=str, help="training conll file", default="/home/zyfeng/data/sample.conllx") + parser.add_argument("--dev", type=str, help="dev conll file", default="/home/zyfeng/data/sample.conllx") + parser.add_argument("--test", type=str, help="test conll file", default=None) + parser.add_argument("-c", "--restart", action="store_true", help="whether to continue training") parser.add_argument("-cp", "--checkpoint", type=str, help="checkpoint of the trained model") args = parser.parse_args() - if args.restart is True: - # 继续训练 python train_pos_tag.py -c -cp ./save/best_model.pkl - if args.checkpoint is None: - raise RuntimeError("Please provide the checkpoint. -cp ") - train(args.checkpoint) + if args.test is not None: + print(run_test(args.test)) else: - # 一次训练 python train_pos_tag.py - train() + if args.restart is True: + # 继续训练 python train_pos_tag.py -c -cp ./save/best_model.pkl + if args.checkpoint is None: + raise RuntimeError("Please provide the checkpoint. -cp ") + train(args.train, args.dev, args.checkpoint) + else: + # 一次训练 python train_pos_tag.py + train(args.train, args.dev) diff --git a/test/core/test_batch.py b/test/core/test_batch.py index 77aebea5..7308ebf0 100644 --- a/test/core/test_batch.py +++ b/test/core/test_batch.py @@ -89,3 +89,12 @@ def test_list_of_list_to_tensor(self): self.assertEqual(tuple(x["x"].shape), (4, 4)) self.assertTrue(isinstance(y["y"], torch.Tensor)) self.assertEqual(tuple(y["y"].shape), (4, 4)) + + def test_list_of_numpy_to_tensor(self): + ds = DataSet([Instance(x=np.array([1, 2]), y=np.array([3, 4])) for _ in range(2)] + + [Instance(x=np.array([1, 2, 3, 4]), y=np.array([3, 4, 5, 6])) for _ in range(2)]) + ds.set_input("x") + ds.set_target("y") + iter = Batch(ds, batch_size=4, sampler=SequentialSampler(), as_numpy=False) + for x, y in iter: + print(x, y) diff --git a/test/core/test_dataset.py b/test/core/test_dataset.py index 261d42b3..72ced912 100644 --- a/test/core/test_dataset.py +++ b/test/core/test_dataset.py @@ -6,15 +6,29 @@ from fastNLP.core.instance import Instance -class TestDataSet(unittest.TestCase): - +class TestDataSetInit(unittest.TestCase): + """初始化DataSet的办法有以下几种： + 1) 用dict: + 1.1) 二维list DataSet({"x": [[1, 2], [3, 4]]}) + 1.2) 二维array DataSet({"x": np.array([[1, 2], [3, 4]])}) + 1.3) 三维list DataSet({"x": [[[1, 2], [3, 4]], [[1, 2], [3, 4]]]}) + 2) 用list of Instance: + 2.1) 一维list DataSet([Instance(x=[1, 2, 3, 4])]) + 2.2) 一维array DataSet([Instance(x=np.array([1, 2, 3, 4]))]) + 2.3) 二维list DataSet([Instance(x=[[1, 2], [3, 4]])]) + 2.4) 二维array DataSet([Instance(x=np.array([[1, 2], [3, 4]]))]) + + 只接受纯list或者最外层ndarray + """ def test_init_v1(self): + # 一维list ds = DataSet([Instance(x=[1, 2, 3, 4], y=[5, 6])] * 40) self.assertTrue("x" in ds.field_arrays and "y" in ds.field_arrays) self.assertEqual(ds.field_arrays["x"].content, [[1, 2, 3, 4], ] * 40) self.assertEqual(ds.field_arrays["y"].content, [[5, 6], ] * 40) def test_init_v2(self): + # 用dict ds = DataSet({"x": [[1, 2, 3, 4]] * 40, "y": [[5, 6]] * 40}) self.assertTrue("x" in ds.field_arrays and "y" in ds.field_arrays) self.assertEqual(ds.field_arrays["x"].content, [[1, 2, 3, 4], ] * 40) @@ -28,6 +42,8 @@ def test_init_assert(self): with self.assertRaises(ValueError): _ = DataSet(0.00001) + +class TestDataSetMethods(unittest.TestCase): def test_append(self): dd = DataSet() for _ in range(3): diff --git a/test/core/test_fieldarray.py b/test/core/test_fieldarray.py index 82285462..da287916 100644 --- a/test/core/test_fieldarray.py +++ b/test/core/test_fieldarray.py @@ -42,13 +42,13 @@ def test_type_conversion(self): self.assertEqual(fa.pytype, str) def test_support_np_array(self): - fa = FieldArray("y", [np.array([1.1, 2.2, 3.3, 4.4, 5.5])], is_input=True) - self.assertEqual(fa.dtype, np.ndarray) - self.assertEqual(fa.pytype, np.ndarray) + fa = FieldArray("y", np.array([[1.1, 2.2, 3.3, 4.4, 5.5]]), is_input=True) + self.assertEqual(fa.dtype, np.float64) + self.assertEqual(fa.pytype, float) fa.append(np.array([1.1, 2.2, 3.3, 4.4, 5.5])) - self.assertEqual(fa.dtype, np.ndarray) - self.assertEqual(fa.pytype, np.ndarray) + self.assertEqual(fa.dtype, np.float64) + self.assertEqual(fa.pytype, float) fa = FieldArray("my_field", np.random.rand(3, 5), is_input=True) # in this case, pytype is actually a float. We do not care about it. diff --git a/test/models/test_biaffine_parser.py b/test/models/test_biaffine_parser.py index d87000a0..88ba09b8 100644 --- a/test/models/test_biaffine_parser.py +++ b/test/models/test_biaffine_parser.py @@ -1,8 +1,8 @@ -from fastNLP.models.biaffine_parser import BiaffineParser, ParserLoss, ParserMetric -import fastNLP - import unittest +import fastNLP +from fastNLP.models.biaffine_parser import BiaffineParser, ParserLoss, ParserMetric + data_file = """ 1 The _ DET DT _ 3 det _ _ 2 new _ ADJ JJ _ 3 amod _ _ @@ -41,6 +41,7 @@ """ + def init_data(): ds = fastNLP.DataSet() v = {'word_seq': fastNLP.Vocabulary(), @@ -60,18 +61,19 @@ def init_data(): data.append(line) for name in ['word_seq', 'pos_seq', 'label_true']: - ds.apply(lambda x: ['']+list(x[name]), new_field_name=name) + ds.apply(lambda x: [''] + list(x[name]), new_field_name=name) ds.apply(lambda x: v[name].add_word_lst(x[name])) for name in ['word_seq', 'pos_seq', 'label_true']: ds.apply(lambda x: [v[name].to_index(w) for w in x[name]], new_field_name=name) - ds.apply(lambda x: [0]+list(map(int, x['arc_true'])), new_field_name='arc_true') + ds.apply(lambda x: [0] + list(map(int, x['arc_true'])), new_field_name='arc_true') ds.apply(lambda x: len(x['word_seq']), new_field_name='seq_lens') ds.set_input('word_seq', 'pos_seq', 'seq_lens', flag=True) ds.set_target('arc_true', 'label_true', 'seq_lens', flag=True) return ds, v['word_seq'], v['pos_seq'], v['label_true'] + class TestBiaffineParser(unittest.TestCase): def test_train(self): ds, v1, v2, v3 = init_data() @@ -84,5 +86,6 @@ def test_train(self): n_epochs=10, use_cuda=False, use_tqdm=False) trainer.train(load_best_model=False) + if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/test/test_tutorial.py b/test/test_tutorial.py deleted file mode 100644 index 68cb6a41..00000000 --- a/test/test_tutorial.py +++ /dev/null @@ -1,91 +0,0 @@ -import unittest - -from fastNLP import DataSet -from fastNLP import Instance -from fastNLP import Tester -from fastNLP import Vocabulary -from fastNLP.core.losses import CrossEntropyLoss -from fastNLP.core.metrics import AccuracyMetric -from fastNLP.models import CNNText - - -class TestTutorial(unittest.TestCase): - def test_tutorial(self): - # 从csv读取数据到DataSet - sample_path = "test/data_for_tests/tutorial_sample_dataset.csv" - dataset = DataSet.read_csv(sample_path, headers=('raw_sentence', 'label'), - sep='\t') - print(len(dataset)) - print(dataset[0]) - - dataset.append(Instance(raw_sentence='fake data', label='0')) - dataset.apply(lambda x: x['raw_sentence'].lower(), new_field_name='raw_sentence') - # label转int - dataset.apply(lambda x: int(x['label']), new_field_name='label') - - # 使用空格分割句子 - def split_sent(ins): - return ins['raw_sentence'].split() - - dataset.apply(split_sent, new_field_name='words') - # 增加长度信息 - dataset.apply(lambda x: len(x['words']), new_field_name='seq_len') - print(len(dataset)) - print(dataset[0]) - - # DataSet.drop(func)筛除数据 - dataset.drop(lambda x: x['seq_len'] <= 3) - print(len(dataset)) - - # 设置DataSet中，哪些field要转为tensor - # set target，loss或evaluate中的golden，计算loss，模型评估时使用 - dataset.set_target("label") - # set input，模型forward时使用 - dataset.set_input("words") - - # 分出测试集、训练集 - test_data, train_data = dataset.split(0.5) - print(len(test_data)) - print(len(train_data)) - - # 构建词表, Vocabulary.add(word) - vocab = Vocabulary(min_freq=2) - train_data.apply(lambda x: [vocab.add(word) for word in x['words']]) - vocab.build_vocab() - - # index句子, Vocabulary.to_index(word) - train_data.apply(lambda x: [vocab.to_index(word) for word in x['words']], new_field_name='words') - test_data.apply(lambda x: [vocab.to_index(word) for word in x['words']], new_field_name='words') - print(test_data[0]) - - model = CNNText(embed_num=len(vocab), embed_dim=50, num_classes=5, padding=2, dropout=0.1) - - from fastNLP import Trainer - from copy import deepcopy - - # 更改DataSet中对应field的名称，要以模型的forward等参数名一致 - train_data.rename_field('words', 'word_seq') # input field 与 forward 参数一致 - train_data.rename_field('label', 'label_seq') - test_data.rename_field('words', 'word_seq') - test_data.rename_field('label', 'label_seq') - - # 实例化Trainer，传入模型和数据，进行训练 - copy_model = deepcopy(model) - overfit_trainer = Trainer(train_data=test_data, model=copy_model, - loss=CrossEntropyLoss(pred="output", target="label_seq"), - metrics=AccuracyMetric(pred="predict", target="label_seq"), n_epochs=10, batch_size=4, - dev_data=test_data, save_path="./save") - overfit_trainer.train() - - trainer = Trainer(train_data=train_data, model=model, - loss=CrossEntropyLoss(pred="output", target="label_seq"), - metrics=AccuracyMetric(pred="predict", target="label_seq"), n_epochs=10, batch_size=4, - dev_data=test_data, save_path="./save") - trainer.train() - print('Train finished!') - - # 使用fastNLP的Tester测试脚本 - tester = Tester(data=test_data, model=model, metrics=AccuracyMetric(pred="predict", target="label_seq"), - batch_size=4) - acc = tester.test() - print(acc) diff --git a/test/test_tutorials.py b/test/test_tutorials.py new file mode 100644 index 00000000..ee48c23b --- /dev/null +++ b/test/test_tutorials.py @@ -0,0 +1,432 @@ +import unittest + +from fastNLP import DataSet +from fastNLP import Instance +from fastNLP import Vocabulary +from fastNLP.core.losses import CrossEntropyLoss +from fastNLP.core.metrics import AccuracyMetric + + +class TestTutorial(unittest.TestCase): + def test_fastnlp_10min_tutorial(self): + # 从csv读取数据到DataSet + sample_path = "tutorials/sample_data/tutorial_sample_dataset.csv" + dataset = DataSet.read_csv(sample_path, headers=('raw_sentence', 'label'), + sep='\t') + print(len(dataset)) + print(dataset[0]) + print(dataset[-3]) + + dataset.append(Instance(raw_sentence='fake data', label='0')) + # 将所有数字转为小写 + dataset.apply(lambda x: x['raw_sentence'].lower(), new_field_name='raw_sentence') + # label转int + dataset.apply(lambda x: int(x['label']), new_field_name='label') + + # 使用空格分割句子 + def split_sent(ins): + return ins['raw_sentence'].split() + + dataset.apply(split_sent, new_field_name='words') + + # 增加长度信息 + dataset.apply(lambda x: len(x['words']), new_field_name='seq_len') + print(len(dataset)) + print(dataset[0]) + + # DataSet.drop(func)筛除数据 + dataset.drop(lambda x: x['seq_len'] <= 3) + print(len(dataset)) + + # 设置DataSet中，哪些field要转为tensor + # set target，loss或evaluate中的golden，计算loss，模型评估时使用 + dataset.set_target("label") + # set input，模型forward时使用 + dataset.set_input("words", "seq_len") + + # 分出测试集、训练集 + test_data, train_data = dataset.split(0.5) + print(len(test_data)) + print(len(train_data)) + + # 构建词表, Vocabulary.add(word) + vocab = Vocabulary(min_freq=2) + train_data.apply(lambda x: [vocab.add(word) for word in x['words']]) + vocab.build_vocab() + + # index句子, Vocabulary.to_index(word) + train_data.apply(lambda x: [vocab.to_index(word) for word in x['words']], new_field_name='words') + test_data.apply(lambda x: [vocab.to_index(word) for word in x['words']], new_field_name='words') + print(test_data[0]) + + # 如果你们需要做强化学习或者GAN之类的项目，你们也可以使用这些数据预处理的工具 + from fastNLP.core.batch import Batch + from fastNLP.core.sampler import RandomSampler + + batch_iterator = Batch(dataset=train_data, batch_size=2, sampler=RandomSampler()) + for batch_x, batch_y in batch_iterator: + print("batch_x has: ", batch_x) + print("batch_y has: ", batch_y) + break + + from fastNLP.models import CNNText + model = CNNText(embed_num=len(vocab), embed_dim=50, num_classes=5, padding=2, dropout=0.1) + + from fastNLP import Trainer + from copy import deepcopy + + # 更改DataSet中对应field的名称，要以模型的forward等参数名一致 + train_data.rename_field('words', 'word_seq') # input field 与 forward 参数一致 + train_data.rename_field('label', 'label_seq') + test_data.rename_field('words', 'word_seq') + test_data.rename_field('label', 'label_seq') + + loss = CrossEntropyLoss(pred="output", target="label_seq") + metric = AccuracyMetric(pred="predict", target="label_seq") + + # 实例化Trainer，传入模型和数据，进行训练 + # 先在test_data拟合（确保模型的实现是正确的） + copy_model = deepcopy(model) + overfit_trainer = Trainer(model=copy_model, train_data=test_data, dev_data=test_data, + loss=loss, + metrics=metric, + save_path=None, + batch_size=32, + n_epochs=5) + overfit_trainer.train() + + # 用train_data训练，在test_data验证 + trainer = Trainer(model=model, train_data=train_data, dev_data=test_data, + loss=CrossEntropyLoss(pred="output", target="label_seq"), + metrics=AccuracyMetric(pred="predict", target="label_seq"), + save_path=None, + batch_size=32, + n_epochs=5) + trainer.train() + print('Train finished!') + + # 调用Tester在test_data上评价效果 + from fastNLP import Tester + + tester = Tester(data=test_data, model=model, metrics=AccuracyMetric(pred="predict", target="label_seq"), + batch_size=4) + acc = tester.test() + print(acc) + + def test_fastnlp_1min_tutorial(self): + # tutorials/fastnlp_1min_tutorial.ipynb + data_path = "tutorials/sample_data/tutorial_sample_dataset.csv" + ds = DataSet.read_csv(data_path, headers=('raw_sentence', 'label'), sep='\t') + print(ds[1]) + + # 将所有数字转为小写 + ds.apply(lambda x: x['raw_sentence'].lower(), new_field_name='raw_sentence') + # label转int + ds.apply(lambda x: int(x['label']), new_field_name='label_seq', is_target=True) + + def split_sent(ins): + return ins['raw_sentence'].split() + + ds.apply(split_sent, new_field_name='words', is_input=True) + + # 分割训练集/验证集 + train_data, dev_data = ds.split(0.3) + print("Train size: ", len(train_data)) + print("Test size: ", len(dev_data)) + + from fastNLP import Vocabulary + vocab = Vocabulary(min_freq=2) + train_data.apply(lambda x: [vocab.add(word) for word in x['words']]) + + # index句子, Vocabulary.to_index(word) + train_data.apply(lambda x: [vocab.to_index(word) for word in x['words']], new_field_name='word_seq', + is_input=True) + dev_data.apply(lambda x: [vocab.to_index(word) for word in x['words']], new_field_name='word_seq', + is_input=True) + + from fastNLP.models import CNNText + model = CNNText(embed_num=len(vocab), embed_dim=50, num_classes=5, padding=2, dropout=0.1) + + from fastNLP import Trainer, CrossEntropyLoss, AccuracyMetric + trainer = Trainer(model=model, + train_data=train_data, + dev_data=dev_data, + loss=CrossEntropyLoss(), + metrics=AccuracyMetric() + ) + trainer.train() + print('Train finished!') + + def test_fastnlp_advanced_tutorial(self): + import os + os.chdir("tutorials/fastnlp_advanced_tutorial") + + from fastNLP import DataSet + from fastNLP import Instance + from fastNLP import Vocabulary + from fastNLP import Trainer + from fastNLP import Tester + + # ### Instance + # Instance表示一个样本，由一个或者多个field（域、属性、特征）组成，每个field具有自己的名字以及值 + # 在初始化Instance的时候可以定义它包含的field，使用"field_name=field_value"的写法 + + # In[2]: + + # 组织一个Instance，这个Instance由premise、hypothesis、label三个field组成 + instance = Instance(premise='an premise example .', hypothesis='an hypothesis example.', label=1) + instance + + # In[3]: + + data_set = DataSet([instance] * 5) + data_set.append(instance) + data_set[-2:] + + # In[4]: + + # 如果某一个field的类型与dataset对应的field类型不一样仍可被加入dataset中 + instance2 = Instance(premise='the second premise example .', hypothesis='the second hypothesis example.', + label='1') + try: + data_set.append(instance2) + except: + pass + data_set[-2:] + + # In[5]: + + # 如果某一个field的名字不对，则该instance不能被append到dataset中 + instance3 = Instance(premises='the third premise example .', hypothesis='the third hypothesis example.', + label=1) + try: + data_set.append(instance3) + except: + print('cannot append instance') + pass + data_set[-2:] + + # In[6]: + + # 除了文本以外，还可以将tensor作为其中一个field的value + import torch + tensor_ins = Instance(image=torch.randn(5, 5), label=0) + ds = DataSet() + ds.append(tensor_ins) + ds + + from fastNLP import DataSet + from fastNLP import Instance + + # 从csv读取数据到DataSet + # 类csv文件，即每一行为一个example的文件，都可以使用这种方法进行数据读取 + dataset = DataSet.read_csv('tutorial_sample_dataset.csv', headers=('raw_sentence', 'label'), sep='\t') + # 查看DataSet的大小 + len(dataset) + + # In[8]: + + # 使用数字索引[k]，获取第k个样本 + dataset[0] + + # In[9]: + + # 获取的样本是一个Instance + type(dataset[0]) + + # In[10]: + + # 使用数字索引[a: b]，获取第a到第b个样本 + dataset[0: 3] + + # In[11]: + + # 索引也可以是负数 + dataset[-1] + + data_path = ['premise', 'hypothesis', 'label'] + + # 读入文件 + with open(data_path[0]) as f: + premise = f.readlines() + + with open(data_path[1]) as f: + hypothesis = f.readlines() + + with open(data_path[2]) as f: + label = f.readlines() + + assert len(premise) == len(hypothesis) and len(hypothesis) == len(label) + + # 组织DataSet + data_set = DataSet() + for p, h, l in zip(premise, hypothesis, label): + p = p.strip() # 将行末空格去除 + h = h.strip() # 将行末空格去除 + data_set.append(Instance(premise=p, hypothesis=h, truth=l)) + + data_set[0] + + # ### DataSet的其他操作 + # 在构建完毕DataSet后，仍然可以对DataSet的内容进行操作，函数接口为DataSet.apply() + + # In[13]: + + # 将premise域的所有文本转成小写 + data_set.apply(lambda x: x['premise'].lower(), new_field_name='premise') + data_set[-2:] + + # In[14]: + + # label转int + data_set.apply(lambda x: int(x['truth']), new_field_name='truth') + data_set[-2:] + + # In[15]: + + # 使用空格分割句子 + def split_sent(ins): + return ins['premise'].split() + + data_set.apply(split_sent, new_field_name='premise') + data_set.apply(lambda x: x['hypothesis'].split(), new_field_name='hypothesis') + data_set[-2:] + + # In[16]: + + # 筛选数据 + origin_data_set_len = len(data_set) + data_set.drop(lambda x: len(x['premise']) <= 6) + origin_data_set_len, len(data_set) + + # In[17]: + + # 增加长度信息 + data_set.apply(lambda x: [1] * len(x['premise']), new_field_name='premise_len') + data_set.apply(lambda x: [1] * len(x['hypothesis']), new_field_name='hypothesis_len') + data_set[-1] + + # In[18]: + + # 设定特征域、标签域 + data_set.set_input("premise", "premise_len", "hypothesis", "hypothesis_len") + data_set.set_target("truth") + + # In[19]: + + # 重命名field + data_set.rename_field('truth', 'label') + data_set[-1] + + # In[20]: + + # 切分训练、验证集、测试集 + train_data, vad_data = data_set.split(0.5) + dev_data, test_data = vad_data.split(0.4) + len(train_data), len(dev_data), len(test_data) + + # In[21]: + + # 深拷贝一个数据集 + import copy + train_data_2, dev_data_2 = copy.deepcopy(train_data), copy.deepcopy(dev_data) + del copy + + # 初始化词表，该词表最大的vocab_size为10000，词表中每个词出现的最低频率为2，''表示未知词语，''表示padding词语 + # Vocabulary默认初始化参数为max_size=None, min_freq=None, unknown='', padding='' + vocab = Vocabulary(max_size=10000, min_freq=2, unknown='', padding='') + + # 构建词表 + train_data.apply(lambda x: [vocab.add(word) for word in x['premise']]) + train_data.apply(lambda x: [vocab.add(word) for word in x['hypothesis']]) + vocab.build_vocab() + + # In[23]: + + # 根据词表index句子 + train_data.apply(lambda x: [vocab.to_index(word) for word in x['premise']], new_field_name='premise') + train_data.apply(lambda x: [vocab.to_index(word) for word in x['hypothesis']], new_field_name='hypothesis') + dev_data.apply(lambda x: [vocab.to_index(word) for word in x['premise']], new_field_name='premise') + dev_data.apply(lambda x: [vocab.to_index(word) for word in x['hypothesis']], new_field_name='hypothesis') + test_data.apply(lambda x: [vocab.to_index(word) for word in x['premise']], new_field_name='premise') + test_data.apply(lambda x: [vocab.to_index(word) for word in x['hypothesis']], new_field_name='hypothesis') + train_data[-1], dev_data[-1], test_data[-1] + + # 读入vocab文件 + with open('vocab.txt') as f: + lines = f.readlines() + vocabs = [] + for line in lines: + vocabs.append(line.strip()) + + # 实例化Vocabulary + vocab_bert = Vocabulary(unknown=None, padding=None) + # 将vocabs列表加入Vocabulary + vocab_bert.add_word_lst(vocabs) + # 构建词表 + vocab_bert.build_vocab() + # 更新unknown与padding的token文本 + vocab_bert.unknown = '[UNK]' + vocab_bert.padding = '[PAD]' + + # In[25]: + + # 根据词表index句子 + train_data_2.apply(lambda x: [vocab_bert.to_index(word) for word in x['premise']], new_field_name='premise') + train_data_2.apply(lambda x: [vocab_bert.to_index(word) for word in x['hypothesis']], + new_field_name='hypothesis') + dev_data_2.apply(lambda x: [vocab_bert.to_index(word) for word in x['premise']], new_field_name='premise') + dev_data_2.apply(lambda x: [vocab_bert.to_index(word) for word in x['hypothesis']], new_field_name='hypothesis') + train_data_2[-1], dev_data_2[-1] + + # step 1：加载模型参数（非必选） + from fastNLP.io.config_io import ConfigSection, ConfigLoader + args = ConfigSection() + ConfigLoader().load_config("./data/config", {"esim_model": args}) + args["vocab_size"] = len(vocab) + args.data + + # In[27]: + + # step 2：加载ESIM模型 + from fastNLP.models import ESIM + model = ESIM(**args.data) + model + + # In[28]: + + # 另一个例子：加载CNN文本分类模型 + from fastNLP.models import CNNText + cnn_text_model = CNNText(embed_num=len(vocab), embed_dim=50, num_classes=5, padding=2, dropout=0.1) + cnn_text_model + + from fastNLP import CrossEntropyLoss + from fastNLP import Adam + from fastNLP import AccuracyMetric + trainer = Trainer( + train_data=train_data, + model=model, + loss=CrossEntropyLoss(pred='pred', target='label'), + metrics=AccuracyMetric(), + n_epochs=5, + batch_size=16, + print_every=-1, + validate_every=-1, + dev_data=dev_data, + use_cuda=True, + optimizer=Adam(lr=1e-3, weight_decay=0), + check_code_level=-1, + metric_key='acc', + use_tqdm=False, + ) + trainer.train() + + tester = Tester( + data=test_data, + model=model, + metrics=AccuracyMetric(), + batch_size=args["batch_size"], + ) + tester.test() + + os.chdir("../..") From b93ca9bb3059b8c82a7a5a7ae71c2d51ec006dee Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Thu, 17 Jan 2019 15:39:13 +0800 Subject: [PATCH 10/32] =?UTF-8?q?*=20FieldArray=E6=B7=BB=E5=8A=A0=E5=AF=B9?= =?UTF-8?q?list=20of=20np.array=E7=9A=84=E6=94=AF=E6=8C=81=20*=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E6=B5=8B=E8=AF=95=EF=BC=9AFieldArray=E7=9A=84?= =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/fieldarray.py | 8 +++- reproduction/POS_tagging/train_pos_tag.py | 1 + test/core/test_fieldarray.py | 53 ++++++++++++++++++++++- test/test_tutorials.py | 4 +- 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/fastNLP/core/fieldarray.py b/fastNLP/core/fieldarray.py index 4cde86ab..20d7e5e0 100644 --- a/fastNLP/core/fieldarray.py +++ b/fastNLP/core/fieldarray.py @@ -112,13 +112,17 @@ def __init__(self, name, content, is_target=None, is_input=None, padder=AutoPadd 2.3) 二维list DataSet([Instance(x=[[1, 2], [3, 4]])]) 2.4) 二维array DataSet([Instance(x=np.array([[1, 2], [3, 4]]))]) - 注意：np.array必须仅在最外层，即np.array([np.array, np.array]) 和 list of np.array不考虑 类型检查(dtype check)发生在当该field被设置为is_input或者is_target时。 """ self.name = name if isinstance(content, list): - content = content + # 如果DataSet使用dict初始化, content 可能是二维list/二维array/三维list + # 如果DataSet使用list of Instance 初始化, content可能是 [list]/[array]/[2D list] + if len(content) == 1 and isinstance(content[0], np.ndarray): + # 这是使用list of Instance 初始化时第一个样本：FieldArray(name, [field]) + # 将[np.array] 转化为 list of list + content[0] = content[0].tolist() elif isinstance(content, np.ndarray): content = content.tolist() # convert np.ndarray into 2-D list else: diff --git a/reproduction/POS_tagging/train_pos_tag.py b/reproduction/POS_tagging/train_pos_tag.py index e817db44..4bdc23c7 100644 --- a/reproduction/POS_tagging/train_pos_tag.py +++ b/reproduction/POS_tagging/train_pos_tag.py @@ -144,6 +144,7 @@ def run_test(test_path): parser.add_argument("--train", type=str, help="training conll file", default="/home/zyfeng/data/sample.conllx") parser.add_argument("--dev", type=str, help="dev conll file", default="/home/zyfeng/data/sample.conllx") parser.add_argument("--test", type=str, help="test conll file", default=None) + parser.add_argument("--save", type=str, help="path to save", default=None) parser.add_argument("-c", "--restart", action="store_true", help="whether to continue training") parser.add_argument("-cp", "--checkpoint", type=str, help="checkpoint of the trained model") diff --git a/test/core/test_fieldarray.py b/test/core/test_fieldarray.py index da287916..834545c0 100644 --- a/test/core/test_fieldarray.py +++ b/test/core/test_fieldarray.py @@ -5,8 +5,59 @@ from fastNLP.core.fieldarray import FieldArray +class TestFieldArrayInit(unittest.TestCase): + """ + 1） 如果DataSet使用dict初始化，那么在add_field中会构造FieldArray： + 1.1) 二维list DataSet({"x": [[1, 2], [3, 4]]}) + 1.2) 二维array DataSet({"x": np.array([[1, 2], [3, 4]])}) + 1.3) 三维list DataSet({"x": [[[1, 2], [3, 4]], [[1, 2], [3, 4]]]}) + 2） 如果DataSet使用list of Instance 初始化,那么在append中会先对第一个样本初始化FieldArray； + 然后后面的样本使用FieldArray.append进行添加。 + 2.1) 一维list DataSet([Instance(x=[1, 2, 3, 4])]) + 2.2) 一维array DataSet([Instance(x=np.array([1, 2, 3, 4]))]) + 2.3) 二维list DataSet([Instance(x=[[1, 2], [3, 4]])]) + 2.4) 二维array DataSet([Instance(x=np.array([[1, 2], [3, 4]]))]) + """ + + def test_init_v1(self): + # 二维list + fa = FieldArray("x", [[1, 2], [3, 4]] * 5, is_input=True) + + def test_init_v2(self): + # 二维array + fa = FieldArray("x", np.array([[1, 2], [3, 4]] * 5), is_input=True) + + def test_init_v3(self): + # 三维list + fa = FieldArray("x", [[[1, 2], [3, 4]], [[1, 2], [3, 4]]], is_input=True) + + def test_init_v4(self): + # 一维list + val = [1, 2, 3, 4] + fa = FieldArray("x", [val], is_input=True) + fa.append(val) + + def test_init_v5(self): + # 一维array + val = np.array([1, 2, 3, 4]) + fa = FieldArray("x", [val], is_input=True) + fa.append(val) + + def test_init_v6(self): + # 二维array + val = [[1, 2], [3, 4]] + fa = FieldArray("x", [val], is_input=True) + fa.append(val) + + def test_init_v7(self): + # 二维list + val = np.array([[1, 2], [3, 4]]) + fa = FieldArray("x", [val], is_input=True) + fa.append(val) + + class TestFieldArray(unittest.TestCase): - def test(self): + def test_main(self): fa = FieldArray("x", [1, 2, 3, 4, 5], is_input=True) self.assertEqual(len(fa), 5) fa.append(6) diff --git a/test/test_tutorials.py b/test/test_tutorials.py index ee48c23b..68c874fa 100644 --- a/test/test_tutorials.py +++ b/test/test_tutorials.py @@ -408,12 +408,12 @@ def split_sent(ins): model=model, loss=CrossEntropyLoss(pred='pred', target='label'), metrics=AccuracyMetric(), - n_epochs=5, + n_epochs=3, batch_size=16, print_every=-1, validate_every=-1, dev_data=dev_data, - use_cuda=True, + use_cuda=False, optimizer=Adam(lr=1e-3, weight_decay=0), check_code_level=-1, metric_key='acc', From 864c2238f895ccc605365cb958473c6b17e0bd45 Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Thu, 17 Jan 2019 22:42:40 +0800 Subject: [PATCH 11/32] =?UTF-8?q?=E6=B7=BB=E5=8A=A0FieldArray=E5=AF=B9list?= =?UTF-8?q?=20of=20np.array=E7=9A=84=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/fieldarray.py | 7 +++++-- reproduction/POS_tagging/train_pos_tag.py | 6 +++--- test/core/test_fieldarray.py | 6 ++++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/fastNLP/core/fieldarray.py b/fastNLP/core/fieldarray.py index 20d7e5e0..96854e72 100644 --- a/fastNLP/core/fieldarray.py +++ b/fastNLP/core/fieldarray.py @@ -105,6 +105,7 @@ def __init__(self, name, content, is_target=None, is_input=None, padder=AutoPadd 1.1) 二维list DataSet({"x": [[1, 2], [3, 4]]}) 1.2) 二维array DataSet({"x": np.array([[1, 2], [3, 4]])}) 1.3) 三维list DataSet({"x": [[[1, 2], [3, 4]], [[1, 2], [3, 4]]]}) + 1.4) list of array: DataSet({"x": [np.array([1,2,3]), np.array([1,2,3])]}) 2） 如果DataSet使用list of Instance 初始化,那么在append中会先对第一个样本初始化FieldArray； 然后后面的样本使用FieldArray.append进行添加。 2.1) 一维list DataSet([Instance(x=[1, 2, 3, 4])]) @@ -119,10 +120,12 @@ def __init__(self, name, content, is_target=None, is_input=None, padder=AutoPadd if isinstance(content, list): # 如果DataSet使用dict初始化, content 可能是二维list/二维array/三维list # 如果DataSet使用list of Instance 初始化, content可能是 [list]/[array]/[2D list] - if len(content) == 1 and isinstance(content[0], np.ndarray): + for idx, item in enumerate(content): # 这是使用list of Instance 初始化时第一个样本：FieldArray(name, [field]) # 将[np.array] 转化为 list of list - content[0] = content[0].tolist() + # 也可以支持[array, array, array]的情况 + if isinstance(item, np.ndarray): + content[idx] = content[idx].tolist() elif isinstance(content, np.ndarray): content = content.tolist() # convert np.ndarray into 2-D list else: diff --git a/reproduction/POS_tagging/train_pos_tag.py b/reproduction/POS_tagging/train_pos_tag.py index 4bdc23c7..6448c32b 100644 --- a/reproduction/POS_tagging/train_pos_tag.py +++ b/reproduction/POS_tagging/train_pos_tag.py @@ -93,7 +93,7 @@ def train(train_data_path, dev_data_path, checkpoint=None): target="truth", seq_lens="word_seq_origin_len"), dev_data=dev_data, metric_key="f", - use_tqdm=True, use_cuda=True, print_every=5, n_epochs=6, save_path="./save_0") + use_tqdm=True, use_cuda=True, print_every=10, n_epochs=20, save_path="./save_0117") trainer.train(load_best_model=True) # save model & pipeline @@ -102,14 +102,14 @@ def train(train_data_path, dev_data_path, checkpoint=None): pp = Pipeline([vocab_proc, seq_len_proc, set_input_proc, model_proc, id2tag]) save_dict = {"pipeline": pp, "model": model, "tag_vocab": tag_proc.vocab} - torch.save(save_dict, "model_pp.pkl") + torch.save(save_dict, "model_pp_0117.pkl") print("pipeline saved") def run_test(test_path): test_data = ZhConllPOSReader().load(test_path) - with open("model_pp.pkl", "rb") as f: + with open("model_pp_0117.pkl", "rb") as f: save_dict = torch.load(f) tag_vocab = save_dict["tag_vocab"] pipeline = save_dict["pipeline"] diff --git a/test/core/test_fieldarray.py b/test/core/test_fieldarray.py index 834545c0..151d9335 100644 --- a/test/core/test_fieldarray.py +++ b/test/core/test_fieldarray.py @@ -31,6 +31,12 @@ def test_init_v3(self): # 三维list fa = FieldArray("x", [[[1, 2], [3, 4]], [[1, 2], [3, 4]]], is_input=True) + def test_init_v7(self): + # list of array + fa = FieldArray("x", [np.array([[1, 2], [3, 4]]), np.array([[1, 2], [3, 4]])], is_input=True) + self.assertEqual(fa.pytype, int) + self.assertEqual(fa.dtype, np.int) + def test_init_v4(self): # 一维list val = [1, 2, 3, 4] From 2e3ef52a7d47598e92707f0ee5c3251eb68bcb95 Mon Sep 17 00:00:00 2001 From: yh_cc Date: Fri, 18 Jan 2019 23:02:15 +0800 Subject: [PATCH 12/32] =?UTF-8?q?=E5=B0=86batch=E5=A2=9E=E5=BC=BA=E4=B8=BA?= =?UTF-8?q?=E5=A4=9A=E8=BF=9B=E7=A8=8Bbatch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/batch.py | 459 +++++++++++++++++++++++++++++++++---- fastNLP/core/fieldarray.py | 2 +- fastNLP/core/trainer.py | 25 +- fastNLP/core/utils.py | 5 +- test/core/test_batch.py | 72 ++++++ 5 files changed, 510 insertions(+), 53 deletions(-) diff --git a/fastNLP/core/batch.py b/fastNLP/core/batch.py index d4fcbf23..05bd5665 100644 --- a/fastNLP/core/batch.py +++ b/fastNLP/core/batch.py @@ -1,63 +1,59 @@ import numpy as np +import random import torch +import torch.multiprocessing as multiprocessing +from torch.utils.data.dataloader import _set_worker_signal_handlers, _update_worker_pids, \ + _remove_worker_pids, _error_if_any_worker_fails +import signal +import sys +import threading +import traceback +import os +from torch._six import FileNotFoundError from fastNLP.core.sampler import RandomSampler - class Batch(object): - """Batch is an iterable object which iterates over mini-batches. + def __init__(self, dataset, batch_size, sampler=RandomSampler(), as_numpy=False, num_workers=0, pin_memory=False, + timeout=0.0): + """ + Batch is an iterable object which iterates over mini-batches. - Example:: + Example:: - for batch_x, batch_y in Batch(data_set, batch_size=16, sampler=SequentialSampler()): - # ... + for batch_x, batch_y in Batch(data_set, batch_size=16, sampler=SequentialSampler()): + # ... - :param DataSet dataset: a DataSet object - :param int batch_size: the size of the batch - :param Sampler sampler: a Sampler object - :param bool as_numpy: If True, return Numpy array. Otherwise, return torch tensors. + :param DataSet dataset: a DataSet object + :param int batch_size: the size of the batch + :param Sampler sampler: a Sampler object + :param bool as_numpy: If True, return Numpy array when possible. Otherwise, return torch tensors. + :param num_workers: int, 使用多少个进程来准备数据。默认为0, 即使用主线程生成数据。 特性处于实验阶段，谨慎使用。 + 如果DataSet较大，且每个batch的准备时间很短，使用多进程可能并不能提速。 + :param pin_memory: bool, 默认为False. 设置为True时，有可能可以节省tensor从cpu移动到gpu的阻塞时间。 + :param timeout: float, 大于0的数，只有在num_workers>0时才有用。超过该时间仍然没有获取到一个batch则报错，可以用于 + 检测是否出现了batch产生阻塞的情况。 + """ - """ + if num_workers < 0: + raise ValueError('num_workers option cannot be negative; ' + 'use num_workers=0 to disable multiprocessing.') + if timeout < 0: + raise ValueError('timeout option should be non-negative') - def __init__(self, dataset, batch_size, sampler=RandomSampler(), as_numpy=False): self.dataset = dataset self.batch_size = batch_size self.sampler = sampler + self.num_workers = num_workers + self.pin_memory = pin_memory + self.timeout = timeout self.as_numpy = as_numpy - self.idx_list = None - self.curidx = 0 self.num_batches = len(dataset) // batch_size + int(len(dataset) % batch_size != 0) self.cur_batch_indices = None def __iter__(self): - self.idx_list = self.sampler(self.dataset) - self.curidx = 0 - self.lengths = self.dataset.get_length() - return self - - def __next__(self): - if self.curidx >= len(self.idx_list): - raise StopIteration - else: - endidx = min(self.curidx + self.batch_size, len(self.idx_list)) - batch_x, batch_y = {}, {} - - indices = self.idx_list[self.curidx:endidx] - self.cur_batch_indices = indices - - for field_name, field in self.dataset.get_all_fields().items(): - if field.is_target or field.is_input: - batch = field.get(indices) - if not self.as_numpy and field.padder is not None: - batch = to_tensor(batch, field.dtype) - if field.is_target: - batch_y[field_name] = batch - if field.is_input: - batch_x[field_name] = batch - - self.curidx = endidx - - return batch_x, batch_y + # TODO 现在多线程的情况下每个循环都会重新创建多进程，开销可能有点大。可以考虑直接复用iterator. + return _DataLoaderIter(self) def __len__(self): return self.num_batches @@ -65,7 +61,6 @@ def __len__(self): def get_batch_indices(self): return self.cur_batch_indices - def to_tensor(batch, dtype): try: if dtype in (int, np.int8, np.int16, np.int32, np.int64): @@ -75,3 +70,383 @@ def to_tensor(batch, dtype): except: pass return batch + + +""" +由于多进程涉及到大量问题，包括系统、安全关闭进程等。所以这里直接从pytorch的官方版本修改DataLoader实现多进程加速 +""" + +IS_WINDOWS = sys.platform == "win32" +if IS_WINDOWS: + import ctypes + from ctypes.wintypes import DWORD, BOOL, HANDLE + +if sys.version_info[0] == 2: + import Queue as queue +else: + import queue + + +class ExceptionWrapper(object): + r"""Wraps an exception plus traceback to communicate across threads""" + + def __init__(self, exc_info): + self.exc_type = exc_info[0] + self.exc_msg = "".join(traceback.format_exception(*exc_info)) + + +_use_shared_memory = False +r"""Whether to use shared memory in default_collate""" + +MANAGER_STATUS_CHECK_INTERVAL = 5.0 + +if IS_WINDOWS: + # On Windows, the parent ID of the worker process remains unchanged when the manager process + # is gone, and the only way to check it through OS is to let the worker have a process handle + # of the manager and ask if the process status has changed. + class ManagerWatchdog(object): + def __init__(self): + self.manager_pid = os.getppid() + + self.kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + self.kernel32.OpenProcess.argtypes = (DWORD, BOOL, DWORD) + self.kernel32.OpenProcess.restype = HANDLE + self.kernel32.WaitForSingleObject.argtypes = (HANDLE, DWORD) + self.kernel32.WaitForSingleObject.restype = DWORD + + # Value obtained from https://msdn.microsoft.com/en-us/library/ms684880.aspx + SYNCHRONIZE = 0x00100000 + self.manager_handle = self.kernel32.OpenProcess(SYNCHRONIZE, 0, self.manager_pid) + + if not self.manager_handle: + raise ctypes.WinError(ctypes.get_last_error()) + + def is_alive(self): + # Value obtained from https://msdn.microsoft.com/en-us/library/windows/desktop/ms687032.aspx + return self.kernel32.WaitForSingleObject(self.manager_handle, 0) != 0 +else: + class ManagerWatchdog(object): + def __init__(self): + self.manager_pid = os.getppid() + + def is_alive(self): + return os.getppid() == self.manager_pid + + +def _worker_loop(dataset, index_queue, data_queue, seed, worker_id, as_numpy): + # 产生数据的循环 + global _use_shared_memory + _use_shared_memory = True + + # Intialize C side signal handlers for SIGBUS and SIGSEGV. Python signal + # module's handlers are executed after Python returns from C low-level + # handlers, likely when the same fatal signal happened again already. + # https://docs.python.org/3/library/signal.html Sec. 18.8.1.1 + _set_worker_signal_handlers() + + torch.set_num_threads(1) + random.seed(seed) + torch.manual_seed(seed) + + watchdog = ManagerWatchdog() + + while True: + try: + # 获取当前batch计数，当前batch的indexes + r = index_queue.get(timeout=MANAGER_STATUS_CHECK_INTERVAL) + except queue.Empty: + if watchdog.is_alive(): + continue + else: + break + if r is None: + break + idx, batch_indices = r + try: + # 获取相应的batch数据。这里需要修改为从dataset中取出数据并且完成padding + samples = _get_batch_from_dataset(dataset, batch_indices, as_numpy) + except Exception: + data_queue.put((idx, ExceptionWrapper(sys.exc_info()), batch_indices)) + else: + data_queue.put((idx, samples, batch_indices)) + del samples + +def _get_batch_from_dataset(dataset, indices, as_numpy): + """ + 给定indices，从DataSet中取出(batch_x, batch_y). 数据从这里产生后，若没有pin_memory, 则直接传递给Trainer了，如果存在 + pin_memory还会经过一道pin_memory()的处理 + :param dataset: fastNLP.DataSet对象 + :param indices: List[int], index + :param as_numpy: bool, 是否只是转换为numpy + :return: (batch_x, batch_y) + """ + batch_x, batch_y = {}, {} + for field_name, field in dataset.get_all_fields().items(): + if field.is_target or field.is_input: + batch = field.get(indices) + if not as_numpy and field.padder is not None: + batch = to_tensor(batch, field.dtype) + if field.is_target: + batch_y[field_name] = batch + if field.is_input: + batch_x[field_name] = batch + + return batch_x, batch_y + + +def _worker_manager_loop(in_queue, out_queue, done_event, pin_memory, device_id): + # 将数据送入到指定的query中. 即如果需要pin_memory, 则 + if pin_memory: + torch.cuda.set_device(device_id) + + while True: + try: + r = in_queue.get() + except Exception: + if done_event.is_set(): + return + raise + if r is None: + break + if isinstance(r[1], ExceptionWrapper): + out_queue.put(r) + continue + idx, batch, batch_indices = r + try: + if pin_memory: + batch = pin_memory_batch(batch) + except Exception: + out_queue.put((idx, ExceptionWrapper(sys.exc_info()), batch_indices)) + else: + out_queue.put((idx, batch, batch_indices)) + + +def pin_memory_batch(batchs): + """ + + :param batchs: (batch_x, batch_y) + :return: (batch_x, batch_y) + """ + for batch_dict in batchs: + for field_name, batch in batch_dict.items(): + if isinstance(batch, torch.Tensor): + batch_dict[field_name] = batch.pin_memory() + return batchs + + +_SIGCHLD_handler_set = False +r"""Whether SIGCHLD handler is set for DataLoader worker failures. Only one +handler needs to be set for all DataLoaders in a process.""" + + +def _set_SIGCHLD_handler(): + # Windows doesn't support SIGCHLD handler + if sys.platform == 'win32': + return + # can't set signal in child threads + if not isinstance(threading.current_thread(), threading._MainThread): + return + global _SIGCHLD_handler_set + if _SIGCHLD_handler_set: + return + previous_handler = signal.getsignal(signal.SIGCHLD) + if not callable(previous_handler): + previous_handler = None + + def handler(signum, frame): + # This following call uses `waitid` with WNOHANG from C side. Therefore, + # Python can still get and update the process status successfully. + _error_if_any_worker_fails() + if previous_handler is not None: + previous_handler(signum, frame) + + signal.signal(signal.SIGCHLD, handler) + _SIGCHLD_handler_set = True + + +class _DataLoaderIter(object): + r"""Iterates once over the DataLoader's dataset, as specified by the sampler""" + + def __init__(self, batcher): + self.batcher = batcher + self.dataset = batcher.dataset + self.sampler = batcher.sampler + self.as_numpy = batcher.as_numpy + self.batch_size = batcher.batch_size + self.num_workers = batcher.num_workers + self.pin_memory = batcher.pin_memory and torch.cuda.is_available() + self.timeout = batcher.timeout + self.done_event = threading.Event() + self.curidx = 0 + self.idx_list = self.sampler(self.dataset) + + # self.sample_iter一次返回一个index. 可以通过其他方式替代 + + base_seed = torch.LongTensor(1).random_().item() + + if self.num_workers > 0: + # 每个worker建立一个index queue + self.index_queues = [multiprocessing.Queue() for _ in range(self.num_workers)] + self.worker_queue_idx = 0 + # 存放获取到的batch + self.worker_result_queue = multiprocessing.SimpleQueue() + self.batches_outstanding = 0 + self.worker_pids_set = False + self.shutdown = False + self.send_idx = 0 + self.rcvd_idx = 0 + self.reorder_dict = {} + + # 这里会将batch的数据输送到self.worker_result_queue中，但是还没有送入到device中 + self.workers = [ + multiprocessing.Process( + target=_worker_loop, + args=(self.dataset, self.index_queues[i], + self.worker_result_queue, base_seed + i, i, self.as_numpy)) + for i in range(self.num_workers)] + + # self.data_queue取数据就行。如果有pin_memory的话，会把数据放到另一个queue + if self.pin_memory or self.timeout > 0: + self.data_queue = queue.Queue() + if self.pin_memory: + maybe_device_id = torch.cuda.current_device() + else: + # do not initialize cuda context if not necessary + maybe_device_id = None + self.worker_manager_thread = threading.Thread( + target=_worker_manager_loop, + args=(self.worker_result_queue, self.data_queue, self.done_event, self.pin_memory, + maybe_device_id)) + self.worker_manager_thread.daemon = True + self.worker_manager_thread.start() + else: + self.data_queue = self.worker_result_queue + + # worker们开始工作 + for w in self.workers: + w.daemon = True # ensure that the worker exits on process exit + w.start() + + _update_worker_pids(id(self), tuple(w.pid for w in self.workers)) + _set_SIGCHLD_handler() + self.worker_pids_set = True + + # prime the prefetch loop + for _ in range(2 * self.num_workers): + self._put_indices() + + def _get_batch(self): + if self.timeout > 0: + try: + return self.data_queue.get(timeout=self.timeout) + except queue.Empty: + raise RuntimeError('DataLoader timed out after {} seconds'.format(self.timeout)) + else: + return self.data_queue.get() + + def __next__(self): + if self.num_workers == 0: # same-process loading + if self.curidx >= len(self.idx_list): + raise StopIteration + endidx = min(self.curidx + self.batch_size, len(self.idx_list)) + # 直接从数据集中采集数据即可 + indices = self.idx_list[self.curidx:endidx] + self.batcher.cur_batch_indices = indices + batch_x, batch_y = _get_batch_from_dataset(dataset=self.dataset, indices=indices, + as_numpy=self.as_numpy) + if self.pin_memory: + batch_x, batch_y = pin_memory_batch((batch_x, batch_y)) + self.curidx = endidx + return batch_x, batch_y + + # check if the next sample has already been generated + if self.rcvd_idx in self.reorder_dict: + batch = self.reorder_dict.pop(self.rcvd_idx) + return self._process_next_batch(batch) + + # 如果生成的数据为0了，则停止 + if self.batches_outstanding == 0: + self._shutdown_workers() + raise StopIteration + + while True: + assert (not self.shutdown and self.batches_outstanding > 0) + idx, batch, batch_indices = self._get_batch() + self.batches_outstanding -= 1 + if idx != self.rcvd_idx: + # store out-of-order samples + self.reorder_dict[idx] = batch + continue + self.batcher.cur_batch_indices = batch_indices + return self._process_next_batch(batch) + + def __iter__(self): + self.curidx = 0 + + return self + + def _put_indices(self): + # 向采集数据的index queue中放入index + assert self.batches_outstanding < 2 * self.num_workers + if self.curidx >= len(self.idx_list): + indices = None + else: + endidx = min(self.curidx + self.batch_size, len(self.idx_list)) + # 直接从数据集中采集数据即可 + indices = self.idx_list[self.curidx:endidx] + if indices is None: + return + self.index_queues[self.worker_queue_idx].put((self.send_idx, indices)) + self.curidx = endidx + self.worker_queue_idx = (self.worker_queue_idx + 1) % self.num_workers + self.batches_outstanding += 1 + self.send_idx += 1 + + def _process_next_batch(self, batch): + # 只是提醒生成下一个batch indice数据 + self.rcvd_idx += 1 + self._put_indices() + if isinstance(batch, ExceptionWrapper): + raise batch.exc_type(batch.exc_msg) + return batch + + def __getstate__(self): + # TODO: add limited pickling support for sharing an iterator + # across multiple threads for HOGWILD. + # Probably the best way to do this is by moving the sample pushing + # to a separate thread and then just sharing the data queue + # but signalling the end is tricky without a non-blocking API + raise NotImplementedError("_DataLoaderIter cannot be pickled") + + def _shutdown_workers(self): + try: + if not self.shutdown: + self.shutdown = True + self.done_event.set() + for q in self.index_queues: + q.put(None) + # if some workers are waiting to put, make place for them + try: + while not self.worker_result_queue.empty(): + self.worker_result_queue.get() + except (FileNotFoundError, ImportError): + # Many weird errors can happen here due to Python + # shutting down. These are more like obscure Python bugs. + # FileNotFoundError can happen when we rebuild the fd + # fetched from the queue but the socket is already closed + # from the worker side. + # ImportError can happen when the unpickler loads the + # resource from `get`. + pass + # done_event should be sufficient to exit worker_manager_thread, + # but be safe here and put another None + self.worker_result_queue.put(None) + finally: + # removes pids no matter what + if self.worker_pids_set: + _remove_worker_pids(id(self)) + self.worker_pids_set = False + + def __del__(self): + if self.num_workers > 0: + self._shutdown_workers() diff --git a/fastNLP/core/fieldarray.py b/fastNLP/core/fieldarray.py index 96854e72..f3fcb3c8 100644 --- a/fastNLP/core/fieldarray.py +++ b/fastNLP/core/fieldarray.py @@ -408,7 +408,7 @@ def _exactly_three_dims(self, contents, field_name): except: raise ValueError("Field:{} only has one dimension.".format(field_name)) try: - value = value[1] + value = value[0] except: raise ValueError("Field:{} only has two dimensions.".format(field_name)) diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index fcafeb32..76a8562b 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -34,8 +34,8 @@ class Trainer(object): def __init__(self, train_data, model, loss=None, metrics=None, n_epochs=3, batch_size=32, print_every=50, validate_every=-1, dev_data=None, save_path=None, optimizer=Adam(lr=0.01, weight_decay=0), - check_code_level=0, metric_key=None, sampler=RandomSampler(), use_tqdm=True, use_cuda=False, - callbacks=None): + check_code_level=0, metric_key=None, sampler=RandomSampler(), num_workers=0, pin_memory=False, + timeout=0, use_tqdm=True, use_cuda=False, callbacks=None): """ :param DataSet train_data: the training data :param torch.nn.modules.module model: a PyTorch model @@ -46,22 +46,27 @@ def __init__(self, train_data, model, loss=None, metrics=None, n_epochs=3, batch :param int print_every: step interval to print next training information. Default: -1(no print). :param int validate_every: step interval to do next validation. Default: -1(validate every epoch). :param DataSet dev_data: the validation data - :param bool use_cuda: whether to use CUDA in training. :param str save_path: file path to save models :param Optimizer optimizer: an optimizer object :param int check_code_level: level of FastNLP code checker. -1: don't check, 0: ignore. 1: warning. 2: strict.\\ `ignore` will not check unused field; `warning` when warn if some field are not used; `strict` means - it will raise error if some field are not used. 检查的原理是通过使用很小的batch(默认两个sample)来检查代码是否能够 - 运行，但是这个过程理论上不会修改任何参数，只是会检查能否运行。但如果(1)模型中存在将batch_size写为某个固定值的情况，；(2) - 模型中存在累加前向计算次数的，可能会多计算几次。建议将check_code_level设置为-1 + it will raise error if some field are not used. 检查的原理是通过使用很小的batch(默认两个sample)来检查代码是 + 否能够运行，但是这个过程理论上不会修改任何参数，只是会检查能否运行。但如果(1)模型中存在将batch_size写为某个 + 固定值的情况；(2)模型中存在累加前向计算次数的，可能会多计算几次。以上情况建议将check_code_level设置为-1 :param str metric_key: a single indicator used to decide the best model based on metric results. It must be one of the keys returned by the FIRST metric in `metrics`. If the overall result gets better if the indicator gets smaller, add "-" in front of the string. For example:: metric_key="-PPL" # language model gets better as perplexity gets smaller :param BaseSampler sampler: method used to generate batch data. + :param num_workers: int, 使用多少个进程来准备数据。默认为0, 即使用主线程生成数据。 特性处于实验阶段，谨慎使用。 + 如果DataSet较大，且每个batch的准备时间很短，使用多进程可能并不能提速。 + :param pin_memory: bool, 默认为False. 设置为True时，有可能可以节省tensor从cpu移动到gpu的阻塞时间。 + :param timeout: float, 大于0的数，只有在num_workers>0时才有用。超过该时间仍然没有获取到一个batch则报错，可以用于 + 检测是否出现了batch产生阻塞的情况。 :param bool use_tqdm: whether to use tqdm to show train progress. - + :param callbacks: List[Callback]. 用于在train过程中起调节作用的回调函数。比如early stop，negative sampling等可以 + 通过callback机制实现。 """ super(Trainer, self).__init__() @@ -117,6 +122,9 @@ def __init__(self, train_data, model, loss=None, metrics=None, n_epochs=3, batch self.validate_every = int(validate_every) if validate_every!=0 else -1 self.best_metric_indicator = None self.sampler = sampler + self.num_workers = num_workers + self.pin_memory = pin_memory + self.timeout = timeout self.callback_manager = CallbackManager(env={"trainer": self}, callbacks=callbacks) if isinstance(optimizer, torch.optim.Optimizer): @@ -237,7 +245,8 @@ def _train(self): len(self.train_data) % self.batch_size != 0)) * self.n_epochs with inner_tqdm(total=total_steps, postfix='loss:{0:<6.5f}', leave=False, dynamic_ncols=True) as pbar: avg_loss = 0 - data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False) + data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False, + num_workers=self.num_workers, pin_memory=self.pin_memory, timeout=self.timeout) for epoch in range(1, self.n_epochs+1): pbar.set_description_str(desc="Epoch {}/{}".format(epoch, self.n_epochs)) # early stopping diff --git a/fastNLP/core/utils.py b/fastNLP/core/utils.py index 2e0f383e..695efdfc 100644 --- a/fastNLP/core/utils.py +++ b/fastNLP/core/utils.py @@ -186,11 +186,12 @@ def _check_function_or_method(func): raise TypeError(f"{type(func)} is not a method or function.") -def _move_dict_value_to_device(*args, device: torch.device): +def _move_dict_value_to_device(*args, device: torch.device, non_blocking=False): """ move data to model's device, element in *args should be dict. This is a inplace change. :param device: torch.device + :param non_blocking: bool, 是否异步将数据转移到cpu, 需要tensor使用pin_memory() :param args: :return: """ @@ -201,7 +202,7 @@ def _move_dict_value_to_device(*args, device: torch.device): if isinstance(arg, dict): for key, value in arg.items(): if isinstance(value, torch.Tensor): - arg[key] = value.to(device) + arg[key] = value.to(device, non_blocking=non_blocking) else: raise TypeError("Only support `dict` type right now.") diff --git a/test/core/test_batch.py b/test/core/test_batch.py index 7308ebf0..29a48559 100644 --- a/test/core/test_batch.py +++ b/test/core/test_batch.py @@ -8,7 +8,35 @@ from fastNLP.core.dataset import construct_dataset from fastNLP.core.instance import Instance from fastNLP.core.sampler import SequentialSampler +import time +def generate_fake_dataset(num_samples=1000): + """ + 产生的DataSet包含以下的field {'1':[], '2':[], '3': [], '4':[]} + :param num_samples: sample的数量 + :return: + """ + + max_len = 50 + min_len = 10 + num_features = 4 + + data_dict = {} + for i in range(num_features): + data = [] + lengths = np.random.randint(min_len, max_len, size=(num_samples)) + for length in lengths: + data.append(np.random.randint(100, size=length)) + data_dict[str(i)] = data + + dataset = DataSet(data_dict) + + for i in range(num_features): + if np.random.randint(2) == 0: + dataset.set_input(str(i)) + else: + dataset.set_target(str(i)) + return dataset class TestCase1(unittest.TestCase): def test_simple(self): @@ -98,3 +126,47 @@ def test_list_of_numpy_to_tensor(self): iter = Batch(ds, batch_size=4, sampler=SequentialSampler(), as_numpy=False) for x, y in iter: print(x, y) + + def test_sequential_batch(self): + batch_size = 32 + pause_seconds = 0.01 + num_samples = 1000 + dataset = generate_fake_dataset(num_samples) + + batch = Batch(dataset, batch_size=batch_size, sampler=SequentialSampler()) + for batch_x, batch_y in batch: + time.sleep(pause_seconds) + + def test_multi_workers_batch(self): + batch_size = 32 + pause_seconds = 0.01 + num_samples = 1000 + dataset = generate_fake_dataset(num_samples) + + num_workers = 1 + batch = Batch(dataset, batch_size=batch_size, sampler=SequentialSampler(), num_workers=num_workers) + for batch_x, batch_y in batch: + time.sleep(pause_seconds) + + num_workers = 2 + batch = Batch(dataset, batch_size=batch_size, sampler=SequentialSampler(), num_workers=num_workers) + end1 = time.time() + for batch_x, batch_y in batch: + time.sleep(pause_seconds) + + def test_pin_memory(self): + batch_size = 32 + pause_seconds = 0.01 + num_samples = 1000 + dataset = generate_fake_dataset(num_samples) + + batch = Batch(dataset, batch_size=batch_size, sampler=SequentialSampler(), pin_memory=True) + for batch_x, batch_y in batch: + time.sleep(pause_seconds) + + num_workers = 2 + batch = Batch(dataset, batch_size=batch_size, sampler=SequentialSampler(), num_workers=num_workers, + pin_memory=True) + for batch_x, batch_y in batch: + time.sleep(pause_seconds) + From d9ac3344093e53b0ce3cbe2c25e45ffaa35b6a99 Mon Sep 17 00:00:00 2001 From: yh_cc Date: Fri, 18 Jan 2019 23:33:19 +0800 Subject: [PATCH 13/32] =?UTF-8?q?=E5=87=8F=E5=B0=91batch=E4=B8=AD=E4=B8=8D?= =?UTF-8?q?=E6=96=AD=E5=88=9B=E5=BB=BA=E5=A4=9A=E8=BF=9B=E7=A8=8B=E7=9A=84?= =?UTF-8?q?=E5=BC=80=E9=94=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/batch.py | 50 ++++++++++++++++++++++++++++++++------- fastNLP/core/trainer.py | 9 ++++--- test/core/test_trainer.py | 31 +++++++++++++++++++++--- 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/fastNLP/core/batch.py b/fastNLP/core/batch.py index 05bd5665..9dbf9604 100644 --- a/fastNLP/core/batch.py +++ b/fastNLP/core/batch.py @@ -15,24 +15,28 @@ class Batch(object): def __init__(self, dataset, batch_size, sampler=RandomSampler(), as_numpy=False, num_workers=0, pin_memory=False, - timeout=0.0): + timeout=0.0, keep_process=False): """ Batch is an iterable object which iterates over mini-batches. Example:: - - for batch_x, batch_y in Batch(data_set, batch_size=16, sampler=SequentialSampler()): - # ... + iterator = Batch(data_set, batch_size=16, sampler=SequentialSampler()) + for epoch in range(num_epochs): + for batch_x, batch_y in iterator: # 每次epoch会重新使用sampler生成index的。 + # ... :param DataSet dataset: a DataSet object :param int batch_size: the size of the batch :param Sampler sampler: a Sampler object - :param bool as_numpy: If True, return Numpy array when possible. Otherwise, return torch tensors. + :param bool as_numpy: If True, return Numpy array. Otherwise, return torch tensors. :param num_workers: int, 使用多少个进程来准备数据。默认为0, 即使用主线程生成数据。 特性处于实验阶段，谨慎使用。 如果DataSet较大，且每个batch的准备时间很短，使用多进程可能并不能提速。 :param pin_memory: bool, 默认为False. 设置为True时，有可能可以节省tensor从cpu移动到gpu的阻塞时间。 :param timeout: float, 大于0的数，只有在num_workers>0时才有用。超过该时间仍然没有获取到一个batch则报错，可以用于 检测是否出现了batch产生阻塞的情况。 + :param keep_process: bool. 默认为False，该参数只在多进程下有效。在多进程的情况下，反复产生batch的iterator会导致 + 不断创建、销毁进程，可能对速度有一定的影响。当keep_process为True时，直到Batch对象被删除之前，多进程都没有关 + 闭。如果设置了keep_process为True，可以通过del BatchObject来删除Batch对象并关闭进程。 """ if num_workers < 0: @@ -45,15 +49,24 @@ def __init__(self, dataset, batch_size, sampler=RandomSampler(), as_numpy=False, self.batch_size = batch_size self.sampler = sampler self.num_workers = num_workers + self.keep_process = keep_process self.pin_memory = pin_memory self.timeout = timeout self.as_numpy = as_numpy self.num_batches = len(dataset) // batch_size + int(len(dataset) % batch_size != 0) self.cur_batch_indices = None + self._data_iterator = None def __iter__(self): - # TODO 现在多线程的情况下每个循环都会重新创建多进程，开销可能有点大。可以考虑直接复用iterator. - return _DataLoaderIter(self) + if self._data_iterator is not None: + # 重新设置index_list + self._data_iterator.reset() + return self._data_iterator + elif self.keep_process and self.num_workers>0: + self._data_iterator = _DataLoaderIter(self) + return self._data_iterator + else: # 大多数情况是这个 + return _DataLoaderIter(self) def __len__(self): return self.num_batches @@ -61,6 +74,12 @@ def __len__(self): def get_batch_indices(self): return self.cur_batch_indices + def __del__(self): + if self.keep_process is True: + del self._data_iterator + + + def to_tensor(batch, dtype): try: if dtype in (int, np.int8, np.int16, np.int32, np.int64): @@ -276,6 +295,7 @@ def __init__(self, batcher): self.num_workers = batcher.num_workers self.pin_memory = batcher.pin_memory and torch.cuda.is_available() self.timeout = batcher.timeout + self.keep_process = batcher.keep_process self.done_event = threading.Event() self.curidx = 0 self.idx_list = self.sampler(self.dataset) @@ -335,6 +355,17 @@ def __init__(self, batcher): for _ in range(2 * self.num_workers): self._put_indices() + def reset(self): + """ + 重置curidx以及重新采样idx_list. 只有再需要keep_process时才有用 + :return: + """ + if self.keep_process: + self.curidx = 0 + self.idx_list = self.sampler(self.dataset) + for _ in range(2 * self.num_workers): + self._put_indices() + def _get_batch(self): if self.timeout > 0: try: @@ -366,7 +397,8 @@ def __next__(self): # 如果生成的数据为0了，则停止 if self.batches_outstanding == 0: - self._shutdown_workers() + if not self.keep_process: + self._shutdown_workers() raise StopIteration while True: @@ -449,4 +481,4 @@ def _shutdown_workers(self): def __del__(self): if self.num_workers > 0: - self._shutdown_workers() + self._shutdown_workers() \ No newline at end of file diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index 76a8562b..07d94d11 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -61,7 +61,8 @@ def __init__(self, train_data, model, loss=None, metrics=None, n_epochs=3, batch :param BaseSampler sampler: method used to generate batch data. :param num_workers: int, 使用多少个进程来准备数据。默认为0, 即使用主线程生成数据。 特性处于实验阶段，谨慎使用。 如果DataSet较大，且每个batch的准备时间很短，使用多进程可能并不能提速。 - :param pin_memory: bool, 默认为False. 设置为True时，有可能可以节省tensor从cpu移动到gpu的阻塞时间。 + :param pin_memory: bool, 默认为False. 当设置为True时，会使用锁页内存，可能导致内存占用变多。如果内存比较充足， + 可以考虑设置为True进行加速, 当pin_memory为True时，默认使用non_blocking=True的方式将数据从cpu移动到gpu。 :param timeout: float, 大于0的数，只有在num_workers>0时才有用。超过该时间仍然没有获取到一个batch则报错，可以用于 检测是否出现了batch产生阻塞的情况。 :param bool use_tqdm: whether to use tqdm to show train progress. @@ -246,7 +247,8 @@ def _train(self): with inner_tqdm(total=total_steps, postfix='loss:{0:<6.5f}', leave=False, dynamic_ncols=True) as pbar: avg_loss = 0 data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False, - num_workers=self.num_workers, pin_memory=self.pin_memory, timeout=self.timeout) + num_workers=self.num_workers, pin_memory=self.pin_memory, timeout=self.timeout, + keep_process=True) for epoch in range(1, self.n_epochs+1): pbar.set_description_str(desc="Epoch {}/{}".format(epoch, self.n_epochs)) # early stopping @@ -255,7 +257,8 @@ def _train(self): indices = data_iterator.get_batch_indices() # negative sampling; replace unknown; re-weight batch_y self.callback_manager.before_batch(batch_x, batch_y, indices) - _move_dict_value_to_device(batch_x, batch_y, device=self._model_device) + _move_dict_value_to_device(batch_x, batch_y, device=self._model_device, + non_blocking=self.pin_memory) # pin_memory, use non_blockling. prediction = self._data_forward(self.model, batch_x) # edit prediction diff --git a/test/core/test_trainer.py b/test/core/test_trainer.py index 624f2587..7c869633 100644 --- a/test/core/test_trainer.py +++ b/test/core/test_trainer.py @@ -237,6 +237,31 @@ def forward(self, x1, x2): use_tqdm=False, print_every=2) - def test_case2(self): - # check metrics Wrong - data_set = prepare_fake_dataset2('x1', 'x2') + def test_trainer_multiprocess(self): + dataset = prepare_fake_dataset2('x1', 'x2') + dataset.set_input('x1', 'x2', 'y', flag=True) + + class Model(nn.Module): + def __init__(self): + super().__init__() + self.fc = nn.Linear(5, 4) + + def forward(self, x1, x2, y): + x1 = self.fc(x1) + x2 = self.fc(x2) + x = x1 + x2 + loss = F.cross_entropy(x, y) + return {'loss': loss} + + model = Model() + trainer = Trainer( + train_data=dataset, + model=model, + use_tqdm=True, + print_every=2, + num_workers=2, + pin_memory=False, + timeout=0, + ) + trainer.train() + From ab953b43ab5e01941edb88b04bd97fd14f6591e8 Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Sat, 19 Jan 2019 15:23:07 +0800 Subject: [PATCH 14/32] =?UTF-8?q?*=20=E9=87=8D=E6=9E=84POS=20API=EF=BC=8C?= =?UTF-8?q?=E6=94=B9=E6=88=90=E6=8E=A5=E5=8F=97word=E4=BD=9C=E4=B8=BA?= =?UTF-8?q?=E8=BE=93=E5=85=A5=20*=20=E6=B7=BB=E5=8A=A0=E4=B8=A4=E7=B1=BBCa?= =?UTF-8?q?llback=20*=20=E5=AE=8C=E5=96=84Trainer=E5=AF=B9error=E7=9A=84?= =?UTF-8?q?=E6=8D=95=E6=8D=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/api/api.py | 53 ++++++++++++++++++------------------ fastNLP/core/callback.py | 51 ++++++++++++++++++++++++++++++++-- fastNLP/core/trainer.py | 7 +++-- fastNLP/io/dataset_loader.py | 12 ++++---- test/core/test_callbacks.py | 32 +++++++++++++++++++++- 5 files changed, 119 insertions(+), 36 deletions(-) diff --git a/fastNLP/api/api.py b/fastNLP/api/api.py index b9bc7b70..38af57b3 100644 --- a/fastNLP/api/api.py +++ b/fastNLP/api/api.py @@ -9,7 +9,7 @@ from fastNLP.api.utils import load_url from fastNLP.api.processor import ModelProcessor -from fastNLP.io.dataset_loader import ConllCWSReader, ZhConllPOSReader, ConllxDataLoader, add_seg_tag +from fastNLP.io.dataset_loader import ConllCWSReader, ConllxDataLoader, add_seg_tag from fastNLP.core.instance import Instance from fastNLP.api.pipeline import Pipeline from fastNLP.core.metrics import SpanFPreRecMetric @@ -77,12 +77,11 @@ def predict(self, content): if not hasattr(self, "pipeline"): raise ValueError("You have to load model first.") - sentence_list = [] + sentence_list = content # 1. 检查sentence的类型 - if isinstance(content, str): - sentence_list.append(content) - elif isinstance(content, list): - sentence_list = content + for sentence in sentence_list: + if not all((type(obj) == str for obj in sentence)): + raise ValueError("Input must be list of list of string.") # 2. 组建dataset dataset = DataSet() @@ -91,33 +90,35 @@ def predict(self, content): # 3. 使用pipeline self.pipeline(dataset) - def decode_tags(ins): - pred_tags = ins["tag"] - chars = ins["words"] - words = [] - start_idx = 0 - for idx, tag in enumerate(pred_tags): - if tag[0] == "S": - words.append(chars[start_idx:idx + 1] + "/" + tag[2:]) - start_idx = idx + 1 - elif tag[0] == "E": - words.append("".join(chars[start_idx:idx + 1]) + "/" + tag[2:]) - start_idx = idx + 1 - return words - - dataset.apply(decode_tags, new_field_name="tag_output") - - output = dataset.field_arrays["tag_output"].content + # def decode_tags(ins): + # pred_tags = ins["tag"] + # chars = ins["words"] + # words = [] + # start_idx = 0 + # for idx, tag in enumerate(pred_tags): + # if tag[0] == "S": + # words.append(chars[start_idx:idx + 1] + "/" + tag[2:]) + # start_idx = idx + 1 + # elif tag[0] == "E": + # words.append("".join(chars[start_idx:idx + 1]) + "/" + tag[2:]) + # start_idx = idx + 1 + # return words + # + # dataset.apply(decode_tags, new_field_name="tag_output") + + output = dataset.field_arrays["tag"].content if isinstance(content, str): return output[0] elif isinstance(content, list): return output def test(self, file_path): - test_data = ZhConllPOSReader().load(file_path) + test_data = ConllxDataLoader().load(file_path) - tag_vocab = self._dict["tag_vocab"] - pipeline = self._dict["pipeline"] + with open("model_pp_0117.pkl", "rb") as f: + save_dict = torch.load(f) + tag_vocab = save_dict["tag_vocab"] + pipeline = save_dict["pipeline"] index_tag = IndexerProcessor(vocab=tag_vocab, field_name="tag", new_added_field_name="truth", is_input=False) pipeline.pipeline = [index_tag] + pipeline.pipeline diff --git a/fastNLP/core/callback.py b/fastNLP/core/callback.py index e6760a28..f354ffc6 100644 --- a/fastNLP/core/callback.py +++ b/fastNLP/core/callback.py @@ -169,7 +169,7 @@ def after_train(self, model): pass @transfer - def on_exception(self, exception, model, indices): + def on_exception(self, exception, model): pass @@ -235,7 +235,12 @@ def after_backward(self, model): self.clip_fun(model.parameters(), self.clip_value) -class EarlyStopError(BaseException): +class CallbackException(BaseException): + def __init__(self, msg): + super(CallbackException, self).__init__(msg) + + +class EarlyStopError(CallbackException): def __init__(self, msg): super(EarlyStopError, self).__init__(msg) @@ -266,6 +271,48 @@ def after_valid(self, eval_result, metric_key, optimizer): def on_exception(self, exception, model): if isinstance(exception, EarlyStopError): print("Early Stopping triggered in epoch {}!".format(self.epoch)) + else: + raise exception # 抛出陌生Error + + +class LRScheduler(Callback): + def __init__(self, lr_scheduler): + """对PyTorch LR Scheduler的包装 + + :param lr_scheduler: PyTorch的lr_scheduler + """ + super(LRScheduler, self).__init__() + import torch.optim + if isinstance(lr_scheduler, torch.optim.lr_scheduler._LRScheduler): + self.scheduler = lr_scheduler + else: + raise ValueError(f"Expect torch.optim.lr_scheduler for LRScheduler. Got {type(lr_scheduler)}.") + + def before_epoch(self, cur_epoch, total_epoch): + self.scheduler.step() + print("scheduler step ", "lr=", self.trainer.optimizer.param_groups[0]["lr"]) + + +class ControlC(Callback): + def __init__(self, quit_all): + """ + + :param quit_all: 若为True,则检测到control+C 直接退出程序；否则只退出Trainer + """ + super(ControlC, self).__init__() + if type(quit_all) != bool: + raise ValueError("In KeyBoardInterrupt, quit_all arguemnt must be a bool.") + self.quit_all = quit_all + + def on_exception(self, exception, model): + if isinstance(exception, KeyboardInterrupt): + if self.quit_all is True: + import sys + sys.exit(0) # 直接退出程序 + else: + pass + else: + raise exception # 抛出陌生Error if __name__ == "__main__": diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index 07d94d11..a5861091 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -14,7 +14,7 @@ from fastNLP.core.utils import pseudo_tqdm as tqdm from fastNLP.core.batch import Batch -from fastNLP.core.callback import CallbackManager +from fastNLP.core.callback import CallbackManager, CallbackException from fastNLP.core.dataset import DataSet from fastNLP.core.losses import _prepare_losser from fastNLP.core.metrics import _prepare_metrics @@ -122,6 +122,9 @@ def __init__(self, train_data, model, loss=None, metrics=None, n_epochs=3, batch self.print_every = int(print_every) self.validate_every = int(validate_every) if validate_every!=0 else -1 self.best_metric_indicator = None + self.best_dev_epoch = None + self.best_dev_step = None + self.best_dev_perf = None self.sampler = sampler self.num_workers = num_workers self.pin_memory = pin_memory @@ -212,7 +215,7 @@ def pass_func(*args, **kwargs): self.callback_manager.before_train() self._train() self.callback_manager.after_train(self.model) - except BaseException as e: + except (CallbackException, KeyboardInterrupt) as e: self.callback_manager.on_exception(e, self.model) if self.dev_data is not None: diff --git a/fastNLP/io/dataset_loader.py b/fastNLP/io/dataset_loader.py index fb781c3e..c1092e53 100644 --- a/fastNLP/io/dataset_loader.py +++ b/fastNLP/io/dataset_loader.py @@ -876,7 +876,7 @@ def get_one(self, sample): class ConllxDataLoader(object): - def load(self, path): + def load(self, path, return_dataset=False): datalist = [] with open(path, 'r', encoding='utf-8') as f: sample = [] @@ -894,10 +894,12 @@ def load(self, path): data = [self.get_one(sample) for sample in datalist] data_list = list(filter(lambda x: x is not None, data)) - ds = DataSet() - for example in data_list: - ds.append(Instance(words=example[0], tag=example[1])) - return ds + if return_dataset is True: + ds = DataSet() + for example in data_list: + ds.append(Instance(words=example[0], tag=example[1])) + data_list = ds + return data_list def get_one(self, sample): sample = list(map(list, zip(*sample))) diff --git a/test/core/test_callbacks.py b/test/core/test_callbacks.py index e5c4dc6b..59f2be1b 100644 --- a/test/core/test_callbacks.py +++ b/test/core/test_callbacks.py @@ -1,8 +1,9 @@ import unittest import numpy as np +import torch -from fastNLP.core.callback import EchoCallback, EarlyStopCallback, GradientClipCallback +from fastNLP.core.callback import EchoCallback, EarlyStopCallback, GradientClipCallback, LRScheduler, ControlC from fastNLP.core.dataset import DataSet from fastNLP.core.instance import Instance from fastNLP.core.losses import BCELoss @@ -76,3 +77,32 @@ def test_early_stop(self): metrics=AccuracyMetric(pred="predict", target="y"), callbacks=[EarlyStopCallback(5)]) trainer.train() + + def test_lr_scheduler(self): + data_set, model = prepare_env() + optimizer = torch.optim.SGD(model.parameters(), lr=0.01) + trainer = Trainer(data_set, model, + loss=BCELoss(pred="predict", target="y"), + n_epochs=50, + batch_size=32, + print_every=50, + optimizer=optimizer, + check_code_level=2, + use_tqdm=False, + dev_data=data_set, + metrics=AccuracyMetric(pred="predict", target="y"), + callbacks=[LRScheduler(torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1))]) + trainer.train() + + def test_KeyBoardInterrupt(self): + data_set, model = prepare_env() + trainer = Trainer(data_set, model, + loss=BCELoss(pred="predict", target="y"), + n_epochs=50, + batch_size=32, + print_every=50, + optimizer=SGD(lr=0.1), + check_code_level=2, + use_tqdm=False, + callbacks=[ControlC(False)]) + trainer.train() From eb55856c784619df07ec18330db5108c97b75666 Mon Sep 17 00:00:00 2001 From: yunfan Date: Sat, 19 Jan 2019 16:07:10 +0800 Subject: [PATCH 15/32] - fix parser train --- fastNLP/core/dataset.py | 2 +- fastNLP/io/dataset_loader.py | 5 +- fastNLP/models/biaffine_parser.py | 34 ++++++-- reproduction/Biaffine_parser/cfg.cfg | 8 +- reproduction/Biaffine_parser/run.py | 118 ++++++++++++--------------- 5 files changed, 90 insertions(+), 77 deletions(-) diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index f4e64c5d..b763ada2 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -275,7 +275,7 @@ def apply(self, func, new_field_name=None, **kwargs): :return results: if new_field_name is not passed, returned values of the function over all instances. """ results = [func(ins) for ins in self._inner_iter()] - if len(list(filter(lambda x: x is not None, results))) == 0 and not (new_field_name is None): # all None + if not (new_field_name is None) and len(list(filter(lambda x: x is not None, results))) == 0: # all None raise ValueError("{} always return None.".format(get_func_signature(func=func))) extra_param = {} diff --git a/fastNLP/io/dataset_loader.py b/fastNLP/io/dataset_loader.py index c1092e53..211d6cc9 100644 --- a/fastNLP/io/dataset_loader.py +++ b/fastNLP/io/dataset_loader.py @@ -897,7 +897,10 @@ def load(self, path, return_dataset=False): if return_dataset is True: ds = DataSet() for example in data_list: - ds.append(Instance(words=example[0], tag=example[1])) + ds.append(Instance(words=example[0], + pos_tags=example[1], + heads=example[2], + labels=example[3])) data_list = ds return data_list diff --git a/fastNLP/models/biaffine_parser.py b/fastNLP/models/biaffine_parser.py index b9b9dd56..dfbaac58 100644 --- a/fastNLP/models/biaffine_parser.py +++ b/fastNLP/models/biaffine_parser.py @@ -216,6 +216,7 @@ def __init__(self, self.word_norm = nn.LayerNorm(word_hid_dim) self.pos_norm = nn.LayerNorm(pos_hid_dim) self.encoder_name = encoder + self.max_len = 512 if encoder == 'var-lstm': self.encoder = VarLSTM(input_size=word_hid_dim + pos_hid_dim, hidden_size=rnn_hidden_size, @@ -233,6 +234,20 @@ def __init__(self, batch_first=True, dropout=dropout, bidirectional=True) + elif encoder == 'transformer': + n_head = 16 + d_k = d_v = int(rnn_out_size / n_head) + if (d_k * n_head) != rnn_out_size: + raise ValueError('unsupported rnn_out_size: {} for transformer'.format(rnn_out_size)) + self.position_emb = nn.Embedding(num_embeddings=self.max_len, + embedding_dim=rnn_out_size,) + self.encoder = TransformerEncoder(num_layers=rnn_layers, + model_size=rnn_out_size, + inner_size=1024, + key_size=d_k, + value_size=d_v, + num_head=n_head, + dropout=dropout,) else: raise ValueError('unsupported encoder type: {}'.format(encoder)) @@ -285,13 +300,18 @@ def forward(self, word_seq, pos_seq, seq_lens, gold_heads=None): x = torch.cat([word, pos], dim=2) # -> [N,L,C] # encoder, extract features - sort_lens, sort_idx = torch.sort(seq_lens, dim=0, descending=True) - x = x[sort_idx] - x = nn.utils.rnn.pack_padded_sequence(x, sort_lens, batch_first=True) - feat, _ = self.encoder(x) # -> [N,L,C] - feat, _ = nn.utils.rnn.pad_packed_sequence(feat, batch_first=True) - _, unsort_idx = torch.sort(sort_idx, dim=0, descending=False) - feat = feat[unsort_idx] + if self.encoder_name.endswith('lstm'): + sort_lens, sort_idx = torch.sort(seq_lens, dim=0, descending=True) + x = x[sort_idx] + x = nn.utils.rnn.pack_padded_sequence(x, sort_lens, batch_first=True) + feat, _ = self.encoder(x) # -> [N,L,C] + feat, _ = nn.utils.rnn.pad_packed_sequence(feat, batch_first=True) + _, unsort_idx = torch.sort(sort_idx, dim=0, descending=False) + feat = feat[unsort_idx] + else: + seq_range = torch.arange(seq_len, dtype=torch.long, device=x.device)[None,:] + x = x + self.position_emb(seq_range) + feat = self.encoder(x, mask.float()) # for arc biaffine # mlp, reduce dim diff --git a/reproduction/Biaffine_parser/cfg.cfg b/reproduction/Biaffine_parser/cfg.cfg index ad06598f..4a56bad5 100644 --- a/reproduction/Biaffine_parser/cfg.cfg +++ b/reproduction/Biaffine_parser/cfg.cfg @@ -1,9 +1,9 @@ [train] -n_epochs = 40 +n_epochs = 1 batch_size = 32 use_cuda = true use_tqdm=true -validate_every = -1 +validate_every = 1000 use_golden_train=true [test] @@ -17,7 +17,7 @@ use_cuda = true [model] word_vocab_size = -1 -word_emb_dim = 100 +word_emb_dim = 300 pos_vocab_size = -1 pos_emb_dim = 100 rnn_layers = 3 @@ -30,5 +30,5 @@ encoder="transformer" use_greedy_infer=false [optim] -lr = 3e-4 +lr = 2e-3 ;weight_decay = 3e-5 diff --git a/reproduction/Biaffine_parser/run.py b/reproduction/Biaffine_parser/run.py index ded7487d..e74018ba 100644 --- a/reproduction/Biaffine_parser/run.py +++ b/reproduction/Biaffine_parser/run.py @@ -4,6 +4,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) import fastNLP +import torch from fastNLP.core.trainer import Trainer from fastNLP.core.instance import Instance @@ -14,10 +15,13 @@ from fastNLP.io.model_io import ModelLoader from fastNLP.io.dataset_loader import ConllxDataLoader from fastNLP.api.processor import * +from fastNLP.io.embed_loader import EmbedLoader +from fastNLP.core.callback import Callback BOS = '' EOS = '' UNK = '' +PAD = '' NUM = '' ENG = '' @@ -28,11 +32,11 @@ def convert(data): dataset = DataSet() for sample in data: - word_seq = [BOS] + sample[0] - pos_seq = [BOS] + sample[1] - heads = [0] + list(map(int, sample[2])) - head_tags = [BOS] + sample[3] - dataset.append(Instance(words=word_seq, + word_seq = [BOS] + sample['words'] + pos_seq = [BOS] + sample['pos_tags'] + heads = [0] + sample['heads'] + head_tags = [BOS] + sample['labels'] + dataset.append(Instance(raw_words=word_seq, pos=pos_seq, gold_heads=heads, arc_true=heads, @@ -45,24 +49,11 @@ def load(path): return convert(data) -# datadir = "/mnt/c/Me/Dev/release-2.2-st-train-dev-data/ud-treebanks-v2.2/UD_English-EWT" -# datadir = "/home/yfshao/UD_English-EWT" -# train_data_name = "en_ewt-ud-train.conllu" -# dev_data_name = "en_ewt-ud-dev.conllu" -# emb_file_name = '/home/yfshao/glove.6B.100d.txt' -# loader = ConlluDataLoader() - -# datadir = '/home/yfshao/workdir/parser-data/' -# train_data_name = "train_ctb5.txt" -# dev_data_name = "dev_ctb5.txt" -# test_data_name = "test_ctb5.txt" - -datadir = "/home/yfshao/workdir/ctb7.0/" +datadir = "/remote-home/yfshao/workdir/ctb9.0/" train_data_name = "train.conllx" dev_data_name = "dev.conllx" test_data_name = "test.conllx" -# emb_file_name = "/home/yfshao/workdir/parser-data/word_OOVthr_30_100v.txt" -emb_file_name = "/home/yfshao/workdir/word_vector/cc.zh.300.vec" +emb_file_name = "/remote-home/yfshao/workdir/word_vector/cc.zh.300.vec" cfgfile = './cfg.cfg' processed_datadir = './save' @@ -108,27 +99,23 @@ def update_v(vocab, data, field): data.apply(lambda x: vocab.add_word_lst(x[field]), new_field_name=None) -print('load raw data and preprocess') # use pretrain embedding -word_v = Vocabulary() -word_v.unknown_label = UNK -pos_v = Vocabulary() +word_v = Vocabulary(unknown=UNK, padding=PAD) +pos_v = Vocabulary(unknown=None, padding=PAD) tag_v = Vocabulary(unknown=None, padding=None) train_data = load(os.path.join(datadir, train_data_name)) dev_data = load(os.path.join(datadir, dev_data_name)) test_data = load(os.path.join(datadir, test_data_name)) -print(train_data[0]) -num_p = Num2TagProcessor('words', 'words') +print('load raw data and preprocess') + +num_p = Num2TagProcessor(tag=NUM, field_name='raw_words', new_added_field_name='words') for ds in (train_data, dev_data, test_data): num_p(ds) - update_v(word_v, train_data, 'words') update_v(pos_v, train_data, 'pos') update_v(tag_v, train_data, 'tags') print('vocab build success {}, {}, {}'.format(len(word_v), len(pos_v), len(tag_v))) -# embed, _ = EmbedLoader.fast_load_embedding(model_args['word_emb_dim'], emb_file_name, word_v) -# print(embed.size()) # Model model_args['word_vocab_size'] = len(word_v) @@ -159,7 +146,6 @@ def update_v(vocab, data, field): if train_args['use_golden_train']: train_data.set_input('gold_heads', flag=True) train_args.data.pop('use_golden_train') -ignore_label = pos_v['punct'] print(test_data[0]) print('train len {}'.format(len(train_data))) @@ -167,45 +153,60 @@ def update_v(vocab, data, field): print('test len {}'.format(len(test_data))) - def train(path): # test saving pipeline save_pipe(path) - # Trainer - trainer = Trainer(model=model, train_data=train_data, dev_data=dev_data, - loss=ParserLoss(), metrics=ParserMetric(), metric_key='UAS', - **train_args.data, - optimizer=fastNLP.Adam(**optim_args.data), - save_path=path) - - # model.word_embedding = torch.nn.Embedding.from_pretrained(embed, freeze=False) + # embed = EmbedLoader.fast_load_embedding(emb_dim=model_args['word_emb_dim'], emb_file=emb_file_name, vocab=word_v) + # embed = torch.tensor(embed, dtype=torch.float32) + # model.word_embedding = torch.nn.Embedding.from_pretrained(embed, freeze=True) model.word_embedding.padding_idx = word_v.padding_idx model.word_embedding.weight.data[word_v.padding_idx].fill_(0) model.pos_embedding.padding_idx = pos_v.padding_idx model.pos_embedding.weight.data[pos_v.padding_idx].fill_(0) - # try: - # ModelLoader.load_pytorch(model, "./save/saved_model.pkl") - # print('model parameter loaded!') - # except Exception as _: - # print("No saved model. Continue.") - # pass + class MyCallback(Callback): + def after_step(self, optimizer): + step = self.trainer.step + # learning rate decay + if step > 0 and step % 1000 == 0: + for pg in optimizer.param_groups: + pg['lr'] *= 0.93 + print('decay lr to {}'.format([pg['lr'] for pg in optimizer.param_groups])) + + if step == 3000: + # start training embedding + print('start training embedding at {}'.format(step)) + model = self.trainer.model + for m in model.modules(): + if isinstance(m, torch.nn.Embedding): + m.weight.requires_grad = True - # Start training - trainer.train() - print("Training finished!") + # Trainer + trainer = Trainer(model=model, train_data=train_data, dev_data=dev_data, + loss=ParserLoss(), metrics=ParserMetric(), metric_key='UAS', + **train_args.data, + optimizer=fastNLP.Adam(**optim_args.data), + save_path=path, + callbacks=[MyCallback()]) - # save pipeline - save_pipe(path) - print('pipe saved') + # Start training + try: + trainer.train() + print("Training finished!") + finally: + # save pipeline + save_pipe(path) + print('pipe saved') def save_pipe(path): pipe = Pipeline(processors=[num_p, word_idxp, pos_idxp, seq_p, set_input_p]) pipe.add_processor(ModelProcessor(model=model, batch_size=32)) pipe.add_processor(label_toword_p) os.makedirs(path, exist_ok=True) - torch.save({'pipeline': pipe}, os.path.join(path, 'pipe.pkl')) + torch.save({'pipeline': pipe, + 'names':['num word_idx pos_idx seq set_input model tag_to_word'.split()], + }, os.path.join(path, 'pipe.pkl')) def test(path): @@ -230,16 +231,11 @@ def test(path): print("Testing Test data") tester.test(model, test_data) -def build_pipe(parser_pipe_path): - parser_pipe = torch.load(parser_pipe_path) - - - if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description='Run a chinese word segmentation model') - parser.add_argument('--mode', help='set the model\'s model', choices=['train', 'test', 'infer', 'save']) + parser.add_argument('--mode', help='set the model\'s model', choices=['train', 'test', 'infer']) parser.add_argument('--path', type=str, default='') # parser.add_argument('--dst', type=str, default='') args = parser.parse_args() @@ -249,12 +245,6 @@ def build_pipe(parser_pipe_path): test(args.path) elif args.mode == 'infer': pass - # elif args.mode == 'save': - # print(f'save model from {args.path} to {args.dst}') - # save_model(args.path, args.dst) - # load_path = os.path.dirname(args.dst) - # print(f'save pipeline in {load_path}') - # build(load_path) else: print('no mode specified for model!') parser.print_help() From de856fb8eb64ece6e7b6e27b0b49a034de247284 Mon Sep 17 00:00:00 2001 From: yunfan Date: Tue, 15 Jan 2019 15:33:39 +0800 Subject: [PATCH 16/32] update reproduction --- fastNLP/io/embed_loader.py | 5 ++++- reproduction/Biaffine_parser/cfg.cfg | 2 +- reproduction/Biaffine_parser/run.py | 6 ++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/fastNLP/io/embed_loader.py b/fastNLP/io/embed_loader.py index e55fc55b..1615fb7f 100644 --- a/fastNLP/io/embed_loader.py +++ b/fastNLP/io/embed_loader.py @@ -101,9 +101,12 @@ def fast_load_embedding(emb_dim, emb_file, vocab): """ if vocab is None: raise RuntimeError("You must provide a vocabulary.") - embedding_matrix = np.zeros(shape=(len(vocab), emb_dim)) + embedding_matrix = np.zeros(shape=(len(vocab), emb_dim), dtype=np.float32) hit_flags = np.zeros(shape=(len(vocab),), dtype=int) with open(emb_file, "r", encoding="utf-8") as f: + startline = f.readline() + if len(startline.split()) > 2: + f.seek(0) for line in f: word, vector = EmbedLoader.parse_glove_line(line) if word in vocab: diff --git a/reproduction/Biaffine_parser/cfg.cfg b/reproduction/Biaffine_parser/cfg.cfg index 4a56bad5..87fccb18 100644 --- a/reproduction/Biaffine_parser/cfg.cfg +++ b/reproduction/Biaffine_parser/cfg.cfg @@ -26,7 +26,7 @@ arc_mlp_size = 500 label_mlp_size = 100 num_label = -1 dropout = 0.3 -encoder="transformer" +encoder="var-lstm" use_greedy_infer=false [optim] diff --git a/reproduction/Biaffine_parser/run.py b/reproduction/Biaffine_parser/run.py index e74018ba..98ef02fa 100644 --- a/reproduction/Biaffine_parser/run.py +++ b/reproduction/Biaffine_parser/run.py @@ -10,9 +10,13 @@ from fastNLP.core.instance import Instance from fastNLP.api.pipeline import Pipeline from fastNLP.models.biaffine_parser import BiaffineParser, ParserMetric, ParserLoss +from fastNLP.core.vocabulary import Vocabulary +from fastNLP.core.dataset import DataSet from fastNLP.core.tester import Tester from fastNLP.io.config_io import ConfigLoader, ConfigSection from fastNLP.io.model_io import ModelLoader +from fastNLP.io.embed_loader import EmbedLoader +from fastNLP.io.model_io import ModelSaver from fastNLP.io.dataset_loader import ConllxDataLoader from fastNLP.api.processor import * from fastNLP.io.embed_loader import EmbedLoader @@ -156,6 +160,8 @@ def update_v(vocab, data, field): def train(path): # test saving pipeline save_pipe(path) + embed = EmbedLoader.fast_load_embedding(model_args['word_emb_dim'], emb_file_name, word_v) + embed = torch.tensor(embed, dtype=torch.float32) # embed = EmbedLoader.fast_load_embedding(emb_dim=model_args['word_emb_dim'], emb_file=emb_file_name, vocab=word_v) # embed = torch.tensor(embed, dtype=torch.float32) From a7f3701bdf3fc48e4caa92210ded14bd8ca19852 Mon Sep 17 00:00:00 2001 From: yunfan Date: Sat, 19 Jan 2019 16:26:39 +0800 Subject: [PATCH 17/32] - revert batch --- fastNLP/core/batch.py | 491 ++++-------------------------------------- 1 file changed, 42 insertions(+), 449 deletions(-) diff --git a/fastNLP/core/batch.py b/fastNLP/core/batch.py index 9dbf9604..d4fcbf23 100644 --- a/fastNLP/core/batch.py +++ b/fastNLP/core/batch.py @@ -1,72 +1,63 @@ import numpy as np -import random import torch -import torch.multiprocessing as multiprocessing -from torch.utils.data.dataloader import _set_worker_signal_handlers, _update_worker_pids, \ - _remove_worker_pids, _error_if_any_worker_fails -import signal -import sys -import threading -import traceback -import os -from torch._six import FileNotFoundError from fastNLP.core.sampler import RandomSampler + class Batch(object): - def __init__(self, dataset, batch_size, sampler=RandomSampler(), as_numpy=False, num_workers=0, pin_memory=False, - timeout=0.0, keep_process=False): - """ - Batch is an iterable object which iterates over mini-batches. + """Batch is an iterable object which iterates over mini-batches. + + Example:: - Example:: - iterator = Batch(data_set, batch_size=16, sampler=SequentialSampler()) - for epoch in range(num_epochs): - for batch_x, batch_y in iterator: # 每次epoch会重新使用sampler生成index的。 - # ... + for batch_x, batch_y in Batch(data_set, batch_size=16, sampler=SequentialSampler()): + # ... - :param DataSet dataset: a DataSet object - :param int batch_size: the size of the batch - :param Sampler sampler: a Sampler object - :param bool as_numpy: If True, return Numpy array. Otherwise, return torch tensors. - :param num_workers: int, 使用多少个进程来准备数据。默认为0, 即使用主线程生成数据。 特性处于实验阶段，谨慎使用。 - 如果DataSet较大，且每个batch的准备时间很短，使用多进程可能并不能提速。 - :param pin_memory: bool, 默认为False. 设置为True时，有可能可以节省tensor从cpu移动到gpu的阻塞时间。 - :param timeout: float, 大于0的数，只有在num_workers>0时才有用。超过该时间仍然没有获取到一个batch则报错，可以用于 - 检测是否出现了batch产生阻塞的情况。 - :param keep_process: bool. 默认为False，该参数只在多进程下有效。在多进程的情况下，反复产生batch的iterator会导致 - 不断创建、销毁进程，可能对速度有一定的影响。当keep_process为True时，直到Batch对象被删除之前，多进程都没有关 - 闭。如果设置了keep_process为True，可以通过del BatchObject来删除Batch对象并关闭进程。 - """ + :param DataSet dataset: a DataSet object + :param int batch_size: the size of the batch + :param Sampler sampler: a Sampler object + :param bool as_numpy: If True, return Numpy array. Otherwise, return torch tensors. - if num_workers < 0: - raise ValueError('num_workers option cannot be negative; ' - 'use num_workers=0 to disable multiprocessing.') - if timeout < 0: - raise ValueError('timeout option should be non-negative') + """ + def __init__(self, dataset, batch_size, sampler=RandomSampler(), as_numpy=False): self.dataset = dataset self.batch_size = batch_size self.sampler = sampler - self.num_workers = num_workers - self.keep_process = keep_process - self.pin_memory = pin_memory - self.timeout = timeout self.as_numpy = as_numpy + self.idx_list = None + self.curidx = 0 self.num_batches = len(dataset) // batch_size + int(len(dataset) % batch_size != 0) self.cur_batch_indices = None - self._data_iterator = None def __iter__(self): - if self._data_iterator is not None: - # 重新设置index_list - self._data_iterator.reset() - return self._data_iterator - elif self.keep_process and self.num_workers>0: - self._data_iterator = _DataLoaderIter(self) - return self._data_iterator - else: # 大多数情况是这个 - return _DataLoaderIter(self) + self.idx_list = self.sampler(self.dataset) + self.curidx = 0 + self.lengths = self.dataset.get_length() + return self + + def __next__(self): + if self.curidx >= len(self.idx_list): + raise StopIteration + else: + endidx = min(self.curidx + self.batch_size, len(self.idx_list)) + batch_x, batch_y = {}, {} + + indices = self.idx_list[self.curidx:endidx] + self.cur_batch_indices = indices + + for field_name, field in self.dataset.get_all_fields().items(): + if field.is_target or field.is_input: + batch = field.get(indices) + if not self.as_numpy and field.padder is not None: + batch = to_tensor(batch, field.dtype) + if field.is_target: + batch_y[field_name] = batch + if field.is_input: + batch_x[field_name] = batch + + self.curidx = endidx + + return batch_x, batch_y def __len__(self): return self.num_batches @@ -74,11 +65,6 @@ def __len__(self): def get_batch_indices(self): return self.cur_batch_indices - def __del__(self): - if self.keep_process is True: - del self._data_iterator - - def to_tensor(batch, dtype): try: @@ -89,396 +75,3 @@ def to_tensor(batch, dtype): except: pass return batch - - -""" -由于多进程涉及到大量问题，包括系统、安全关闭进程等。所以这里直接从pytorch的官方版本修改DataLoader实现多进程加速 -""" - -IS_WINDOWS = sys.platform == "win32" -if IS_WINDOWS: - import ctypes - from ctypes.wintypes import DWORD, BOOL, HANDLE - -if sys.version_info[0] == 2: - import Queue as queue -else: - import queue - - -class ExceptionWrapper(object): - r"""Wraps an exception plus traceback to communicate across threads""" - - def __init__(self, exc_info): - self.exc_type = exc_info[0] - self.exc_msg = "".join(traceback.format_exception(*exc_info)) - - -_use_shared_memory = False -r"""Whether to use shared memory in default_collate""" - -MANAGER_STATUS_CHECK_INTERVAL = 5.0 - -if IS_WINDOWS: - # On Windows, the parent ID of the worker process remains unchanged when the manager process - # is gone, and the only way to check it through OS is to let the worker have a process handle - # of the manager and ask if the process status has changed. - class ManagerWatchdog(object): - def __init__(self): - self.manager_pid = os.getppid() - - self.kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) - self.kernel32.OpenProcess.argtypes = (DWORD, BOOL, DWORD) - self.kernel32.OpenProcess.restype = HANDLE - self.kernel32.WaitForSingleObject.argtypes = (HANDLE, DWORD) - self.kernel32.WaitForSingleObject.restype = DWORD - - # Value obtained from https://msdn.microsoft.com/en-us/library/ms684880.aspx - SYNCHRONIZE = 0x00100000 - self.manager_handle = self.kernel32.OpenProcess(SYNCHRONIZE, 0, self.manager_pid) - - if not self.manager_handle: - raise ctypes.WinError(ctypes.get_last_error()) - - def is_alive(self): - # Value obtained from https://msdn.microsoft.com/en-us/library/windows/desktop/ms687032.aspx - return self.kernel32.WaitForSingleObject(self.manager_handle, 0) != 0 -else: - class ManagerWatchdog(object): - def __init__(self): - self.manager_pid = os.getppid() - - def is_alive(self): - return os.getppid() == self.manager_pid - - -def _worker_loop(dataset, index_queue, data_queue, seed, worker_id, as_numpy): - # 产生数据的循环 - global _use_shared_memory - _use_shared_memory = True - - # Intialize C side signal handlers for SIGBUS and SIGSEGV. Python signal - # module's handlers are executed after Python returns from C low-level - # handlers, likely when the same fatal signal happened again already. - # https://docs.python.org/3/library/signal.html Sec. 18.8.1.1 - _set_worker_signal_handlers() - - torch.set_num_threads(1) - random.seed(seed) - torch.manual_seed(seed) - - watchdog = ManagerWatchdog() - - while True: - try: - # 获取当前batch计数，当前batch的indexes - r = index_queue.get(timeout=MANAGER_STATUS_CHECK_INTERVAL) - except queue.Empty: - if watchdog.is_alive(): - continue - else: - break - if r is None: - break - idx, batch_indices = r - try: - # 获取相应的batch数据。这里需要修改为从dataset中取出数据并且完成padding - samples = _get_batch_from_dataset(dataset, batch_indices, as_numpy) - except Exception: - data_queue.put((idx, ExceptionWrapper(sys.exc_info()), batch_indices)) - else: - data_queue.put((idx, samples, batch_indices)) - del samples - -def _get_batch_from_dataset(dataset, indices, as_numpy): - """ - 给定indices，从DataSet中取出(batch_x, batch_y). 数据从这里产生后，若没有pin_memory, 则直接传递给Trainer了，如果存在 - pin_memory还会经过一道pin_memory()的处理 - :param dataset: fastNLP.DataSet对象 - :param indices: List[int], index - :param as_numpy: bool, 是否只是转换为numpy - :return: (batch_x, batch_y) - """ - batch_x, batch_y = {}, {} - for field_name, field in dataset.get_all_fields().items(): - if field.is_target or field.is_input: - batch = field.get(indices) - if not as_numpy and field.padder is not None: - batch = to_tensor(batch, field.dtype) - if field.is_target: - batch_y[field_name] = batch - if field.is_input: - batch_x[field_name] = batch - - return batch_x, batch_y - - -def _worker_manager_loop(in_queue, out_queue, done_event, pin_memory, device_id): - # 将数据送入到指定的query中. 即如果需要pin_memory, 则 - if pin_memory: - torch.cuda.set_device(device_id) - - while True: - try: - r = in_queue.get() - except Exception: - if done_event.is_set(): - return - raise - if r is None: - break - if isinstance(r[1], ExceptionWrapper): - out_queue.put(r) - continue - idx, batch, batch_indices = r - try: - if pin_memory: - batch = pin_memory_batch(batch) - except Exception: - out_queue.put((idx, ExceptionWrapper(sys.exc_info()), batch_indices)) - else: - out_queue.put((idx, batch, batch_indices)) - - -def pin_memory_batch(batchs): - """ - - :param batchs: (batch_x, batch_y) - :return: (batch_x, batch_y) - """ - for batch_dict in batchs: - for field_name, batch in batch_dict.items(): - if isinstance(batch, torch.Tensor): - batch_dict[field_name] = batch.pin_memory() - return batchs - - -_SIGCHLD_handler_set = False -r"""Whether SIGCHLD handler is set for DataLoader worker failures. Only one -handler needs to be set for all DataLoaders in a process.""" - - -def _set_SIGCHLD_handler(): - # Windows doesn't support SIGCHLD handler - if sys.platform == 'win32': - return - # can't set signal in child threads - if not isinstance(threading.current_thread(), threading._MainThread): - return - global _SIGCHLD_handler_set - if _SIGCHLD_handler_set: - return - previous_handler = signal.getsignal(signal.SIGCHLD) - if not callable(previous_handler): - previous_handler = None - - def handler(signum, frame): - # This following call uses `waitid` with WNOHANG from C side. Therefore, - # Python can still get and update the process status successfully. - _error_if_any_worker_fails() - if previous_handler is not None: - previous_handler(signum, frame) - - signal.signal(signal.SIGCHLD, handler) - _SIGCHLD_handler_set = True - - -class _DataLoaderIter(object): - r"""Iterates once over the DataLoader's dataset, as specified by the sampler""" - - def __init__(self, batcher): - self.batcher = batcher - self.dataset = batcher.dataset - self.sampler = batcher.sampler - self.as_numpy = batcher.as_numpy - self.batch_size = batcher.batch_size - self.num_workers = batcher.num_workers - self.pin_memory = batcher.pin_memory and torch.cuda.is_available() - self.timeout = batcher.timeout - self.keep_process = batcher.keep_process - self.done_event = threading.Event() - self.curidx = 0 - self.idx_list = self.sampler(self.dataset) - - # self.sample_iter一次返回一个index. 可以通过其他方式替代 - - base_seed = torch.LongTensor(1).random_().item() - - if self.num_workers > 0: - # 每个worker建立一个index queue - self.index_queues = [multiprocessing.Queue() for _ in range(self.num_workers)] - self.worker_queue_idx = 0 - # 存放获取到的batch - self.worker_result_queue = multiprocessing.SimpleQueue() - self.batches_outstanding = 0 - self.worker_pids_set = False - self.shutdown = False - self.send_idx = 0 - self.rcvd_idx = 0 - self.reorder_dict = {} - - # 这里会将batch的数据输送到self.worker_result_queue中，但是还没有送入到device中 - self.workers = [ - multiprocessing.Process( - target=_worker_loop, - args=(self.dataset, self.index_queues[i], - self.worker_result_queue, base_seed + i, i, self.as_numpy)) - for i in range(self.num_workers)] - - # self.data_queue取数据就行。如果有pin_memory的话，会把数据放到另一个queue - if self.pin_memory or self.timeout > 0: - self.data_queue = queue.Queue() - if self.pin_memory: - maybe_device_id = torch.cuda.current_device() - else: - # do not initialize cuda context if not necessary - maybe_device_id = None - self.worker_manager_thread = threading.Thread( - target=_worker_manager_loop, - args=(self.worker_result_queue, self.data_queue, self.done_event, self.pin_memory, - maybe_device_id)) - self.worker_manager_thread.daemon = True - self.worker_manager_thread.start() - else: - self.data_queue = self.worker_result_queue - - # worker们开始工作 - for w in self.workers: - w.daemon = True # ensure that the worker exits on process exit - w.start() - - _update_worker_pids(id(self), tuple(w.pid for w in self.workers)) - _set_SIGCHLD_handler() - self.worker_pids_set = True - - # prime the prefetch loop - for _ in range(2 * self.num_workers): - self._put_indices() - - def reset(self): - """ - 重置curidx以及重新采样idx_list. 只有再需要keep_process时才有用 - :return: - """ - if self.keep_process: - self.curidx = 0 - self.idx_list = self.sampler(self.dataset) - for _ in range(2 * self.num_workers): - self._put_indices() - - def _get_batch(self): - if self.timeout > 0: - try: - return self.data_queue.get(timeout=self.timeout) - except queue.Empty: - raise RuntimeError('DataLoader timed out after {} seconds'.format(self.timeout)) - else: - return self.data_queue.get() - - def __next__(self): - if self.num_workers == 0: # same-process loading - if self.curidx >= len(self.idx_list): - raise StopIteration - endidx = min(self.curidx + self.batch_size, len(self.idx_list)) - # 直接从数据集中采集数据即可 - indices = self.idx_list[self.curidx:endidx] - self.batcher.cur_batch_indices = indices - batch_x, batch_y = _get_batch_from_dataset(dataset=self.dataset, indices=indices, - as_numpy=self.as_numpy) - if self.pin_memory: - batch_x, batch_y = pin_memory_batch((batch_x, batch_y)) - self.curidx = endidx - return batch_x, batch_y - - # check if the next sample has already been generated - if self.rcvd_idx in self.reorder_dict: - batch = self.reorder_dict.pop(self.rcvd_idx) - return self._process_next_batch(batch) - - # 如果生成的数据为0了，则停止 - if self.batches_outstanding == 0: - if not self.keep_process: - self._shutdown_workers() - raise StopIteration - - while True: - assert (not self.shutdown and self.batches_outstanding > 0) - idx, batch, batch_indices = self._get_batch() - self.batches_outstanding -= 1 - if idx != self.rcvd_idx: - # store out-of-order samples - self.reorder_dict[idx] = batch - continue - self.batcher.cur_batch_indices = batch_indices - return self._process_next_batch(batch) - - def __iter__(self): - self.curidx = 0 - - return self - - def _put_indices(self): - # 向采集数据的index queue中放入index - assert self.batches_outstanding < 2 * self.num_workers - if self.curidx >= len(self.idx_list): - indices = None - else: - endidx = min(self.curidx + self.batch_size, len(self.idx_list)) - # 直接从数据集中采集数据即可 - indices = self.idx_list[self.curidx:endidx] - if indices is None: - return - self.index_queues[self.worker_queue_idx].put((self.send_idx, indices)) - self.curidx = endidx - self.worker_queue_idx = (self.worker_queue_idx + 1) % self.num_workers - self.batches_outstanding += 1 - self.send_idx += 1 - - def _process_next_batch(self, batch): - # 只是提醒生成下一个batch indice数据 - self.rcvd_idx += 1 - self._put_indices() - if isinstance(batch, ExceptionWrapper): - raise batch.exc_type(batch.exc_msg) - return batch - - def __getstate__(self): - # TODO: add limited pickling support for sharing an iterator - # across multiple threads for HOGWILD. - # Probably the best way to do this is by moving the sample pushing - # to a separate thread and then just sharing the data queue - # but signalling the end is tricky without a non-blocking API - raise NotImplementedError("_DataLoaderIter cannot be pickled") - - def _shutdown_workers(self): - try: - if not self.shutdown: - self.shutdown = True - self.done_event.set() - for q in self.index_queues: - q.put(None) - # if some workers are waiting to put, make place for them - try: - while not self.worker_result_queue.empty(): - self.worker_result_queue.get() - except (FileNotFoundError, ImportError): - # Many weird errors can happen here due to Python - # shutting down. These are more like obscure Python bugs. - # FileNotFoundError can happen when we rebuild the fd - # fetched from the queue but the socket is already closed - # from the worker side. - # ImportError can happen when the unpickler loads the - # resource from `get`. - pass - # done_event should be sufficient to exit worker_manager_thread, - # but be safe here and put another None - self.worker_result_queue.put(None) - finally: - # removes pids no matter what - if self.worker_pids_set: - _remove_worker_pids(id(self)) - self.worker_pids_set = False - - def __del__(self): - if self.num_workers > 0: - self._shutdown_workers() \ No newline at end of file From 62ea4f7fed30671d816364ff40bc937daf7d97a5 Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Sat, 19 Jan 2019 18:40:43 +0800 Subject: [PATCH 18/32] =?UTF-8?q?=E6=B7=BB=E5=8A=A0LR=20finder=EF=BC=8C?= =?UTF-8?q?=E7=94=A8=E7=AC=AC=E4=B8=80=E4=B8=AAepoch=E6=89=BE=E6=9C=80?= =?UTF-8?q?=E4=BD=B3lr,=E4=BB=8E=E7=AC=AC=E4=BA=8C=E4=B8=AAepoch=E5=BC=80?= =?UTF-8?q?=E5=A7=8B=E8=AE=AD=E7=BB=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/callback.py | 80 +++++++++++++++++++++++++++++++++++++ test/core/test_callbacks.py | 23 ++++++++--- 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/fastNLP/core/callback.py b/fastNLP/core/callback.py index f354ffc6..e0053124 100644 --- a/fastNLP/core/callback.py +++ b/fastNLP/core/callback.py @@ -1,3 +1,8 @@ +import torch + +from fastNLP.io.model_io import ModelSaver, ModelLoader + + class Callback(object): """An Interface for all callbacks. @@ -315,6 +320,81 @@ def on_exception(self, exception, model): raise exception # 抛出陌生Error +class SmoothValue(object): + def __init__(self, beta: float): + self.beta, self.n, self.mov_avg = beta, 0, 0 + self.smooth = None + + def add_value(self, val: float) -> None: + "Add `val` to calculate updated smoothed value." + self.n += 1 + self.mov_avg = self.beta * self.mov_avg + (1 - self.beta) * val + self.smooth = self.mov_avg / (1 - self.beta ** self.n) + + +class LRFinder(Callback): + """fastai lr_finder""" + + def __init__(self, n_batch, start_lr=1e-6, end_lr=10): + """用第一个 epoch 找最佳的学习率，从第二个epoch开始应用它 + + :param n_batch: 一个epoch内的iteration数 + :param start_lr: 学习率下界 + :param end_lr: 学习率上界 + """ + super(LRFinder, self).__init__() + self.start_lr, self.end_lr = start_lr, end_lr + self.num_it = n_batch + self.stop = False + self.best_loss = 0. + self.best_lr = None + self.loss_history = [] + self.smooth_value = SmoothValue(0.8) + self.opt = None + scale = (self.end_lr - self.start_lr) / self.num_it + + self.lr_gen = (self.start_lr + scale * (step + 1) for step in range(self.num_it)) + self.find = None + self.loader = ModelLoader() + + def before_epoch(self, cur_epoch, total_epoch): + if cur_epoch == 1: + self.opt = self.trainer.optimizer # pytorch optimizer + self.opt.param_groups[0]["lr"] = self.start_lr + # save model + ModelSaver("tmp").save_pytorch(self.trainer.model, param_only=True) + self.find = True + + def before_backward(self, loss, model): + if self.find: + if torch.isnan(loss) or self.stop is True: + self.stop = True + return + loss_val = loss.detach().cpu().data + self.loss_history.append(loss_val) + self.smooth_value.add_value(loss_val) + if self.best_loss == 0. or self.smooth_value.smooth < self.best_loss: + self.best_loss = self.smooth_value.smooth + self.best_lr = self.opt.param_groups[0]["lr"] + + def after_batch(self, *args): + if self.find: + lr = next(self.lr_gen, None) + if lr is None or self.stop is True or self.loss_history[-1] > 4 * self.best_loss: + self.stop = True + return + self.opt.param_groups[0]["lr"] = lr + # self.loader.load_pytorch(self.trainer.model, "tmp") + + def after_epoch(self, cur_epoch, n_epoch, optimizer): + if cur_epoch == 1: + self.opt.param_groups[0]["lr"] = self.best_lr + self.find = False + # reset model + ModelLoader().load_pytorch(self.trainer.model, "tmp") + print("Model reset. \nFind best lr={}".format(self.best_lr)) + + if __name__ == "__main__": manager = CallbackManager(env={"n_epoch": 3}, callbacks=[DummyCallback(), DummyCallback()]) manager.before_train(10, 11, 12) diff --git a/test/core/test_callbacks.py b/test/core/test_callbacks.py index 59f2be1b..d0c1fb13 100644 --- a/test/core/test_callbacks.py +++ b/test/core/test_callbacks.py @@ -3,7 +3,7 @@ import numpy as np import torch -from fastNLP.core.callback import EchoCallback, EarlyStopCallback, GradientClipCallback, LRScheduler, ControlC +from fastNLP.core.callback import EchoCallback, EarlyStopCallback, GradientClipCallback, LRScheduler, ControlC, LRFinder from fastNLP.core.dataset import DataSet from fastNLP.core.instance import Instance from fastNLP.core.losses import BCELoss @@ -52,7 +52,7 @@ def test_gradient_clip(self): data_set, model = prepare_env() trainer = Trainer(data_set, model, loss=BCELoss(pred="predict", target="y"), - n_epochs=30, + n_epochs=20, batch_size=32, print_every=50, optimizer=SGD(lr=0.1), @@ -67,7 +67,7 @@ def test_early_stop(self): data_set, model = prepare_env() trainer = Trainer(data_set, model, loss=BCELoss(pred="predict", target="y"), - n_epochs=50, + n_epochs=20, batch_size=32, print_every=50, optimizer=SGD(lr=0.01), @@ -83,7 +83,7 @@ def test_lr_scheduler(self): optimizer = torch.optim.SGD(model.parameters(), lr=0.01) trainer = Trainer(data_set, model, loss=BCELoss(pred="predict", target="y"), - n_epochs=50, + n_epochs=5, batch_size=32, print_every=50, optimizer=optimizer, @@ -98,7 +98,7 @@ def test_KeyBoardInterrupt(self): data_set, model = prepare_env() trainer = Trainer(data_set, model, loss=BCELoss(pred="predict", target="y"), - n_epochs=50, + n_epochs=5, batch_size=32, print_every=50, optimizer=SGD(lr=0.1), @@ -106,3 +106,16 @@ def test_KeyBoardInterrupt(self): use_tqdm=False, callbacks=[ControlC(False)]) trainer.train() + + def test_LRFinder(self): + data_set, model = prepare_env() + trainer = Trainer(data_set, model, + loss=BCELoss(pred="predict", target="y"), + n_epochs=5, + batch_size=32, + print_every=50, + optimizer=SGD(lr=0.1), + check_code_level=2, + use_tqdm=False, + callbacks=[LRFinder(len(data_set) // 32)]) + trainer.train() From b14dd588285d0452722b6529991e181fa3e65219 Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Sat, 19 Jan 2019 18:48:57 +0800 Subject: [PATCH 19/32] Update POS API --- fastNLP/api/api.py | 2 +- fastNLP/api/examples.py | 6 +++++- reproduction/POS_tagging/train_pos_tag.py | 22 +++++++++++----------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/fastNLP/api/api.py b/fastNLP/api/api.py index 38af57b3..0c5f17bc 100644 --- a/fastNLP/api/api.py +++ b/fastNLP/api/api.py @@ -18,7 +18,7 @@ # TODO add pretrain urls model_urls = { "cws": "http://123.206.98.91:8888/download/cws_crf_1_11-457fc899.pkl", - "pos": "http://123.206.98.91:8888/download/pos_tag_model_20190108-f3c60ee5.pkl", + "pos": "http://123.206.98.91:8888/download/pos_tag_model_20190119-43f8b435.pkl", "parser": "http://123.206.98.91:8888/download/biaffine_parser-3a2f052c.pkl" } diff --git a/fastNLP/api/examples.py b/fastNLP/api/examples.py index 10cc6edc..447d127a 100644 --- a/fastNLP/api/examples.py +++ b/fastNLP/api/examples.py @@ -16,6 +16,10 @@ def chinese_word_segmentation(): def pos_tagging(): + # 输入已分词序列 + text = ['编者 按： 7月 12日 ， 英国 航空 航天 系统 公司 公布 了 该 公司 研制 的 第一款 高科技 隐形 无人机 雷电之神 。'] + text = [text[0].split()] + print(text) pos = POS(device='cpu') print(pos.predict(text)) @@ -26,4 +30,4 @@ def syntactic_parsing(): if __name__ == "__main__": - syntactic_parsing() + pos_tagging() diff --git a/reproduction/POS_tagging/train_pos_tag.py b/reproduction/POS_tagging/train_pos_tag.py index 6448c32b..06547701 100644 --- a/reproduction/POS_tagging/train_pos_tag.py +++ b/reproduction/POS_tagging/train_pos_tag.py @@ -14,7 +14,7 @@ from fastNLP.core.trainer import Trainer from fastNLP.io.config_io import ConfigLoader, ConfigSection from fastNLP.models.sequence_modeling import AdvSeqLabel -from fastNLP.io.dataset_loader import ZhConllPOSReader, ConllxDataLoader +from fastNLP.io.dataset_loader import ConllxDataLoader from fastNLP.api.processor import ModelProcessor, Index2WordProcessor @@ -35,7 +35,7 @@ def load_tencent_embed(embed_path, word2id): return embedding_tensor -def train(train_data_path, dev_data_path, checkpoint=None): +def train(train_data_path, dev_data_path, checkpoint=None, save=None): # load config train_param = ConfigSection() model_param = ConfigSection() @@ -44,9 +44,9 @@ def train(train_data_path, dev_data_path, checkpoint=None): # Data Loader print("loading training set...") - dataset = ConllxDataLoader().load(train_data_path) + dataset = ConllxDataLoader().load(train_data_path, return_dataset=True) print("loading dev set...") - dev_data = ConllxDataLoader().load(dev_data_path) + dev_data = ConllxDataLoader().load(dev_data_path, return_dataset=True) print(dataset) print("================= dataset ready =====================") @@ -54,9 +54,9 @@ def train(train_data_path, dev_data_path, checkpoint=None): dev_data.rename_field("tag", "truth") vocab_proc = VocabIndexerProcessor("words", new_added_filed_name="word_seq") - tag_proc = VocabIndexerProcessor("truth") + tag_proc = VocabIndexerProcessor("truth", is_input=True) seq_len_proc = SeqLenProcessor(field_name="word_seq", new_added_field_name="word_seq_origin_len", is_input=True) - set_input_proc = SetInputProcessor("word_seq", "word_seq_origin_len", "truth") + set_input_proc = SetInputProcessor("word_seq", "word_seq_origin_len") vocab_proc(dataset) tag_proc(dataset) @@ -93,7 +93,7 @@ def train(train_data_path, dev_data_path, checkpoint=None): target="truth", seq_lens="word_seq_origin_len"), dev_data=dev_data, metric_key="f", - use_tqdm=True, use_cuda=True, print_every=10, n_epochs=20, save_path="./save_0117") + use_tqdm=True, use_cuda=True, print_every=10, n_epochs=20, save_path=save) trainer.train(load_best_model=True) # save model & pipeline @@ -102,12 +102,12 @@ def train(train_data_path, dev_data_path, checkpoint=None): pp = Pipeline([vocab_proc, seq_len_proc, set_input_proc, model_proc, id2tag]) save_dict = {"pipeline": pp, "model": model, "tag_vocab": tag_proc.vocab} - torch.save(save_dict, "model_pp_0117.pkl") + torch.save(save_dict, os.path.join(save, "model_pp.pkl")) print("pipeline saved") def run_test(test_path): - test_data = ZhConllPOSReader().load(test_path) + test_data = ConllxDataLoader().load(test_path, return_dataset=True) with open("model_pp_0117.pkl", "rb") as f: save_dict = torch.load(f) @@ -157,7 +157,7 @@ def run_test(test_path): # 继续训练 python train_pos_tag.py -c -cp ./save/best_model.pkl if args.checkpoint is None: raise RuntimeError("Please provide the checkpoint. -cp ") - train(args.train, args.dev, args.checkpoint) + train(args.train, args.dev, args.checkpoint, save=args.save) else: # 一次训练 python train_pos_tag.py - train(args.train, args.dev) + train(args.train, args.dev, save=args.save) From 03f49c8264cf3a3c1b4912afcad2b7b11bb985a0 Mon Sep 17 00:00:00 2001 From: yunfan Date: Sat, 19 Jan 2019 19:44:32 +0800 Subject: [PATCH 20/32] - batch with multiprocessing --- fastNLP/core/batch.py | 55 +++++++++++++++++++++++++++++------- fastNLP/core/trainer.py | 12 +++----- fastNLP/io/dataset_loader.py | 18 ++++++------ 3 files changed, 57 insertions(+), 28 deletions(-) diff --git a/fastNLP/core/batch.py b/fastNLP/core/batch.py index d4fcbf23..3faab8c0 100644 --- a/fastNLP/core/batch.py +++ b/fastNLP/core/batch.py @@ -2,7 +2,7 @@ import torch from fastNLP.core.sampler import RandomSampler - +import torch.multiprocessing as mp class Batch(object): """Batch is an iterable object which iterates over mini-batches. @@ -29,15 +29,9 @@ def __init__(self, dataset, batch_size, sampler=RandomSampler(), as_numpy=False) self.num_batches = len(dataset) // batch_size + int(len(dataset) % batch_size != 0) self.cur_batch_indices = None - def __iter__(self): - self.idx_list = self.sampler(self.dataset) - self.curidx = 0 - self.lengths = self.dataset.get_length() - return self - - def __next__(self): + def fetch_one(self): if self.curidx >= len(self.idx_list): - raise StopIteration + return None else: endidx = min(self.curidx + self.batch_size, len(self.idx_list)) batch_x, batch_y = {}, {} @@ -56,9 +50,15 @@ def __next__(self): batch_x[field_name] = batch self.curidx = endidx - return batch_x, batch_y + def __iter__(self): + """ + Iterate on dataset, fetch batch data. Fetch process don't block the iterate process + :return: + """ + return run_batch_iter(self) + def __len__(self): return self.num_batches @@ -75,3 +75,38 @@ def to_tensor(batch, dtype): except: pass return batch + + +def run_fetch(batch, q): + batch.idx_list = batch.sampler(batch.dataset) + batch.curidx = 0 + batch.lengths = batch.dataset.get_length() + # print('start fetch') + while 1: + res = batch.fetch_one() + # print('fetch one') + q.put(res) + if res is None: + # print('fetch done, waiting processing') + q.join() + break + # print('fetch exit') + + +def run_batch_iter(batch): + q = mp.JoinableQueue(maxsize=10) + fetch_p = mp.Process(target=run_fetch, args=(batch, q)) + fetch_p.daemon = True + fetch_p.start() + # print('fork fetch process') + while 1: + res = q.get() + q.task_done() + # print('get fetched') + if res is None: + break + yield res + fetch_p.terminate() + fetch_p.join() + # print('iter done') + diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index a5861091..faa0d0a2 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -34,8 +34,8 @@ class Trainer(object): def __init__(self, train_data, model, loss=None, metrics=None, n_epochs=3, batch_size=32, print_every=50, validate_every=-1, dev_data=None, save_path=None, optimizer=Adam(lr=0.01, weight_decay=0), - check_code_level=0, metric_key=None, sampler=RandomSampler(), num_workers=0, pin_memory=False, - timeout=0, use_tqdm=True, use_cuda=False, callbacks=None): + check_code_level=0, metric_key=None, sampler=RandomSampler(), num_workers=0, + use_tqdm=True, use_cuda=False, callbacks=None): """ :param DataSet train_data: the training data :param torch.nn.modules.module model: a PyTorch model @@ -127,8 +127,6 @@ def __init__(self, train_data, model, loss=None, metrics=None, n_epochs=3, batch self.best_dev_perf = None self.sampler = sampler self.num_workers = num_workers - self.pin_memory = pin_memory - self.timeout = timeout self.callback_manager = CallbackManager(env={"trainer": self}, callbacks=callbacks) if isinstance(optimizer, torch.optim.Optimizer): @@ -249,9 +247,7 @@ def _train(self): len(self.train_data) % self.batch_size != 0)) * self.n_epochs with inner_tqdm(total=total_steps, postfix='loss:{0:<6.5f}', leave=False, dynamic_ncols=True) as pbar: avg_loss = 0 - data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False, - num_workers=self.num_workers, pin_memory=self.pin_memory, timeout=self.timeout, - keep_process=True) + data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False) for epoch in range(1, self.n_epochs+1): pbar.set_description_str(desc="Epoch {}/{}".format(epoch, self.n_epochs)) # early stopping @@ -261,7 +257,7 @@ def _train(self): # negative sampling; replace unknown; re-weight batch_y self.callback_manager.before_batch(batch_x, batch_y, indices) _move_dict_value_to_device(batch_x, batch_y, device=self._model_device, - non_blocking=self.pin_memory) # pin_memory, use non_blockling. + non_blocking=self.use_cuda) # pin_memory, use non_blockling. prediction = self._data_forward(self.model, batch_x) # edit prediction diff --git a/fastNLP/io/dataset_loader.py b/fastNLP/io/dataset_loader.py index 211d6cc9..1fcdb7d9 100644 --- a/fastNLP/io/dataset_loader.py +++ b/fastNLP/io/dataset_loader.py @@ -876,7 +876,7 @@ def get_one(self, sample): class ConllxDataLoader(object): - def load(self, path, return_dataset=False): + def load(self, path): datalist = [] with open(path, 'r', encoding='utf-8') as f: sample = [] @@ -894,15 +894,13 @@ def load(self, path, return_dataset=False): data = [self.get_one(sample) for sample in datalist] data_list = list(filter(lambda x: x is not None, data)) - if return_dataset is True: - ds = DataSet() - for example in data_list: - ds.append(Instance(words=example[0], - pos_tags=example[1], - heads=example[2], - labels=example[3])) - data_list = ds - return data_list + ds = DataSet() + for example in data_list: + ds.append(Instance(words=example[0], + pos_tags=example[1], + heads=example[2], + labels=example[3])) + return ds def get_one(self, sample): sample = list(map(list, zip(*sample))) From f3cb8125544199fa51d8329c54e8bdecc4218fe4 Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Sun, 20 Jan 2019 16:37:58 +0800 Subject: [PATCH 21/32] =?UTF-8?q?=E5=B0=86tesorboardX=E5=A4=84=E7=90=86?= =?UTF-8?q?=E4=B8=BAcallback,=20=E4=BB=8Etrainer=E7=A7=BB=E9=99=A4tensorbo?= =?UTF-8?q?ardX=E7=9B=B8=E5=85=B3=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/callback.py | 71 +++++++++++++++++++++++++++++++++++-- fastNLP/core/trainer.py | 31 +++------------- test/core/test_batch.py | 14 ++++---- test/core/test_callbacks.py | 19 +++++++++- 4 files changed, 99 insertions(+), 36 deletions(-) diff --git a/fastNLP/core/callback.py b/fastNLP/core/callback.py index e0053124..48d7333c 100644 --- a/fastNLP/core/callback.py +++ b/fastNLP/core/callback.py @@ -1,4 +1,7 @@ +import os + import torch +from tensorboardX import SummaryWriter from fastNLP.io.model_io import ModelSaver, ModelLoader @@ -12,6 +15,7 @@ class Callback(object): def __init__(self): super(Callback, self).__init__() + self.trainer = None # 在Trainer内部被重新赋值 def before_train(self): # before the main training loop @@ -333,8 +337,6 @@ def add_value(self, val: float) -> None: class LRFinder(Callback): - """fastai lr_finder""" - def __init__(self, n_batch, start_lr=1e-6, end_lr=10): """用第一个 epoch 找最佳的学习率，从第二个epoch开始应用它 @@ -395,6 +397,71 @@ def after_epoch(self, cur_epoch, n_epoch, optimizer): print("Model reset. \nFind best lr={}".format(self.best_lr)) +class TensorboardCallback(Callback): + """ + 接受以下一个或多个字符串作为参数： + - "model" + - "loss" + - "metric" + """ + + def __init__(self, *options): + super(TensorboardCallback, self).__init__() + args = {"model", "loss", "metric"} + for opt in options: + if opt not in args: + raise ValueError("Unrecognized argument {}. Expect one of {}".format(opt, args)) + self.options = options + self._summary_writer = None + self.graph_added = False + + def before_train(self): + save_dir = self.trainer.save_path + if save_dir is None: + path = os.path.join("./", 'tensorboard_logs_{}'.format(self.trainer.start_time)) + else: + path = os.path.join(save_dir, 'tensorboard_logs_{}'.format(self.trainer.start_time)) + self._summary_writer = SummaryWriter(path) + + def before_batch(self, batch_x, batch_y, indices): + if "model" in self.options and self.graph_added is False: + # tesorboardX 这里有大bug，暂时没法画模型图 + # from fastNLP.core.utils import _build_args + # inputs = _build_args(self.trainer.model, **batch_x) + # args = tuple([value for value in inputs.values()]) + # args = args[0] if len(args) == 1 else args + # self._summary_writer.add_graph(self.trainer.model, torch.zeros(32, 2)) + self.graph_added = True + + def before_backward(self, loss, model): + if "loss" in self.options: + self._summary_writer.add_scalar("loss", loss.item(), global_step=self.trainer.step) + + if "model" in self.options: + for name, param in self.trainer.model.named_parameters(): + if param.requires_grad: + self._summary_writer.add_scalar(name + "_mean", param.mean(), global_step=self.trainer.step) + # self._summary_writer.add_scalar(name + "_std", param.std(), global_step=self.trainer.step) + self._summary_writer.add_scalar(name + "_grad_mean", param.grad.mean(), + global_step=self.trainer.step) + + def after_valid(self, eval_result, metric_key, optimizer): + if "metric" in self.options: + for name, metric in eval_result.items(): + for metric_key, metric_val in metric.items(): + self._summary_writer.add_scalar("valid_{}_{}".format(name, metric_key), metric_val, + global_step=self.trainer.step) + + def after_train(self, model): + self._summary_writer.close() + del self._summary_writer + + def on_exception(self, exception, model): + if hasattr(self, "_summary_writer"): + self._summary_writer.close() + del self._summary_writer + + if __name__ == "__main__": manager = CallbackManager(env={"n_epoch": 3}, callbacks=[DummyCallback(), DummyCallback()]) manager.before_train(10, 11, 12) diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index a5861091..b7a8f72b 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -5,7 +5,6 @@ import numpy as np import torch -from tensorboardX import SummaryWriter from torch import nn try: @@ -195,21 +194,9 @@ def train(self, load_best_model=True): self._model_device = self.model.parameters().__next__().device self._mode(self.model, is_test=False) - self.start_time = str(datetime.now().strftime('%Y-%m-%d %H-%M-%S')) + self.start_time = str(datetime.now().strftime('%Y-%m-%d-%H-%M-%S')) start_time = time.time() print("training epochs started " + self.start_time, flush=True) - if self.save_path is None: - class psudoSW: - def __getattr__(self, item): - def pass_func(*args, **kwargs): - pass - - return pass_func - - self._summary_writer = psudoSW() - else: - path = os.path.join(self.save_path, 'tensorboard_logs_{}'.format(self.start_time)) - self._summary_writer = SummaryWriter(path) try: self.callback_manager.before_train() @@ -232,8 +219,7 @@ def pass_func(*args, **kwargs): else: print("Fail to reload best model.") finally: - self._summary_writer.close() - del self._summary_writer + pass results['seconds'] = round(time.time() - start_time, 2) return results @@ -261,7 +247,7 @@ def _train(self): # negative sampling; replace unknown; re-weight batch_y self.callback_manager.before_batch(batch_x, batch_y, indices) _move_dict_value_to_device(batch_x, batch_y, device=self._model_device, - non_blocking=self.pin_memory) # pin_memory, use non_blockling. + non_blocking=self.pin_memory) # pin_memory, use non_blocking. prediction = self._data_forward(self.model, batch_x) # edit prediction @@ -279,12 +265,6 @@ def _train(self): # lr scheduler; lr_finder; one_cycle self.callback_manager.after_step(self.optimizer) - self._summary_writer.add_scalar("loss", loss.item(), global_step=self.step) - for name, param in self.model.named_parameters(): - if param.requires_grad: - self._summary_writer.add_scalar(name + "_mean", param.mean(), global_step=self.step) - # self._summary_writer.add_scalar(name + "_std", param.std(), global_step=self.step) - # self._summary_writer.add_scalar(name + "_grad_sum", param.sum(), global_step=self.step) if (self.step+1) % self.print_every == 0: if self.use_tqdm: print_output = "loss:{0:<6.5f}".format(avg_loss / self.print_every) @@ -319,10 +299,7 @@ def _train(self): def _do_validation(self, epoch, step): res = self.tester.test() - for name, metric in res.items(): - for metric_key, metric_val in metric.items(): - self._summary_writer.add_scalar("valid_{}_{}".format(name, metric_key), metric_val, - global_step=self.step) + if self._better_eval_result(res): if self.save_path is not None: self._save_model(self.model, diff --git a/test/core/test_batch.py b/test/core/test_batch.py index 29a48559..e1561942 100644 --- a/test/core/test_batch.py +++ b/test/core/test_batch.py @@ -1,3 +1,4 @@ +import time import unittest import numpy as np @@ -8,7 +9,7 @@ from fastNLP.core.dataset import construct_dataset from fastNLP.core.instance import Instance from fastNLP.core.sampler import SequentialSampler -import time + def generate_fake_dataset(num_samples=1000): """ @@ -161,12 +162,13 @@ def test_pin_memory(self): dataset = generate_fake_dataset(num_samples) batch = Batch(dataset, batch_size=batch_size, sampler=SequentialSampler(), pin_memory=True) - for batch_x, batch_y in batch: - time.sleep(pause_seconds) + # 这里发生OOM + # for batch_x, batch_y in batch: + # time.sleep(pause_seconds) num_workers = 2 batch = Batch(dataset, batch_size=batch_size, sampler=SequentialSampler(), num_workers=num_workers, pin_memory=True) - for batch_x, batch_y in batch: - time.sleep(pause_seconds) - + # 这里发生OOM + # for batch_x, batch_y in batch: + # time.sleep(pause_seconds) diff --git a/test/core/test_callbacks.py b/test/core/test_callbacks.py index d0c1fb13..74ce4876 100644 --- a/test/core/test_callbacks.py +++ b/test/core/test_callbacks.py @@ -3,7 +3,9 @@ import numpy as np import torch -from fastNLP.core.callback import EchoCallback, EarlyStopCallback, GradientClipCallback, LRScheduler, ControlC, LRFinder +from fastNLP.core.callback import EchoCallback, EarlyStopCallback, GradientClipCallback, LRScheduler, ControlC, \ + LRFinder, \ + TensorboardCallback from fastNLP.core.dataset import DataSet from fastNLP.core.instance import Instance from fastNLP.core.losses import BCELoss @@ -119,3 +121,18 @@ def test_LRFinder(self): use_tqdm=False, callbacks=[LRFinder(len(data_set) // 32)]) trainer.train() + + def test_TensorboardCallback(self): + data_set, model = prepare_env() + trainer = Trainer(data_set, model, + loss=BCELoss(pred="predict", target="y"), + n_epochs=5, + batch_size=32, + print_every=50, + optimizer=SGD(lr=0.1), + check_code_level=2, + use_tqdm=False, + dev_data=data_set, + metrics=AccuracyMetric(pred="predict", target="y"), + callbacks=[TensorboardCallback("loss", "metric")]) + trainer.train() From 47ec69ea96b484458448f4a0d0eda4de8e8b5562 Mon Sep 17 00:00:00 2001 From: yh Date: Mon, 21 Jan 2019 14:44:31 +0800 Subject: [PATCH 22/32] =?UTF-8?q?trainer=E6=A0=B9=E6=8D=AEsyf=E7=9A=84?= =?UTF-8?q?=E5=A4=9A=E8=BF=9B=E7=A8=8Bbatch=E8=BF=9B=E8=A1=8C=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/trainer.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index b7a8f72b..8ca3d22a 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -33,8 +33,8 @@ class Trainer(object): def __init__(self, train_data, model, loss=None, metrics=None, n_epochs=3, batch_size=32, print_every=50, validate_every=-1, dev_data=None, save_path=None, optimizer=Adam(lr=0.01, weight_decay=0), - check_code_level=0, metric_key=None, sampler=RandomSampler(), num_workers=0, pin_memory=False, - timeout=0, use_tqdm=True, use_cuda=False, callbacks=None): + check_code_level=0, metric_key=None, sampler=RandomSampler(), prefetch=False, use_tqdm=True, + use_cuda=False, callbacks=None): """ :param DataSet train_data: the training data :param torch.nn.modules.module model: a PyTorch model @@ -58,12 +58,7 @@ def __init__(self, train_data, model, loss=None, metrics=None, n_epochs=3, batch metric_key="-PPL" # language model gets better as perplexity gets smaller :param BaseSampler sampler: method used to generate batch data. - :param num_workers: int, 使用多少个进程来准备数据。默认为0, 即使用主线程生成数据。 特性处于实验阶段，谨慎使用。 - 如果DataSet较大，且每个batch的准备时间很短，使用多进程可能并不能提速。 - :param pin_memory: bool, 默认为False. 当设置为True时，会使用锁页内存，可能导致内存占用变多。如果内存比较充足， - 可以考虑设置为True进行加速, 当pin_memory为True时，默认使用non_blocking=True的方式将数据从cpu移动到gpu。 - :param timeout: float, 大于0的数，只有在num_workers>0时才有用。超过该时间仍然没有获取到一个batch则报错，可以用于 - 检测是否出现了batch产生阻塞的情况。 + :param prefetch: bool, 是否使用额外的进程对产生batch数据。 :param bool use_tqdm: whether to use tqdm to show train progress. :param callbacks: List[Callback]. 用于在train过程中起调节作用的回调函数。比如early stop，negative sampling等可以 通过callback机制实现。 @@ -125,9 +120,7 @@ def __init__(self, train_data, model, loss=None, metrics=None, n_epochs=3, batch self.best_dev_step = None self.best_dev_perf = None self.sampler = sampler - self.num_workers = num_workers - self.pin_memory = pin_memory - self.timeout = timeout + self.prefetch = prefetch self.callback_manager = CallbackManager(env={"trainer": self}, callbacks=callbacks) if isinstance(optimizer, torch.optim.Optimizer): @@ -236,8 +229,7 @@ def _train(self): with inner_tqdm(total=total_steps, postfix='loss:{0:<6.5f}', leave=False, dynamic_ncols=True) as pbar: avg_loss = 0 data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False, - num_workers=self.num_workers, pin_memory=self.pin_memory, timeout=self.timeout, - keep_process=True) + prefetch=self.prefetch, device=self._model_device) for epoch in range(1, self.n_epochs+1): pbar.set_description_str(desc="Epoch {}/{}".format(epoch, self.n_epochs)) # early stopping @@ -246,8 +238,6 @@ def _train(self): indices = data_iterator.get_batch_indices() # negative sampling; replace unknown; re-weight batch_y self.callback_manager.before_batch(batch_x, batch_y, indices) - _move_dict_value_to_device(batch_x, batch_y, device=self._model_device, - non_blocking=self.pin_memory) # pin_memory, use non_blocking. prediction = self._data_forward(self.model, batch_x) # edit prediction From a37de4344d4e84dc650d5472bf1f43e1249d561f Mon Sep 17 00:00:00 2001 From: yunfan Date: Mon, 21 Jan 2019 14:50:41 +0800 Subject: [PATCH 23/32] add batch device --- fastNLP/core/batch.py | 31 +++++++++++++++++++++++++------ fastNLP/core/trainer.py | 12 ++++++++---- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/fastNLP/core/batch.py b/fastNLP/core/batch.py index 3faab8c0..ead7087e 100644 --- a/fastNLP/core/batch.py +++ b/fastNLP/core/batch.py @@ -16,10 +16,12 @@ class Batch(object): :param int batch_size: the size of the batch :param Sampler sampler: a Sampler object :param bool as_numpy: If True, return Numpy array. Otherwise, return torch tensors. - + :param bool prefetch: If True, use multiprocessing to fetch next batch when training. + :param str or torch.device device: the batch's device, if as_numpy is True, device is ignored. """ - def __init__(self, dataset, batch_size, sampler=RandomSampler(), as_numpy=False): + def __init__(self, dataset, batch_size, sampler=RandomSampler(), as_numpy=False, prefetch=False, + device='cpu'): self.dataset = dataset self.batch_size = batch_size self.sampler = sampler @@ -28,6 +30,10 @@ def __init__(self, dataset, batch_size, sampler=RandomSampler(), as_numpy=False) self.curidx = 0 self.num_batches = len(dataset) // batch_size + int(len(dataset) % batch_size != 0) self.cur_batch_indices = None + self.prefetch = prefetch + self.lengths = 0 + if not as_numpy: + self.device = device if isinstance(device, torch.device) else torch.device(device) def fetch_one(self): if self.curidx >= len(self.idx_list): @@ -44,6 +50,7 @@ def fetch_one(self): batch = field.get(indices) if not self.as_numpy and field.padder is not None: batch = to_tensor(batch, field.dtype) + batch = batch.to(self.device) if field.is_target: batch_y[field_name] = batch if field.is_input: @@ -57,7 +64,21 @@ def __iter__(self): Iterate on dataset, fetch batch data. Fetch process don't block the iterate process :return: """ - return run_batch_iter(self) + if self.prefetch: + return run_batch_iter(self) + def batch_iter(): + self.init_iter() + while 1: + res = self.fetch_one() + if res is None: + break + yield res + return batch_iter() + + def init_iter(self): + self.idx_list = self.sampler(self.dataset) + self.curidx = 0 + self.lengths = self.dataset.get_length() def __len__(self): return self.num_batches @@ -78,9 +99,7 @@ def to_tensor(batch, dtype): def run_fetch(batch, q): - batch.idx_list = batch.sampler(batch.dataset) - batch.curidx = 0 - batch.lengths = batch.dataset.get_length() + batch.init_iter() # print('start fetch') while 1: res = batch.fetch_one() diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index faa0d0a2..a5861091 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -34,8 +34,8 @@ class Trainer(object): def __init__(self, train_data, model, loss=None, metrics=None, n_epochs=3, batch_size=32, print_every=50, validate_every=-1, dev_data=None, save_path=None, optimizer=Adam(lr=0.01, weight_decay=0), - check_code_level=0, metric_key=None, sampler=RandomSampler(), num_workers=0, - use_tqdm=True, use_cuda=False, callbacks=None): + check_code_level=0, metric_key=None, sampler=RandomSampler(), num_workers=0, pin_memory=False, + timeout=0, use_tqdm=True, use_cuda=False, callbacks=None): """ :param DataSet train_data: the training data :param torch.nn.modules.module model: a PyTorch model @@ -127,6 +127,8 @@ def __init__(self, train_data, model, loss=None, metrics=None, n_epochs=3, batch self.best_dev_perf = None self.sampler = sampler self.num_workers = num_workers + self.pin_memory = pin_memory + self.timeout = timeout self.callback_manager = CallbackManager(env={"trainer": self}, callbacks=callbacks) if isinstance(optimizer, torch.optim.Optimizer): @@ -247,7 +249,9 @@ def _train(self): len(self.train_data) % self.batch_size != 0)) * self.n_epochs with inner_tqdm(total=total_steps, postfix='loss:{0:<6.5f}', leave=False, dynamic_ncols=True) as pbar: avg_loss = 0 - data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False) + data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False, + num_workers=self.num_workers, pin_memory=self.pin_memory, timeout=self.timeout, + keep_process=True) for epoch in range(1, self.n_epochs+1): pbar.set_description_str(desc="Epoch {}/{}".format(epoch, self.n_epochs)) # early stopping @@ -257,7 +261,7 @@ def _train(self): # negative sampling; replace unknown; re-weight batch_y self.callback_manager.before_batch(batch_x, batch_y, indices) _move_dict_value_to_device(batch_x, batch_y, device=self._model_device, - non_blocking=self.use_cuda) # pin_memory, use non_blockling. + non_blocking=self.pin_memory) # pin_memory, use non_blockling. prediction = self._data_forward(self.model, batch_x) # edit prediction From 9474ab4b341c0c81c7350b37dcf1b06bc509b7bb Mon Sep 17 00:00:00 2001 From: yunfan Date: Mon, 21 Jan 2019 22:28:31 +0800 Subject: [PATCH 24/32] remove device in batch --- fastNLP/core/batch.py | 24 +++++++++++++----------- fastNLP/core/trainer.py | 3 ++- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/fastNLP/core/batch.py b/fastNLP/core/batch.py index ead7087e..88d9185d 100644 --- a/fastNLP/core/batch.py +++ b/fastNLP/core/batch.py @@ -20,8 +20,7 @@ class Batch(object): :param str or torch.device device: the batch's device, if as_numpy is True, device is ignored. """ - def __init__(self, dataset, batch_size, sampler=RandomSampler(), as_numpy=False, prefetch=False, - device='cpu'): + def __init__(self, dataset, batch_size, sampler=RandomSampler(), as_numpy=False, prefetch=False): self.dataset = dataset self.batch_size = batch_size self.sampler = sampler @@ -32,8 +31,6 @@ def __init__(self, dataset, batch_size, sampler=RandomSampler(), as_numpy=False, self.cur_batch_indices = None self.prefetch = prefetch self.lengths = 0 - if not as_numpy: - self.device = device if isinstance(device, torch.device) else torch.device(device) def fetch_one(self): if self.curidx >= len(self.idx_list): @@ -50,7 +47,6 @@ def fetch_one(self): batch = field.get(indices) if not self.as_numpy and field.padder is not None: batch = to_tensor(batch, field.dtype) - batch = batch.to(self.device) if field.is_target: batch_y[field_name] = batch if field.is_input: @@ -119,12 +115,18 @@ def run_batch_iter(batch): fetch_p.start() # print('fork fetch process') while 1: - res = q.get() - q.task_done() - # print('get fetched') - if res is None: - break - yield res + try: + res = q.get(timeout=1) + q.task_done() + # print('get fetched') + if res is None: + break + yield res + except Exception as e: + if fetch_p.is_alive(): + continue + else: + break fetch_p.terminate() fetch_p.join() # print('iter done') diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index 8ca3d22a..8112af88 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -229,12 +229,13 @@ def _train(self): with inner_tqdm(total=total_steps, postfix='loss:{0:<6.5f}', leave=False, dynamic_ncols=True) as pbar: avg_loss = 0 data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False, - prefetch=self.prefetch, device=self._model_device) + prefetch=self.prefetch) for epoch in range(1, self.n_epochs+1): pbar.set_description_str(desc="Epoch {}/{}".format(epoch, self.n_epochs)) # early stopping self.callback_manager.before_epoch(epoch, self.n_epochs) for batch_x, batch_y in data_iterator: + _move_dict_value_to_device(batch_x, batch_y, device=self._model_device) indices = data_iterator.get_batch_indices() # negative sampling; replace unknown; re-weight batch_y self.callback_manager.before_batch(batch_x, batch_y, indices) From d4b4ffa28bbf3a76cdf05330bc37bdf683a1d0e5 Mon Sep 17 00:00:00 2001 From: xuyige Date: Wed, 23 Jan 2019 14:56:25 +0800 Subject: [PATCH 25/32] add testing tutorial --- tutorials/fastnlp_test_tutorial.ipynb | 97 +++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 tutorials/fastnlp_test_tutorial.ipynb diff --git a/tutorials/fastnlp_test_tutorial.ipynb b/tutorials/fastnlp_test_tutorial.ipynb new file mode 100644 index 00000000..9b0c1b2e --- /dev/null +++ b/tutorials/fastnlp_test_tutorial.ipynb @@ -0,0 +1,97 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## fastNLP测试说明\n", + "### 测试环境\n", + "fastNLP使用pytest对代码进行单元测试，测试代码在test文件夹下，测试所需数据在test/data_for_tests文件夹下\n", + "测试的步骤主要分为准备数据，执行测试，比对结果，清除环境四步\n", + "测试代码以test_xxx.py命名，以DataSet的测试代码为例，测试代码文件名为test_dataset.py" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import unittest # 单元测试需要用到unittest\n", + "\n", + "from fastNLP.core.dataset import DataSet\n", + "from fastNLP.core.fieldarray import FieldArray\n", + "from fastNLP.core.instance import Instance\n", + "# 在这个单元测试文件中，需要测试DataSet、FieldArray、以及Instance" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class TestDataSet(unittest.TestCase): # 类名字以Test打头，继承unittest.TestCase\n", + "\n", + " def test_init_v1(self): # 测试样例1， 函数名称以test_打头\n", + " # 该测试样例测试的是DataSet的初始化\n", + " ins = Instance(x=[1, 2, 3, 4], y=[5, 6]) # 准备数据\n", + " ds = DataSet([ins] * 40) # 执行测试（调用DataSet的初始化函数）\n", + " self.assertTrue(\"x\" in ds.field_arrays and \"y\" in ds.field_arrays) # 比对结果：'x'跟'y'都是ds的field\n", + " self.assertEqual(ds.field_arrays[\"x\"].content, [[1, 2, 3, 4], ] * 40) # 比对结果： field 'x'的内容正确\n", + " self.assertEqual(ds.field_arrays[\"y\"].content, [[5, 6], ] * 40) # 比对结果： field 'y'的内容正确\n", + " \n", + " def test_init_v2(self): # 测试样例2，该样例测试DataSet的另一种初始化方式\n", + " ds = DataSet({\"x\": [[1, 2, 3, 4]] * 40, \"y\": [[5, 6]] * 40})\n", + " self.assertTrue(\"x\" in ds.field_arrays and \"y\" in ds.field_arrays)\n", + " self.assertEqual(ds.field_arrays[\"x\"].content, [[1, 2, 3, 4], ] * 40)\n", + " self.assertEqual(ds.field_arrays[\"y\"].content, [[5, 6], ] * 40)\n", + " \n", + " def test_init_assert(self): # 测试样例3，该样例测试不规范初始化DataSet时是否会报正确错误\n", + " with self.assertRaises(AssertionError):\n", + " _ = DataSet({\"x\": [[1, 2, 3, 4]] * 40, \"y\": [[5, 6]] * 100})\n", + " with self.assertRaises(AssertionError):\n", + " _ = DataSet([[1, 2, 3, 4]] * 10)\n", + " with self.assertRaises(ValueError):\n", + " _ = DataSet(0.00001)\n", + " \n", + " def test_contains(self): # 测试样例4，该样例测试DataSet的contains函数，是功能测试\n", + " ds = DataSet({\"x\": [[1, 2, 3, 4]] * 40, \"y\": [[5, 6]] * 40})\n", + " self.assertTrue(\"x\" in ds)\n", + " self.assertTrue(\"y\" in ds)\n", + " self.assertFalse(\"z\" in ds)\n", + " \n", + " # 更多测试样例见test/core/test_dataset.py" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From e0d6a259ae75ba7fdf4a37f8560a8097a1081f9d Mon Sep 17 00:00:00 2001 From: xuyige Date: Wed, 23 Jan 2019 17:09:28 +0800 Subject: [PATCH 26/32] skip training while n_epoch in trainer is not greater than 0 --- fastNLP/core/trainer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index 8112af88..ed2f366b 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -181,6 +181,10 @@ def train(self, load_best_model=True): """ results = {} + if self.n_epochs <= 0: + print(f"training epoch is {self.n_epochs}, nothing was done.") + results['seconds'] = 0. + return results try: if torch.cuda.is_available() and self.use_cuda: self.model = self.model.cuda() From 887fc9281fbfe7836fcae435e10954c6e9f26a4d Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Fri, 25 Jan 2019 21:43:24 +0800 Subject: [PATCH 27/32] update callbacks: * rename callback methods. Use fastai's notation. * add a new callback method - on_valid_begin --- fastNLP/core/callback.py | 97 ++++++++++++++++------------- fastNLP/core/predictor.py | 68 +++++++++++++------- fastNLP/core/trainer.py | 26 ++++---- reproduction/Biaffine_parser/run.py | 7 +-- test/core/test_predictor.py | 27 ++++++-- 5 files changed, 131 insertions(+), 94 deletions(-) diff --git a/fastNLP/core/callback.py b/fastNLP/core/callback.py index 48d7333c..b1a480cc 100644 --- a/fastNLP/core/callback.py +++ b/fastNLP/core/callback.py @@ -17,37 +17,40 @@ def __init__(self): super(Callback, self).__init__() self.trainer = None # 在Trainer内部被重新赋值 - def before_train(self): + def on_train_begin(self): # before the main training loop pass - def before_epoch(self, cur_epoch, total_epoch): + def on_epoch_begin(self, cur_epoch, total_epoch): # at the beginning of each epoch pass - def before_batch(self, batch_x, batch_y, indices): + def on_batch_begin(self, batch_x, batch_y, indices): # at the beginning of each step/mini-batch pass - def before_loss(self, batch_y, predict_y): + def on_loss_begin(self, batch_y, predict_y): # after data_forward, and before loss computation pass - def before_backward(self, loss, model): + def on_backward_begin(self, loss, model): # after loss computation, and before gradient backward pass - def after_backward(self, model): + def on_backward_end(self, model): pass - def after_step(self, optimizer): + def on_step_end(self, optimizer): pass - def after_batch(self, *args): + def on_batch_end(self, *args): # at the end of each step/mini-batch pass - def after_valid(self, eval_result, metric_key, optimizer): + def on_valid_begin(self): + pass + + def on_valid_end(self, eval_result, metric_key, optimizer): """ 每次执行验证机的evaluation后会调用。传入eval_result @@ -58,7 +61,7 @@ def after_valid(self, eval_result, metric_key, optimizer): """ pass - def after_epoch(self, cur_epoch, n_epoch, optimizer): + def on_epoch_end(self, cur_epoch, n_epoch, optimizer): """ 每个epoch结束将会调用该方法 @@ -69,7 +72,7 @@ def after_epoch(self, cur_epoch, n_epoch, optimizer): """ pass - def after_train(self, model): + def on_train_end(self, model): """ 训练结束，调用该方法 @@ -134,47 +137,51 @@ def __init__(self, env, callbacks=None): raise TypeError(f"Expect callbacks in CallbackManager(callbacks) to be list. Got {type(callbacks)}.") @transfer - def before_train(self): + def on_train_begin(self): + pass + + @transfer + def on_epoch_begin(self, cur_epoch, total_epoch): pass @transfer - def before_epoch(self, cur_epoch, total_epoch): + def on_batch_begin(self, batch_x, batch_y, indices): pass @transfer - def before_batch(self, batch_x, batch_y, indices): + def on_loss_begin(self, batch_y, predict_y): pass @transfer - def before_loss(self, batch_y, predict_y): + def on_backward_begin(self, loss, model): pass @transfer - def before_backward(self, loss, model): + def on_backward_end(self, model): pass @transfer - def after_backward(self, model): + def on_step_end(self, optimizer): pass @transfer - def after_step(self, optimizer): + def on_batch_end(self): pass @transfer - def after_batch(self): + def on_valid_begin(self): pass @transfer - def after_valid(self, eval_result, metric_key, optimizer): + def on_valid_end(self, eval_result, metric_key, optimizer): pass @transfer - def after_epoch(self, cur_epoch, n_epoch, optimizer): + def on_epoch_end(self, cur_epoch, n_epoch, optimizer): pass @transfer - def after_train(self, model): + def on_train_end(self, model): pass @transfer @@ -183,36 +190,36 @@ def on_exception(self, exception, model): class DummyCallback(Callback): - def before_train(self, *arg): + def on_train_begin(self, *arg): print(arg) - def after_epoch(self, cur_epoch, n_epoch, optimizer): + def on_epoch_end(self, cur_epoch, n_epoch, optimizer): print(cur_epoch, n_epoch, optimizer) class EchoCallback(Callback): - def before_train(self): + def on_train_begin(self): print("before_train") - def before_epoch(self, cur_epoch, total_epoch): + def on_epoch_begin(self, cur_epoch, total_epoch): print("before_epoch") - def before_batch(self, batch_x, batch_y, indices): + def on_batch_begin(self, batch_x, batch_y, indices): print("before_batch") - def before_loss(self, batch_y, predict_y): + def on_loss_begin(self, batch_y, predict_y): print("before_loss") - def before_backward(self, loss, model): + def on_backward_begin(self, loss, model): print("before_backward") - def after_batch(self): + def on_batch_end(self): print("after_batch") - def after_epoch(self, cur_epoch, n_epoch, optimizer): + def on_epoch_end(self, cur_epoch, n_epoch, optimizer): print("after_epoch") - def after_train(self, model): + def on_train_end(self, model): print("after_train") @@ -240,7 +247,7 @@ def __init__(self, parameters=None, clip_value=1, clip_type='norm'): self.parameters = parameters self.clip_value = clip_value - def after_backward(self, model): + def on_backward_end(self, model): self.clip_fun(model.parameters(), self.clip_value) @@ -266,7 +273,7 @@ def __init__(self, patience): self.wait = 0 self.epoch = 0 - def after_valid(self, eval_result, metric_key, optimizer): + def on_valid_end(self, eval_result, metric_key, optimizer): self.epoch += 1 if not self.trainer._better_eval_result(eval_result): # current result is getting worse @@ -297,7 +304,7 @@ def __init__(self, lr_scheduler): else: raise ValueError(f"Expect torch.optim.lr_scheduler for LRScheduler. Got {type(lr_scheduler)}.") - def before_epoch(self, cur_epoch, total_epoch): + def on_epoch_begin(self, cur_epoch, total_epoch): self.scheduler.step() print("scheduler step ", "lr=", self.trainer.optimizer.param_groups[0]["lr"]) @@ -359,7 +366,7 @@ def __init__(self, n_batch, start_lr=1e-6, end_lr=10): self.find = None self.loader = ModelLoader() - def before_epoch(self, cur_epoch, total_epoch): + def on_epoch_begin(self, cur_epoch, total_epoch): if cur_epoch == 1: self.opt = self.trainer.optimizer # pytorch optimizer self.opt.param_groups[0]["lr"] = self.start_lr @@ -367,7 +374,7 @@ def before_epoch(self, cur_epoch, total_epoch): ModelSaver("tmp").save_pytorch(self.trainer.model, param_only=True) self.find = True - def before_backward(self, loss, model): + def on_backward_begin(self, loss, model): if self.find: if torch.isnan(loss) or self.stop is True: self.stop = True @@ -379,7 +386,7 @@ def before_backward(self, loss, model): self.best_loss = self.smooth_value.smooth self.best_lr = self.opt.param_groups[0]["lr"] - def after_batch(self, *args): + def on_batch_end(self, *args): if self.find: lr = next(self.lr_gen, None) if lr is None or self.stop is True or self.loss_history[-1] > 4 * self.best_loss: @@ -388,7 +395,7 @@ def after_batch(self, *args): self.opt.param_groups[0]["lr"] = lr # self.loader.load_pytorch(self.trainer.model, "tmp") - def after_epoch(self, cur_epoch, n_epoch, optimizer): + def on_epoch_end(self, cur_epoch, n_epoch, optimizer): if cur_epoch == 1: self.opt.param_groups[0]["lr"] = self.best_lr self.find = False @@ -415,7 +422,7 @@ def __init__(self, *options): self._summary_writer = None self.graph_added = False - def before_train(self): + def on_train_begin(self): save_dir = self.trainer.save_path if save_dir is None: path = os.path.join("./", 'tensorboard_logs_{}'.format(self.trainer.start_time)) @@ -423,7 +430,7 @@ def before_train(self): path = os.path.join(save_dir, 'tensorboard_logs_{}'.format(self.trainer.start_time)) self._summary_writer = SummaryWriter(path) - def before_batch(self, batch_x, batch_y, indices): + def on_batch_begin(self, batch_x, batch_y, indices): if "model" in self.options and self.graph_added is False: # tesorboardX 这里有大bug，暂时没法画模型图 # from fastNLP.core.utils import _build_args @@ -433,7 +440,7 @@ def before_batch(self, batch_x, batch_y, indices): # self._summary_writer.add_graph(self.trainer.model, torch.zeros(32, 2)) self.graph_added = True - def before_backward(self, loss, model): + def on_backward_begin(self, loss, model): if "loss" in self.options: self._summary_writer.add_scalar("loss", loss.item(), global_step=self.trainer.step) @@ -445,14 +452,14 @@ def before_backward(self, loss, model): self._summary_writer.add_scalar(name + "_grad_mean", param.grad.mean(), global_step=self.trainer.step) - def after_valid(self, eval_result, metric_key, optimizer): + def on_valid_end(self, eval_result, metric_key, optimizer): if "metric" in self.options: for name, metric in eval_result.items(): for metric_key, metric_val in metric.items(): self._summary_writer.add_scalar("valid_{}_{}".format(name, metric_key), metric_val, global_step=self.trainer.step) - def after_train(self, model): + def on_train_end(self, model): self._summary_writer.close() del self._summary_writer @@ -464,5 +471,5 @@ def on_exception(self, exception, model): if __name__ == "__main__": manager = CallbackManager(env={"n_epoch": 3}, callbacks=[DummyCallback(), DummyCallback()]) - manager.before_train(10, 11, 12) + manager.on_train_begin(10, 11, 12) # print(manager.after_epoch()) diff --git a/fastNLP/core/predictor.py b/fastNLP/core/predictor.py index de9ddc8c..ae648e47 100644 --- a/fastNLP/core/predictor.py +++ b/fastNLP/core/predictor.py @@ -1,7 +1,11 @@ +from collections import defaultdict + import torch -from fastNLP.core.batch import Batch -from fastNLP.core.sampler import SequentialSampler +from fastNLP.core import Batch +from fastNLP.core import DataSet +from fastNLP.core import SequentialSampler +from fastNLP.core.utils import _build_args class Predictor(object): @@ -13,37 +17,55 @@ class Predictor(object): Currently, Predictor does not support GPU. """ - def __init__(self): + def __init__(self, network): + if not isinstance(network, torch.nn.Module): + raise ValueError( + "Only fastNLP.models.BaseModel or torch.nn,Module is allowed, not {}".format(type(network))) + self.network = network self.batch_size = 1 self.batch_output = [] - def predict(self, network, data): + def predict(self, data, seq_len_field_name=None): """Perform inference using the trained model. - :param network: a PyTorch model (cpu) :param data: a DataSet object. + :param str seq_len_field_name: field name indicating sequence lengths :return: list of batch outputs """ - # turn on the testing mode; clean up the history - self.mode(network, test=True) - batch_output = [] + if not isinstance(data, DataSet): + raise ValueError("Only Dataset class is allowed, not {}.".format(type(data))) + if seq_len_field_name is not None and seq_len_field_name not in data.field_arrays: + raise ValueError("Field name {} not found in DataSet {}.".format(seq_len_field_name, data)) - data_iterator = Batch(data, batch_size=self.batch_size, sampler=SequentialSampler(), as_numpy=False) + self.network.eval() + batch_output = defaultdict(list) + data_iterator = Batch(data, batch_size=self.batch_size, sampler=SequentialSampler(), as_numpy=False, + prefetch=False) - for batch_x, _ in data_iterator: - with torch.no_grad(): - prediction = self.data_forward(network, batch_x) - batch_output.append(prediction) + if hasattr(self.network, "predict"): + predict_func = self.network.predict + else: + predict_func = self.network.forward - return batch_output + with torch.no_grad(): + for batch_x, _ in data_iterator: + refined_batch_x = _build_args(predict_func, **batch_x) + prediction = predict_func(**refined_batch_x) - def mode(self, network, test=True): - if test: - network.eval() - else: - network.train() + if seq_len_field_name is not None: + seq_lens = batch_x[seq_len_field_name].tolist() + + for key, value in prediction.items(): + value = value.cpu().numpy() + if len(value.shape) == 1 or (len(value.shape) == 2 and value.shape[1] == 1): + batch_output[key].extend(value.tolist()) + else: + if seq_len_field_name is not None: + tmp_batch = [] + for idx, seq_len in enumerate(seq_lens): + tmp_batch.append(value[idx, :seq_len]) + batch_output[key].extend(tmp_batch) + else: + batch_output[key].append(value) - def data_forward(self, network, x): - """Forward through network.""" - y = network(**x) - return y + return batch_output diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index ed2f366b..ddd35b28 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -196,9 +196,9 @@ def train(self, load_best_model=True): print("training epochs started " + self.start_time, flush=True) try: - self.callback_manager.before_train() + self.callback_manager.on_train_begin() self._train() - self.callback_manager.after_train(self.model) + self.callback_manager.on_train_end(self.model) except (CallbackException, KeyboardInterrupt) as e: self.callback_manager.on_exception(e, self.model) @@ -237,28 +237,26 @@ def _train(self): for epoch in range(1, self.n_epochs+1): pbar.set_description_str(desc="Epoch {}/{}".format(epoch, self.n_epochs)) # early stopping - self.callback_manager.before_epoch(epoch, self.n_epochs) + self.callback_manager.on_epoch_begin(epoch, self.n_epochs) for batch_x, batch_y in data_iterator: _move_dict_value_to_device(batch_x, batch_y, device=self._model_device) indices = data_iterator.get_batch_indices() # negative sampling; replace unknown; re-weight batch_y - self.callback_manager.before_batch(batch_x, batch_y, indices) + self.callback_manager.on_batch_begin(batch_x, batch_y, indices) prediction = self._data_forward(self.model, batch_x) # edit prediction - self.callback_manager.before_loss(batch_y, prediction) + self.callback_manager.on_loss_begin(batch_y, prediction) loss = self._compute_loss(prediction, batch_y) avg_loss += loss.item() # Is loss NaN or inf? requires_grad = False - self.callback_manager.before_backward(loss, self.model) + self.callback_manager.on_backward_begin(loss, self.model) self._grad_backward(loss) - # gradient clipping - self.callback_manager.after_backward(self.model) + self.callback_manager.on_backward_end(self.model) self._update() - # lr scheduler; lr_finder; one_cycle - self.callback_manager.after_step(self.optimizer) + self.callback_manager.on_step_end(self.optimizer) if (self.step+1) % self.print_every == 0: if self.use_tqdm: @@ -272,8 +270,7 @@ def _train(self): pbar.set_postfix_str(print_output) avg_loss = 0 self.step += 1 - # do nothing - self.callback_manager.after_batch() + self.callback_manager.on_batch_end() if ((self.validate_every > 0 and self.step % self.validate_every == 0) or (self.validate_every < 0 and self.step % len(data_iterator) == 0)) \ @@ -287,12 +284,13 @@ def _train(self): # ================= mini-batch end ==================== # # lr decay; early stopping - self.callback_manager.after_epoch(epoch, self.n_epochs, self.optimizer) + self.callback_manager.on_epoch_end(epoch, self.n_epochs, self.optimizer) # =============== epochs end =================== # pbar.close() # ============ tqdm end ============== # def _do_validation(self, epoch, step): + self.callback_manager.on_valid_begin() res = self.tester.test() if self._better_eval_result(res): @@ -305,7 +303,7 @@ def _do_validation(self, epoch, step): self.best_dev_epoch = epoch self.best_dev_step = step # get validation results; adjust optimizer - self.callback_manager.after_valid(res, self.metric_key, self.optimizer) + self.callback_manager.on_valid_end(res, self.metric_key, self.optimizer) return res def _mode(self, model, is_test=False): diff --git a/reproduction/Biaffine_parser/run.py b/reproduction/Biaffine_parser/run.py index 98ef02fa..c226ce69 100644 --- a/reproduction/Biaffine_parser/run.py +++ b/reproduction/Biaffine_parser/run.py @@ -4,19 +4,14 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) import fastNLP -import torch from fastNLP.core.trainer import Trainer from fastNLP.core.instance import Instance from fastNLP.api.pipeline import Pipeline from fastNLP.models.biaffine_parser import BiaffineParser, ParserMetric, ParserLoss -from fastNLP.core.vocabulary import Vocabulary -from fastNLP.core.dataset import DataSet from fastNLP.core.tester import Tester from fastNLP.io.config_io import ConfigLoader, ConfigSection from fastNLP.io.model_io import ModelLoader -from fastNLP.io.embed_loader import EmbedLoader -from fastNLP.io.model_io import ModelSaver from fastNLP.io.dataset_loader import ConllxDataLoader from fastNLP.api.processor import * from fastNLP.io.embed_loader import EmbedLoader @@ -172,7 +167,7 @@ def train(path): model.pos_embedding.weight.data[pos_v.padding_idx].fill_(0) class MyCallback(Callback): - def after_step(self, optimizer): + def on_step_end(self, optimizer): step = self.trainer.step # learning rate decay if step > 0 and step % 1000 == 0: diff --git a/test/core/test_predictor.py b/test/core/test_predictor.py index 8be5f289..c779e3ac 100644 --- a/test/core/test_predictor.py +++ b/test/core/test_predictor.py @@ -1,4 +1,5 @@ import unittest +from collections import defaultdict import numpy as np import torch @@ -23,12 +24,26 @@ def prepare_fake_dataset(): return data_set +class LinearModel(torch.nn.Module): + def __init__(self): + super(LinearModel, self).__init__() + self.linear = Linear(2, 1) + + def forward(self, x): + return {"predict": self.linear(x)} + + class TestPredictor(unittest.TestCase): - def test(self): - predictor = Predictor() - model = Linear(2, 1) + def test_simple(self): + model = LinearModel() + predictor = Predictor(model) data = prepare_fake_dataset() data.set_input("x") - ans = predictor.predict(model, data) - self.assertEqual(len(ans), 2000) - self.assertTrue(isinstance(ans[0], torch.Tensor)) + ans = predictor.predict(data) + self.assertTrue(isinstance(ans, defaultdict)) + self.assertTrue("predict" in ans) + self.assertTrue(isinstance(ans["predict"], list)) + + def test_sequence(self): + # test sequence input/output + pass From bfaf09df8cba78e02ad2aa73dab11c5ff6d7a7b9 Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Tue, 29 Jan 2019 20:35:12 +0800 Subject: [PATCH 28/32] add BERT model * load pre-trained BERT weights from local binary * add tests --- fastNLP/models/bert.py | 342 ++++++++++++++++++++++++ fastNLP/modules/aggregator/attention.py | 17 +- fastNLP/modules/encoder/transformer.py | 1 - test/models/test_bert.py | 21 ++ 4 files changed, 372 insertions(+), 9 deletions(-) create mode 100644 fastNLP/models/bert.py create mode 100644 test/models/test_bert.py diff --git a/fastNLP/models/bert.py b/fastNLP/models/bert.py new file mode 100644 index 00000000..754d1bbb --- /dev/null +++ b/fastNLP/models/bert.py @@ -0,0 +1,342 @@ +import copy +import json +import math +import os + +import torch +from torch import nn + +CONFIG_FILE = 'bert_config.json' +MODEL_WEIGHTS = 'pytorch_model.bin' + + +def gelu(x): + return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) + + +def swish(x): + return x * torch.sigmoid(x) + + +ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish} + + +class BertLayerNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-12): + super(BertLayerNorm, self).__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.bias = nn.Parameter(torch.zeros(hidden_size)) + self.variance_epsilon = eps + + def forward(self, x): + u = x.mean(-1, keepdim=True) + s = (x - u).pow(2).mean(-1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.variance_epsilon) + return self.weight * x + self.bias + + +class BertEmbeddings(nn.Module): + def __init__(self, vocab_size, hidden_size, max_position_embeddings, type_vocab_size, hidden_dropout_prob): + super(BertEmbeddings, self).__init__() + self.word_embeddings = nn.Embedding(vocab_size, hidden_size) + self.position_embeddings = nn.Embedding(max_position_embeddings, hidden_size) + self.token_type_embeddings = nn.Embedding(type_vocab_size, hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = BertLayerNorm(hidden_size, eps=1e-12) + self.dropout = nn.Dropout(hidden_dropout_prob) + + def forward(self, input_ids, token_type_ids=None): + seq_length = input_ids.size(1) + position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + words_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +class BertSelfAttention(nn.Module): + def __init__(self, hidden_size, num_attention_heads, attention_probs_dropout_prob): + super(BertSelfAttention, self).__init__() + if hidden_size % num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (hidden_size, num_attention_heads)) + self.num_attention_heads = num_attention_heads + self.attention_head_size = int(hidden_size / num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(hidden_size, self.all_head_size) + self.key = nn.Linear(hidden_size, self.all_head_size) + self.value = nn.Linear(hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(attention_probs_dropout_prob) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, hidden_states, attention_mask): + mixed_query_layer = self.query(hidden_states) + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + context_layer = torch.matmul(attention_probs, value_layer) + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + context_layer = context_layer.view(*new_context_layer_shape) + return context_layer + + +class BertSelfOutput(nn.Module): + def __init__(self, hidden_size, hidden_dropout_prob): + super(BertSelfOutput, self).__init__() + self.dense = nn.Linear(hidden_size, hidden_size) + self.LayerNorm = BertLayerNorm(hidden_size, eps=1e-12) + self.dropout = nn.Dropout(hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertAttention(nn.Module): + def __init__(self, hidden_size, num_attention_heads, attention_probs_dropout_prob, hidden_dropout_prob): + super(BertAttention, self).__init__() + self.self = BertSelfAttention(hidden_size, num_attention_heads, attention_probs_dropout_prob) + self.output = BertSelfOutput(hidden_size, hidden_dropout_prob) + + def forward(self, input_tensor, attention_mask): + self_output = self.self(input_tensor, attention_mask) + attention_output = self.output(self_output, input_tensor) + return attention_output + + +class BertIntermediate(nn.Module): + def __init__(self, hidden_size, intermediate_size, hidden_act): + super(BertIntermediate, self).__init__() + self.dense = nn.Linear(hidden_size, intermediate_size) + self.intermediate_act_fn = ACT2FN[hidden_act] \ + if isinstance(hidden_act, str) else hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class BertOutput(nn.Module): + def __init__(self, hidden_size, intermediate_size, hidden_dropout_prob): + super(BertOutput, self).__init__() + self.dense = nn.Linear(intermediate_size, hidden_size) + self.LayerNorm = BertLayerNorm(hidden_size, eps=1e-12) + self.dropout = nn.Dropout(hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertLayer(nn.Module): + def __init__(self, hidden_size, num_attention_heads, attention_probs_dropout_prob, hidden_dropout_prob, + intermediate_size, hidden_act): + super(BertLayer, self).__init__() + self.attention = BertAttention(hidden_size, num_attention_heads, attention_probs_dropout_prob, + hidden_dropout_prob) + self.intermediate = BertIntermediate(hidden_size, intermediate_size, hidden_act) + self.output = BertOutput(hidden_size, intermediate_size, hidden_dropout_prob) + + def forward(self, hidden_states, attention_mask): + attention_output = self.attention(hidden_states, attention_mask) + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class BertEncoder(nn.Module): + def __init__(self, num_hidden_layers, hidden_size, num_attention_heads, attention_probs_dropout_prob, + hidden_dropout_prob, + intermediate_size, hidden_act): + super(BertEncoder, self).__init__() + layer = BertLayer(hidden_size, num_attention_heads, attention_probs_dropout_prob, hidden_dropout_prob, + intermediate_size, hidden_act) + self.layer = nn.ModuleList([copy.deepcopy(layer) for _ in range(num_hidden_layers)]) + + def forward(self, hidden_states, attention_mask, output_all_encoded_layers=True): + all_encoder_layers = [] + for layer_module in self.layer: + hidden_states = layer_module(hidden_states, attention_mask) + if output_all_encoded_layers: + all_encoder_layers.append(hidden_states) + if not output_all_encoded_layers: + all_encoder_layers.append(hidden_states) + return all_encoder_layers + + +class BertPooler(nn.Module): + def __init__(self, hidden_size): + super(BertPooler, self).__init__() + self.dense = nn.Linear(hidden_size, hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class BertModel(nn.Module): + """BERT model ("Bidirectional Embedding Representations from a Transformer"). + + """ + + def __init__(self, vocab_size, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, **kwargs): + super(BertModel, self).__init__() + self.embeddings = BertEmbeddings(vocab_size, hidden_size, max_position_embeddings, + type_vocab_size, hidden_dropout_prob) + self.encoder = BertEncoder(num_hidden_layers, hidden_size, num_attention_heads, + attention_probs_dropout_prob, hidden_dropout_prob, intermediate_size, + hidden_act) + self.pooler = BertPooler(hidden_size) + self.initializer_range = initializer_range + + self.apply(self.init_bert_weights) + + def init_bert_weights(self, module): + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=self.initializer_range) + elif isinstance(module, BertLayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + def forward(self, input_ids, token_type_ids=None, attention_mask=None, output_all_encoded_layers=True): + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + embedding_output = self.embeddings(input_ids, token_type_ids) + encoded_layers = self.encoder(embedding_output, + extended_attention_mask, + output_all_encoded_layers=output_all_encoded_layers) + sequence_output = encoded_layers[-1] + pooled_output = self.pooler(sequence_output) + if not output_all_encoded_layers: + encoded_layers = encoded_layers[-1] + return encoded_layers, pooled_output + + @classmethod + def from_pretrained(cls, pretrained_model_dir, state_dict=None, *inputs, **kwargs): + # Load config + config_file = os.path.join(pretrained_model_dir, CONFIG_FILE) + config = json.load(open(config_file, "r")) + # config = BertConfig.from_json_file(config_file) + # logger.info("Model config {}".format(config)) + # Instantiate model. + model = cls(*inputs, **config, **kwargs) + if state_dict is None: + weights_path = os.path.join(pretrained_model_dir, MODEL_WEIGHTS) + state_dict = torch.load(weights_path) + + old_keys = [] + new_keys = [] + for key in state_dict.keys(): + new_key = None + if 'gamma' in key: + new_key = key.replace('gamma', 'weight') + if 'beta' in key: + new_key = key.replace('beta', 'bias') + if new_key: + old_keys.append(key) + new_keys.append(new_key) + for old_key, new_key in zip(old_keys, new_keys): + state_dict[new_key] = state_dict.pop(old_key) + + missing_keys = [] + unexpected_keys = [] + error_msgs = [] + # copy state_dict so _load_from_state_dict can modify it + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + def load(module, prefix=''): + local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {}) + module._load_from_state_dict( + state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + load(model, prefix='' if hasattr(model, 'bert') else 'bert.') + if len(missing_keys) > 0: + print("Weights of {} not initialized from pretrained model: {}".format( + model.__class__.__name__, missing_keys)) + if len(unexpected_keys) > 0: + print("Weights from pretrained model not used in {}: {}".format( + model.__class__.__name__, unexpected_keys)) + return model diff --git a/fastNLP/modules/aggregator/attention.py b/fastNLP/modules/aggregator/attention.py index ef3f3fe5..ef9d159d 100644 --- a/fastNLP/modules/aggregator/attention.py +++ b/fastNLP/modules/aggregator/attention.py @@ -4,8 +4,8 @@ import torch.nn.functional as F from torch import nn -from fastNLP.modules.utils import mask_softmax from fastNLP.modules.dropout import TimestepDropout +from fastNLP.modules.utils import mask_softmax class Attention(torch.nn.Module): @@ -49,27 +49,27 @@ def forward(self, Q, K, V, mask_out=None): class MultiHeadAtte(nn.Module): - def __init__(self, model_size, key_size, value_size, num_head, dropout=0.1): + def __init__(self, input_size, key_size, value_size, num_head, dropout=0.1): """ - :param model_size: int, 输入维度的大小。同时也是输出维度的大小。 + :param input_size: int, 输入维度的大小。同时也是输出维度的大小。 :param key_size: int, 每个head的维度大小。 :param value_size: int，每个head中value的维度。 :param num_head: int，head的数量。 :param dropout: float。 """ super(MultiHeadAtte, self).__init__() - self.input_size = model_size + self.input_size = input_size self.key_size = key_size self.value_size = value_size self.num_head = num_head in_size = key_size * num_head - self.q_in = nn.Linear(model_size, in_size) - self.k_in = nn.Linear(model_size, in_size) - self.v_in = nn.Linear(model_size, in_size) + self.q_in = nn.Linear(input_size, in_size) + self.k_in = nn.Linear(input_size, in_size) + self.v_in = nn.Linear(input_size, in_size) self.attention = DotAtte(key_size=key_size, value_size=value_size) - self.out = nn.Linear(value_size * num_head, model_size) + self.out = nn.Linear(value_size * num_head, input_size) self.drop = TimestepDropout(dropout) self.reset_parameters() @@ -108,6 +108,7 @@ def forward(self, Q, K, V, atte_mask_out=None): output = self.drop(self.out(atte)) return output + class Bi_Attention(nn.Module): def __init__(self): super(Bi_Attention, self).__init__() diff --git a/fastNLP/modules/encoder/transformer.py b/fastNLP/modules/encoder/transformer.py index 92ccc3fe..fe716bf7 100644 --- a/fastNLP/modules/encoder/transformer.py +++ b/fastNLP/modules/encoder/transformer.py @@ -1,4 +1,3 @@ -import torch from torch import nn from ..aggregator.attention import MultiHeadAtte diff --git a/test/models/test_bert.py b/test/models/test_bert.py new file mode 100644 index 00000000..b2899a89 --- /dev/null +++ b/test/models/test_bert.py @@ -0,0 +1,21 @@ +import unittest + +import torch + +from fastNLP.models.bert import BertModel + + +class TestBert(unittest.TestCase): + def test_bert_1(self): + # model = BertModel.from_pretrained("/home/zyfeng/data/bert-base-chinese") + model = BertModel(vocab_size=32000, hidden_size=768, + num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) + + input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) + input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) + token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]]) + + all_encoder_layers, pooled_output = model(input_ids, token_type_ids, input_mask) + for layer in all_encoder_layers: + self.assertEqual(tuple(layer.shape), (2, 3, 768)) + self.assertEqual(tuple(pooled_output.shape), (2, 768)) From 986541139af5761ddf05914ab75a9ae5a1e0c706 Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Sat, 2 Feb 2019 16:46:42 +0800 Subject: [PATCH 29/32] =?UTF-8?q?=E6=95=B4=E7=90=86=E6=89=80=E6=9C=89datas?= =?UTF-8?q?et=20loader=EF=BC=8C=E5=BB=BA=E7=AB=8B=E5=8D=95=E5=85=83?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/io/base_loader.py | 6 + fastNLP/io/config_io.py | 9 +- fastNLP/io/dataset_loader.py | 240 ++++++------------ fastNLP/models/biaffine_parser.py | 18 +- .../main.py | 2 +- test/core/test_batch.py | 5 +- test/core/test_trainer.py | 3 +- test/io/test_dataset_loader.py | 29 ++- 8 files changed, 113 insertions(+), 199 deletions(-) diff --git a/fastNLP/io/base_loader.py b/fastNLP/io/base_loader.py index ccfa1169..5d5fe63a 100644 --- a/fastNLP/io/base_loader.py +++ b/fastNLP/io/base_loader.py @@ -11,18 +11,24 @@ def __init__(self): @staticmethod def load_lines(data_path): + """按行读取，舍弃每行两侧空白字符，返回list of str + """ with open(data_path, "r", encoding="utf=8") as f: text = f.readlines() return [line.strip() for line in text] @classmethod def load(cls, data_path): + """先按行读取，去除一行两侧空白，再提取每行的字符。返回list of list of str + """ with open(data_path, "r", encoding="utf-8") as f: text = f.readlines() return [[word for word in sent.strip()] for sent in text] @classmethod def load_with_cache(cls, data_path, cache_path): + """缓存版的load + """ if os.path.isfile(cache_path) and os.path.getmtime(data_path) < os.path.getmtime(cache_path): with open(cache_path, 'rb') as f: return pickle.load(f) diff --git a/fastNLP/io/config_io.py b/fastNLP/io/config_io.py index 8be59a35..5a64b96c 100644 --- a/fastNLP/io/config_io.py +++ b/fastNLP/io/config_io.py @@ -11,7 +11,6 @@ class ConfigLoader(BaseLoader): :param str data_path: path to the config """ - def __init__(self, data_path=None): super(ConfigLoader, self).__init__() if data_path is not None: @@ -30,7 +29,7 @@ def load_config(file_path, sections): Example:: test_args = ConfigSection() - ConfigLoader("config.cfg", "").load_config("./data_for_tests/config", {"POS_test": test_args}) + ConfigLoader("config.cfg").load_config("./data_for_tests/config", {"POS_test": test_args}) """ assert isinstance(sections, dict) @@ -202,8 +201,6 @@ def _read_section(self): continue if '=' not in line: - # log = create_logger(__name__, './config_saver.log') - # log.error("can NOT load config file [%s]" % self.file_path) raise RuntimeError("can NOT load config file {}".__format__(self.file_path)) key = line.split('=', maxsplit=1)[0].strip() @@ -263,10 +260,6 @@ def save_config_file(self, section_name, section): change_file = True break if section_file[k] != section[k]: - # logger = create_logger(__name__, "./config_loader.log") - # logger.warning("section [%s] in config file [%s] has been changed" % ( - # section_name, self.file_path - # )) change_file = True break if not change_file: diff --git a/fastNLP/io/dataset_loader.py b/fastNLP/io/dataset_loader.py index 1fcdb7d9..07b721c5 100644 --- a/fastNLP/io/dataset_loader.py +++ b/fastNLP/io/dataset_loader.py @@ -126,8 +126,8 @@ def convert(self, data): DataLoaderRegister.set_reader(RawDataSetLoader, 'read_rawdata') -class POSDataSetLoader(DataSetLoader): - """Dataset Loader for a POS Tag dataset. +class DummyPOSReader(DataSetLoader): + """A simple reader for a dummy POS tagging dataset. In these datasets, each line are divided by "\t". The first Col is the vocabulary and the second Col is the label. Different sentence are divided by an empty line. @@ -146,7 +146,7 @@ class POSDataSetLoader(DataSetLoader): """ def __init__(self): - super(POSDataSetLoader, self).__init__() + super(DummyPOSReader, self).__init__() def load(self, data_path): """ @@ -194,16 +194,14 @@ def convert(self, data): return convert_seq2seq_dataset(data) -DataLoaderRegister.set_reader(POSDataSetLoader, 'read_pos') +DataLoaderRegister.set_reader(DummyPOSReader, 'read_pos') -class TokenizeDataSetLoader(DataSetLoader): +class DummyCWSReader(DataSetLoader): + """Load pku dataset for Chinese word segmentation. """ - Data set loader for tokenization data sets - """ - def __init__(self): - super(TokenizeDataSetLoader, self).__init__() + super(DummyCWSReader, self).__init__() def load(self, data_path, max_seq_len=32): """Load pku dataset for Chinese word segmentation. @@ -256,11 +254,11 @@ def convert(self, data): return convert_seq2seq_dataset(data) -class ClassDataSetLoader(DataSetLoader): +class DummyClassificationReader(DataSetLoader): """Loader for a dummy classification data set""" def __init__(self): - super(ClassDataSetLoader, self).__init__() + super(DummyClassificationReader, self).__init__() def load(self, data_path): assert os.path.exists(data_path) @@ -271,7 +269,7 @@ def load(self, data_path): @staticmethod def parse(lines): - """ + """每行第一个token是标签，其余是字/词；由空格分隔。 :param lines: lines from dataset :return: list(list(list())): the three level of lists are words, sentence, and dataset @@ -327,16 +325,11 @@ def convert(self, data): pass -class LMDataSetLoader(DataSetLoader): - """Language Model Dataset Loader - - This loader produces data for language model training in a supervised way. - That means it has X and Y. - +class DummyLMReader(DataSetLoader): + """A Dummy Language Model Dataset Reader """ - def __init__(self): - super(LMDataSetLoader, self).__init__() + super(DummyLMReader, self).__init__() def load(self, data_path): if not os.path.exists(data_path): @@ -364,19 +357,25 @@ def convert(self, data): class PeopleDailyCorpusLoader(DataSetLoader): + """人民日报数据集 """ - People Daily Corpus: Chinese word segmentation, POS tag, NER - """ - def __init__(self): super(PeopleDailyCorpusLoader, self).__init__() + self.pos = True + self.ner = True - def load(self, data_path): + def load(self, data_path, pos=True, ner=True): + """ + + :param str data_path: 数据路径 + :param bool pos: 是否使用词性标签 + :param bool ner: 是否使用命名实体标签 + :return: a DataSet object + """ + self.pos, self.ner = pos, ner with open(data_path, "r", encoding="utf-8") as f: sents = f.readlines() - - pos_tag_examples = [] - ner_examples = [] + examples = [] for sent in sents: if len(sent) <= 2: continue @@ -410,40 +409,44 @@ def load(self, data_path): sent_ner.append(ner_tag) sent_pos_tag.append(pos) sent_words.append(token) - pos_tag_examples.append([sent_words, sent_pos_tag]) - ner_examples.append([sent_words, sent_ner]) - # List[List[List[str], List[str]]] - # ner_examples not used - return self.convert(pos_tag_examples) + example = [sent_words] + if self.pos is True: + example.append(sent_pos_tag) + if self.ner is True: + example.append(sent_ner) + examples.append(example) + return self.convert(examples) def convert(self, data): data_set = DataSet() for item in data: - sent_words, sent_pos_tag = item[0], item[1] - data_set.append(Instance(words=sent_words, tags=sent_pos_tag)) - data_set.apply(lambda ins: len(ins), new_field_name="seq_len") - data_set.set_target("tags") - data_set.set_input("sent_words") - data_set.set_input("seq_len") + sent_words = item[0] + if self.pos is True and self.ner is True: + instance = Instance(words=sent_words, pos_tags=item[1], ner=item[2]) + elif self.pos is True: + instance = Instance(words=sent_words, pos_tags=item[1]) + elif self.ner is True: + instance = Instance(words=sent_words, ner=item[1]) + else: + instance = Instance(words=sent_words) + data_set.append(instance) + data_set.apply(lambda ins: len(ins["words"]), new_field_name="seq_len") return data_set class Conll2003Loader(DataSetLoader): - """Self-defined loader of conll2003 dataset + """Loader for conll2003 dataset More information about the given dataset cound be found on https://sites.google.com/site/ermasoftware/getting-started/ne-tagging-conll2003-data """ - def __init__(self): super(Conll2003Loader, self).__init__() def load(self, dataset_path): with open(dataset_path, "r", encoding="utf-8") as f: lines = f.readlines() - - ##Parse the dataset line by line parsed_data = [] sentence = [] tokens = [] @@ -470,21 +473,20 @@ def convert(self, parsed_data): lambda labels: labels[1], sample[1])) label2_list = list(map( lambda labels: labels[2], sample[1])) - dataset.append(Instance(token_list=sample[0], - label0_list=label0_list, - label1_list=label1_list, - label2_list=label2_list)) + dataset.append(Instance(tokens=sample[0], + pos=label0_list, + chucks=label1_list, + ner=label2_list)) return dataset -class SNLIDataSetLoader(DataSetLoader): +class SNLIDataSetReader(DataSetLoader): """A data set loader for SNLI data set. """ - def __init__(self): - super(SNLIDataSetLoader, self).__init__() + super(SNLIDataSetReader, self).__init__() def load(self, path_list): """ @@ -553,6 +555,8 @@ def load(self, path, cut_long_sent=False): """ 返回的DataSet只包含raw_sentence这个field，内容为str。 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 + :: + 1 编者按 编者按 NN O 11 nmod:topic 2 ： ： PU O 11 punct 3 7月 7月 NT DATE 4 compound:nn @@ -564,6 +568,7 @@ def load(self, path, cut_long_sent=False): 3 飞行 飞行 NN O 8 nsubj 4 从 从 P O 5 case 5 外型 外型 NN O 8 nmod:prep + """ datalist = [] with open(path, 'r', encoding='utf-8') as f: @@ -575,7 +580,7 @@ def load(self, path, cut_long_sent=False): elif line.startswith('#'): continue else: - sample.append(line.split('\t')) + sample.append(line.strip().split()) if len(sample) > 0: datalist.append(sample) @@ -592,7 +597,6 @@ def load(self, path, cut_long_sent=False): sents = [line] for raw_sentence in sents: ds.append(Instance(raw_sentence=raw_sentence)) - return ds def get_char_lst(self, sample): @@ -607,70 +611,22 @@ def get_char_lst(self, sample): return text -class POSCWSReader(DataSetLoader): - """ - 支持读取以下的情况, 即每一行是一个词, 用空行作为两句话的界限. - 迈 N - 向 N - 充 N - ... - 泽 I-PER - 民 I-PER - - （ N - 一 N - 九 N - ... - - - :param filepath: - :return: - """ - - def __init__(self, in_word_splitter=None): - super().__init__() - self.in_word_splitter = in_word_splitter - - def load(self, filepath, in_word_splitter=None, cut_long_sent=False): - if in_word_splitter is None: - in_word_splitter = self.in_word_splitter - dataset = DataSet() - with open(filepath, 'r') as f: - words = [] - for line in f: - line = line.strip() - if len(line) == 0: # new line - if len(words) == 0: # 不能接受空行 - continue - line = ' '.join(words) - if cut_long_sent: - sents = cut_long_sentence(line) - else: - sents = [line] - for sent in sents: - instance = Instance(raw_sentence=sent) - dataset.append(instance) - words = [] - else: - line = line.split()[0] - if in_word_splitter is None: - words.append(line) - else: - words.append(line.split(in_word_splitter)[0]) - return dataset - - class NaiveCWSReader(DataSetLoader): """ 这个reader假设了分词数据集为以下形式, 即已经用空格分割好内容了 + 例如:: + 这是 fastNLP , 一个 非常 good 的 包 . + 或者,即每个part后面还有一个pos tag + 例如:: + 也/D 在/P 團員/Na 之中/Ng ，/COMMACATEGORY + """ def __init__(self, in_word_splitter=None): - super().__init__() - + super(NaiveCWSReader, self).__init__() self.in_word_splitter = in_word_splitter def load(self, filepath, in_word_splitter=None, cut_long_sent=False): @@ -680,8 +636,10 @@ def load(self, filepath, in_word_splitter=None, cut_long_sent=False): 和 也/D 在/P 團員/Na 之中/Ng ，/COMMACATEGORY 如果splitter不为None则认为是第二种情况, 且我们会按splitter分割"也/D", 然后取第一部分. 例如"也/D".split('/')[0] + :param filepath: :param in_word_splitter: + :param cut_long_sent: :return: """ if in_word_splitter == None: @@ -740,7 +698,9 @@ def cut_long_sentence(sent, max_sample_length=200): class ZhConllPOSReader(object): - # 中文colln格式reader + """读取中文Conll格式。返回“字级别”的标签，使用BMES记号扩展原来的词级别标签。 + + """ def __init__(self): pass @@ -750,6 +710,8 @@ def load(self, path): words：list of str, tag: list of str, 被加入了BMES tag, 比如原来的序列为['VP', 'NN', 'NN', ..]，会被认为是["S-VP", "B-NN", "M-NN",..] 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 + :: + 1 编者按 编者按 NN O 11 nmod:topic 2 ： ： PU O 11 punct 3 7月 7月 NT DATE 4 compound:nn @@ -761,6 +723,7 @@ def load(self, path): 3 飞行 飞行 NN O 8 nsubj 4 从 从 P O 5 case 5 外型 外型 NN O 8 nmod:prep + """ datalist = [] with open(path, 'r', encoding='utf-8') as f: @@ -815,67 +778,10 @@ def get_one(self, sample): return text, pos_tags -class ConllPOSReader(object): - # 返回的Dataset包含words(list of list, 里层的list是character), tag两个field(list of str, str是标有BIO的tag)。 - def __init__(self): - pass - - def load(self, path): - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - sample = [] - for line in f: - if line.startswith('\n'): - datalist.append(sample) - sample = [] - elif line.startswith('#'): - continue - else: - sample.append(line.split('\t')) - if len(sample) > 0: - datalist.append(sample) - - ds = DataSet() - for sample in datalist: - # print(sample) - res = self.get_one(sample) - if res is None: - continue - char_seq = [] - pos_seq = [] - for word, tag in zip(res[0], res[1]): - if len(word) == 1: - char_seq.append(word) - pos_seq.append('S-{}'.format(tag)) - elif len(word) > 1: - pos_seq.append('B-{}'.format(tag)) - for _ in range(len(word) - 2): - pos_seq.append('M-{}'.format(tag)) - pos_seq.append('E-{}'.format(tag)) - char_seq.extend(list(word)) - else: - raise ValueError("Zero length of word detected.") - - ds.append(Instance(words=char_seq, - tag=pos_seq)) - return ds - - def get_one(self, sample): - if len(sample) == 0: - return None - text = [] - pos_tags = [] - for w in sample: - t1, t2, t3, t4 = w[1], w[3], w[6], w[7] - if t3 == '_': - return None - text.append(t1) - pos_tags.append(t2) - return text, pos_tags - - - class ConllxDataLoader(object): + """返回“词级别”的标签信息，包括词、词性、（句法）头依赖、（句法）边标签。跟``ZhConllPOSReader``完全不同。 + + """ def load(self, path): datalist = [] with open(path, 'r', encoding='utf-8') as f: diff --git a/fastNLP/models/biaffine_parser.py b/fastNLP/models/biaffine_parser.py index dfbaac58..dc294eb3 100644 --- a/fastNLP/models/biaffine_parser.py +++ b/fastNLP/models/biaffine_parser.py @@ -1,18 +1,20 @@ -import copy +from collections import defaultdict + import numpy as np import torch -from collections import defaultdict from torch import nn from torch.nn import functional as F -from fastNLP.modules.utils import initial_parameter -from fastNLP.modules.encoder.variational_rnn import VarLSTM -from fastNLP.modules.encoder.transformer import TransformerEncoder -from fastNLP.modules.dropout import TimestepDropout -from fastNLP.models.base_model import BaseModel -from fastNLP.modules.utils import seq_mask + from fastNLP.core.losses import LossFunc from fastNLP.core.metrics import MetricBase from fastNLP.core.utils import seq_lens_to_masks +from fastNLP.models.base_model import BaseModel +from fastNLP.modules.dropout import TimestepDropout +from fastNLP.modules.encoder.transformer import TransformerEncoder +from fastNLP.modules.encoder.variational_rnn import VarLSTM +from fastNLP.modules.utils import initial_parameter +from fastNLP.modules.utils import seq_mask + def mst(scores): """ diff --git a/reproduction/LSTM+self_attention_sentiment_analysis/main.py b/reproduction/LSTM+self_attention_sentiment_analysis/main.py index 61ab79f4..ff2d7a67 100644 --- a/reproduction/LSTM+self_attention_sentiment_analysis/main.py +++ b/reproduction/LSTM+self_attention_sentiment_analysis/main.py @@ -4,7 +4,7 @@ from fastNLP.core.utils import ClassPreprocess as Preprocess from fastNLP.io.config_io import ConfigLoader from fastNLP.io.config_io import ConfigSection -from fastNLP.io.dataset_loader import ClassDataSetLoader as Dataset_loader +from fastNLP.io.dataset_loader import DummyClassificationReader as Dataset_loader from fastNLP.models.base_model import BaseModel from fastNLP.modules.aggregator.self_attention import SelfAttention from fastNLP.modules.decoder.MLP import MLP diff --git a/test/core/test_batch.py b/test/core/test_batch.py index e1561942..abc2b3e2 100644 --- a/test/core/test_batch.py +++ b/test/core/test_batch.py @@ -138,6 +138,7 @@ def test_sequential_batch(self): for batch_x, batch_y in batch: time.sleep(pause_seconds) + """ def test_multi_workers_batch(self): batch_size = 32 pause_seconds = 0.01 @@ -154,7 +155,8 @@ def test_multi_workers_batch(self): end1 = time.time() for batch_x, batch_y in batch: time.sleep(pause_seconds) - + """ + """ def test_pin_memory(self): batch_size = 32 pause_seconds = 0.01 @@ -172,3 +174,4 @@ def test_pin_memory(self): # 这里发生OOM # for batch_x, batch_y in batch: # time.sleep(pause_seconds) + """ diff --git a/test/core/test_trainer.py b/test/core/test_trainer.py index 7c869633..36062ef7 100644 --- a/test/core/test_trainer.py +++ b/test/core/test_trainer.py @@ -237,6 +237,7 @@ def forward(self, x1, x2): use_tqdm=False, print_every=2) + """ def test_trainer_multiprocess(self): dataset = prepare_fake_dataset2('x1', 'x2') dataset.set_input('x1', 'x2', 'y', flag=True) @@ -264,4 +265,4 @@ def forward(self, x1, x2, y): timeout=0, ) trainer.train() - + """ diff --git a/test/io/test_dataset_loader.py b/test/io/test_dataset_loader.py index cf38c973..16e7d7ea 100644 --- a/test/io/test_dataset_loader.py +++ b/test/io/test_dataset_loader.py @@ -1,24 +1,27 @@ import unittest -from fastNLP.io.dataset_loader import Conll2003Loader +from fastNLP.io.dataset_loader import Conll2003Loader, PeopleDailyCorpusLoader, ConllCWSReader, \ + ZhConllPOSReader, ConllxDataLoader class TestDatasetLoader(unittest.TestCase): - def test_case_1(self): - ''' + def test_Conll2003Loader(self): + """ Test the the loader of Conll2003 dataset - ''' - + """ dataset_path = "test/data_for_tests/conll_2003_example.txt" loader = Conll2003Loader() dataset_2003 = loader.load(dataset_path) - for item in dataset_2003: - len0 = len(item["label0_list"]) - len1 = len(item["label1_list"]) - len2 = len(item["label2_list"]) - lentoken = len(item["token_list"]) - self.assertNotEqual(len0, 0) - self.assertEqual(len0, len1) - self.assertEqual(len1, len2) + def test_PeopleDailyCorpusLoader(self): + data_set = PeopleDailyCorpusLoader().load("test/data_for_tests/people_daily_raw.txt") + + def test_ConllCWSReader(self): + dataset = ConllCWSReader().load("test/data_for_tests/conll_example.txt") + + def test_ZhConllPOSReader(self): + dataset = ZhConllPOSReader().load("test/data_for_tests/zh_sample.conllx") + + def test_ConllxDataLoader(self): + dataset = ConllxDataLoader().load("test/data_for_tests/zh_sample.conllx") From 0c5630bd16c2cba1623ceacfdd21ab789dfdac56 Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Mon, 4 Feb 2019 09:44:54 +0800 Subject: [PATCH 30/32] =?UTF-8?q?Ready=20for=20V0.3.1=20*=20=E5=8D=87?= =?UTF-8?q?=E7=BA=A7parser=20API=E5=92=8C=E6=A8=A1=E5=9E=8B=20*=20update?= =?UTF-8?q?=20docs:=20add=20new=20pages=20for=20tutorials=20*=20upgrade=20?= =?UTF-8?q?CWS=20api=20download=20source=20*=20add=20a=20new=20method=20fo?= =?UTF-8?q?r=20dataset=20field=20access=20*=20add=20introduction=20for=20b?= =?UTF-8?q?ert=20*=20add=20more=20unit=20tests=20for=20api/processor=20*?= =?UTF-8?q?=20remove=20unused=20test=20data.=20Add=20new=20test=20data.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tutorials/fastnlp_10tmin_tutorial.rst | 5 +- .../tutorials/fastnlp_1_minute_tutorial.rst | 2 + .../tutorials/fastnlp_advanced_tutorial.rst | 5 + .../tutorials/fastnlp_developer_guide.rst | 5 + docs/source/user/installation.rst | 1 + docs/source/user/quickstart.rst | 2 + fastNLP/api/api.py | 98 +- fastNLP/api/examples.py | 23 +- fastNLP/api/processor.py | 9 +- fastNLP/core/dataset.py | 4 + fastNLP/models/bert.py | 22 +- reproduction/Biaffine_parser/cfg.cfg | 2 +- test/api/test_processor.py | 50 +- test/data_for_tests/charlm.txt | 3370 ----------------- test/data_for_tests/people_infer.txt | 2 - test/data_for_tests/zh_sample.conllx | 100 + 16 files changed, 288 insertions(+), 3412 deletions(-) create mode 100644 docs/source/tutorials/fastnlp_advanced_tutorial.rst create mode 100644 docs/source/tutorials/fastnlp_developer_guide.rst delete mode 100644 test/data_for_tests/charlm.txt delete mode 100644 test/data_for_tests/people_infer.txt create mode 100644 test/data_for_tests/zh_sample.conllx diff --git a/docs/source/tutorials/fastnlp_10tmin_tutorial.rst b/docs/source/tutorials/fastnlp_10tmin_tutorial.rst index 30293796..4c5fc65e 100644 --- a/docs/source/tutorials/fastnlp_10tmin_tutorial.rst +++ b/docs/source/tutorials/fastnlp_10tmin_tutorial.rst @@ -1,7 +1,8 @@ - -fastNLP上手教程 +fastNLP 10分钟上手教程 =============== +教程原文见 https://github.com/fastnlp/fastNLP/blob/master/tutorials/fastnlp_10min_tutorial.ipynb + fastNLP提供方便的数据预处理，训练和测试模型的功能 DataSet & Instance diff --git a/docs/source/tutorials/fastnlp_1_minute_tutorial.rst b/docs/source/tutorials/fastnlp_1_minute_tutorial.rst index b4471e00..b4c6c8c4 100644 --- a/docs/source/tutorials/fastnlp_1_minute_tutorial.rst +++ b/docs/source/tutorials/fastnlp_1_minute_tutorial.rst @@ -2,6 +2,8 @@ FastNLP 1分钟上手教程 ===================== +教程原文见 https://github.com/fastnlp/fastNLP/blob/master/tutorials/fastnlp_1min_tutorial.ipynb + step 1 ------ diff --git a/docs/source/tutorials/fastnlp_advanced_tutorial.rst b/docs/source/tutorials/fastnlp_advanced_tutorial.rst new file mode 100644 index 00000000..d788e9d6 --- /dev/null +++ b/docs/source/tutorials/fastnlp_advanced_tutorial.rst @@ -0,0 +1,5 @@ +fastNLP 进阶教程 +=============== + +教程原文见 https://github.com/fastnlp/fastNLP/blob/master/tutorials/fastnlp_advanced_tutorial/advance_tutorial.ipynb + diff --git a/docs/source/tutorials/fastnlp_developer_guide.rst b/docs/source/tutorials/fastnlp_developer_guide.rst new file mode 100644 index 00000000..73b75f02 --- /dev/null +++ b/docs/source/tutorials/fastnlp_developer_guide.rst @@ -0,0 +1,5 @@ +fastNLP 开发者指南 +=============== + +原文见 https://github.com/fastnlp/fastNLP/blob/master/tutorials/tutorial_for_developer.md + diff --git a/docs/source/user/installation.rst b/docs/source/user/installation.rst index 7dc39b3b..5dfe4a11 100644 --- a/docs/source/user/installation.rst +++ b/docs/source/user/installation.rst @@ -5,6 +5,7 @@ Installation .. contents:: :local: +Make sure your environment satisfies https://github.com/fastnlp/fastNLP/blob/master/requirements.txt . Run the following commands to install fastNLP package: diff --git a/docs/source/user/quickstart.rst b/docs/source/user/quickstart.rst index baa49eef..a5eb9402 100644 --- a/docs/source/user/quickstart.rst +++ b/docs/source/user/quickstart.rst @@ -6,4 +6,6 @@ Quickstart ../tutorials/fastnlp_1_minute_tutorial ../tutorials/fastnlp_10tmin_tutorial + ../tutorials/fastnlp_advanced_tutorial + ../tutorials/fastnlp_developer_guide diff --git a/fastNLP/api/api.py b/fastNLP/api/api.py index 0c5f17bc..53a80131 100644 --- a/fastNLP/api/api.py +++ b/fastNLP/api/api.py @@ -9,7 +9,7 @@ from fastNLP.api.utils import load_url from fastNLP.api.processor import ModelProcessor -from fastNLP.io.dataset_loader import ConllCWSReader, ConllxDataLoader, add_seg_tag +from fastNLP.io.dataset_loader import ConllCWSReader, ConllxDataLoader from fastNLP.core.instance import Instance from fastNLP.api.pipeline import Pipeline from fastNLP.core.metrics import SpanFPreRecMetric @@ -17,9 +17,9 @@ # TODO add pretrain urls model_urls = { - "cws": "http://123.206.98.91:8888/download/cws_crf_1_11-457fc899.pkl", + "cws": "http://123.206.98.91:8888/download/cws_lstm_ctb9_1_20-09908656.pkl", "pos": "http://123.206.98.91:8888/download/pos_tag_model_20190119-43f8b435.pkl", - "parser": "http://123.206.98.91:8888/download/biaffine_parser-3a2f052c.pkl" + "parser": "http://123.206.98.91:8888/download/parser_20190204-c72ca5c0.pkl" } @@ -90,38 +90,28 @@ def predict(self, content): # 3. 使用pipeline self.pipeline(dataset) - # def decode_tags(ins): - # pred_tags = ins["tag"] - # chars = ins["words"] - # words = [] - # start_idx = 0 - # for idx, tag in enumerate(pred_tags): - # if tag[0] == "S": - # words.append(chars[start_idx:idx + 1] + "/" + tag[2:]) - # start_idx = idx + 1 - # elif tag[0] == "E": - # words.append("".join(chars[start_idx:idx + 1]) + "/" + tag[2:]) - # start_idx = idx + 1 - # return words - # - # dataset.apply(decode_tags, new_field_name="tag_output") + def merge_tag(words_list, tags_list): + rtn = [] + for words, tags in zip(words_list, tags_list): + rtn.append([w + "/" + t for w, t in zip(words, tags)]) + return rtn output = dataset.field_arrays["tag"].content if isinstance(content, str): return output[0] elif isinstance(content, list): - return output + return merge_tag(content, output) def test(self, file_path): test_data = ConllxDataLoader().load(file_path) - with open("model_pp_0117.pkl", "rb") as f: - save_dict = torch.load(f) + save_dict = self._dict tag_vocab = save_dict["tag_vocab"] pipeline = save_dict["pipeline"] index_tag = IndexerProcessor(vocab=tag_vocab, field_name="tag", new_added_field_name="truth", is_input=False) pipeline.pipeline = [index_tag] + pipeline.pipeline + test_data.rename_field("pos_tags", "tag") pipeline(test_data) test_data.set_target("truth") prediction = test_data.field_arrays["predict"].content @@ -235,7 +225,7 @@ def test(self, filepath): rec = eval_res['BMESF1PreRecMetric']['rec'] # print("f1:{:.2f}, pre:{:.2f}, rec:{:.2f}".format(f1, pre, rec)) - return f1, pre, rec + return {"F1": f1, "precision": pre, "recall": rec} class Parser(API): @@ -260,6 +250,7 @@ def predict(self, content): dataset.add_field('wp', pos_out) dataset.apply(lambda x: [''] + [w.split('/')[0] for w in x['wp']], new_field_name='words') dataset.apply(lambda x: [''] + [w.split('/')[1] for w in x['wp']], new_field_name='pos') + dataset.rename_field("words", "raw_words") # 3. 使用pipeline self.pipeline(dataset) @@ -269,31 +260,74 @@ def predict(self, content): # output like: [['2/top', '0/root', '4/nn', '2/dep']] return dataset.field_arrays['output'].content - def test(self, filepath): - data = ConllxDataLoader().load(filepath) - ds = DataSet() - for ins1, ins2 in zip(add_seg_tag(data), data): - ds.append(Instance(words=ins1[0], tag=ins1[1], - gold_words=ins2[0], gold_pos=ins2[1], - gold_heads=ins2[2], gold_head_tags=ins2[3])) + def load_test_file(self, path): + def get_one(sample): + sample = list(map(list, zip(*sample))) + if len(sample) == 0: + return None + for w in sample[7]: + if w == '_': + print('Error Sample {}'.format(sample)) + return None + # return word_seq, pos_seq, head_seq, head_tag_seq + return sample[1], sample[3], list(map(int, sample[6])), sample[7] + + datalist = [] + with open(path, 'r', encoding='utf-8') as f: + sample = [] + for line in f: + if line.startswith('\n'): + datalist.append(sample) + sample = [] + elif line.startswith('#'): + continue + else: + sample.append(line.split('\t')) + if len(sample) > 0: + datalist.append(sample) + + data = [get_one(sample) for sample in datalist] + data_list = list(filter(lambda x: x is not None, data)) + return data_list + def test(self, filepath): + data = self.load_test_file(filepath) + + def convert(data): + BOS = '' + dataset = DataSet() + for sample in data: + word_seq = [BOS] + sample[0] + pos_seq = [BOS] + sample[1] + heads = [0] + sample[2] + head_tags = [BOS] + sample[3] + dataset.append(Instance(raw_words=word_seq, + pos=pos_seq, + gold_heads=heads, + arc_true=heads, + tags=head_tags)) + return dataset + + ds = convert(data) pp = self.pipeline for p in pp: if p.field_name == 'word_list': p.field_name = 'gold_words' elif p.field_name == 'pos_list': p.field_name = 'gold_pos' + # ds.rename_field("words", "raw_words") + # ds.rename_field("tag", "pos") pp(ds) head_cor, label_cor, total = 0, 0, 0 for ins in ds: head_gold = ins['gold_heads'] - head_pred = ins['heads'] + head_pred = ins['arc_pred'] length = len(head_gold) total += length for i in range(length): head_cor += 1 if head_pred[i] == head_gold[i] else 0 uas = head_cor / total - print('uas:{:.2f}'.format(uas)) + # print('uas:{:.2f}'.format(uas)) for p in pp: if p.field_name == 'gold_words': @@ -301,7 +335,7 @@ def test(self, filepath): elif p.field_name == 'gold_pos': p.field_name = 'pos_list' - return uas + return {"USA": round(uas, 5)} class Analyzer: diff --git a/fastNLP/api/examples.py b/fastNLP/api/examples.py index 447d127a..9d9f190e 100644 --- a/fastNLP/api/examples.py +++ b/fastNLP/api/examples.py @@ -15,19 +15,40 @@ def chinese_word_segmentation(): print(cws.predict(text)) +def chinese_word_segmentation_test(): + cws = CWS(device='cpu') + print(cws.test("../../test/data_for_tests/zh_sample.conllx")) + + def pos_tagging(): # 输入已分词序列 text = ['编者 按： 7月 12日 ， 英国 航空 航天 系统 公司 公布 了 该 公司 研制 的 第一款 高科技 隐形 无人机 雷电之神 。'] text = [text[0].split()] - print(text) pos = POS(device='cpu') print(pos.predict(text)) +def pos_tagging_test(): + pos = POS(device='cpu') + print(pos.test("../../test/data_for_tests/zh_sample.conllx")) + + def syntactic_parsing(): + text = ['编者 按： 7月 12日 ， 英国 航空 航天 系统 公司 公布 了 该 公司 研制 的 第一款 高科技 隐形 无人机 雷电之神 。'] + text = [text[0].split()] parser = Parser(device='cpu') print(parser.predict(text)) +def syntactic_parsing_test(): + parser = Parser(device='cpu') + print(parser.test("../../test/data_for_tests/zh_sample.conllx")) + + if __name__ == "__main__": + chinese_word_segmentation() + chinese_word_segmentation_test() pos_tagging() + pos_tagging_test() + syntactic_parsing() + syntactic_parsing_test() diff --git a/fastNLP/api/processor.py b/fastNLP/api/processor.py index 6867dae8..0bba96c0 100644 --- a/fastNLP/api/processor.py +++ b/fastNLP/api/processor.py @@ -102,6 +102,7 @@ class PreAppendProcessor(Processor): [data] + instance[field_name] """ + def __init__(self, data, field_name, new_added_field_name=None): super(PreAppendProcessor, self).__init__(field_name, new_added_field_name) self.data = data @@ -116,6 +117,7 @@ class SliceProcessor(Processor): 从某个field中只取部分内容。等价于instance[field_name][start:end:step] """ + def __init__(self, start, end, step, field_name, new_added_field_name=None): super(SliceProcessor, self).__init__(field_name, new_added_field_name) for o in (start, end, step): @@ -132,6 +134,7 @@ class Num2TagProcessor(Processor): 将一句话中的数字转换为某个tag。 """ + def __init__(self, tag, field_name, new_added_field_name=None): """ @@ -163,6 +166,7 @@ class IndexerProcessor(Processor): 给定一个vocabulary , 将指定field转换为index形式。指定field应该是一维的list，比如 ['我', '是', xxx] """ + def __init__(self, vocab, field_name, new_added_field_name, delete_old_field=False, is_input=True): assert isinstance(vocab, Vocabulary), "Only Vocabulary class is allowed, not {}.".format(type(vocab)) @@ -215,6 +219,7 @@ class SeqLenProcessor(Processor): 根据某个field新增一个sequence length的field。取该field的第一维 """ + def __init__(self, field_name, new_added_field_name='seq_lens', is_input=True): super(SeqLenProcessor, self).__init__(field_name, new_added_field_name) self.is_input = is_input @@ -229,6 +234,7 @@ def process(self, dataset): from fastNLP.core.utils import _build_args + class ModelProcessor(Processor): def __init__(self, model, seq_len_field_name='seq_lens', batch_size=32): """ @@ -292,6 +298,7 @@ class Index2WordProcessor(Processor): 将DataSet中某个为index的field根据vocab转换为str """ + def __init__(self, vocab, field_name, new_added_field_name): super(Index2WordProcessor, self).__init__(field_name, new_added_field_name) self.vocab = vocab @@ -303,7 +310,6 @@ def process(self, dataset): class SetTargetProcessor(Processor): - # TODO; remove it. def __init__(self, *fields, flag=True): super(SetTargetProcessor, self).__init__(None, None) self.fields = fields @@ -313,6 +319,7 @@ def process(self, dataset): dataset.set_target(*self.fields, flag=self.flag) return dataset + class SetInputProcessor(Processor): def __init__(self, *fields, flag=True): super(SetInputProcessor, self).__init__(None, None) diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index b763ada2..601fa589 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -92,6 +92,10 @@ def __getitem__(self, idx): data_set.add_field(name=field.name, fields=field.content[idx], padder=field.padder, is_input=field.is_input, is_target=field.is_target) return data_set + elif isinstance(idx, str): + if idx not in self: + raise KeyError("No such field called {} in DataSet.".format(idx)) + return self.field_arrays[idx] else: raise KeyError("Unrecognized type {} for idx in __getitem__ method".format(type(idx))) diff --git a/fastNLP/models/bert.py b/fastNLP/models/bert.py index 754d1bbb..e87f6f5d 100644 --- a/fastNLP/models/bert.py +++ b/fastNLP/models/bert.py @@ -1,3 +1,7 @@ +""" +bert.py is modified from huggingface/pytorch-pretrained-BERT, which is licensed under the Apache License 2.0. + +""" import copy import json import math @@ -220,7 +224,23 @@ def forward(self, hidden_states): class BertModel(nn.Module): - """BERT model ("Bidirectional Embedding Representations from a Transformer"). + """Bidirectional Embedding Representations from Transformers. + + If you want to use pre-trained weights, please download from the following sources provided by pytorch-pretrained-BERT. + sources:: + + 'bert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased.tar.gz", + 'bert-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased.tar.gz", + 'bert-base-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased.tar.gz", + 'bert-large-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased.tar.gz", + 'bert-base-multilingual-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased.tar.gz", + 'bert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased.tar.gz", + 'bert-base-chinese': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese.tar.gz", + + + Construct a BERT model with pre-trained weights:: + + model = BertModel.from_pretrained("path/to/weights/directory") """ diff --git a/reproduction/Biaffine_parser/cfg.cfg b/reproduction/Biaffine_parser/cfg.cfg index 87fccb18..03040600 100644 --- a/reproduction/Biaffine_parser/cfg.cfg +++ b/reproduction/Biaffine_parser/cfg.cfg @@ -1,5 +1,5 @@ [train] -n_epochs = 1 +n_epochs = 20 batch_size = 32 use_cuda = true use_tqdm=true diff --git a/test/api/test_processor.py b/test/api/test_processor.py index f515e507..d0c27c40 100644 --- a/test/api/test_processor.py +++ b/test/api/test_processor.py @@ -1,9 +1,12 @@ import random import unittest -from fastNLP import Vocabulary +import numpy as np + +from fastNLP import Vocabulary, Instance from fastNLP.api.processor import FullSpaceToHalfSpaceProcessor, PreAppendProcessor, SliceProcessor, Num2TagProcessor, \ - IndexerProcessor, VocabProcessor, SeqLenProcessor + IndexerProcessor, VocabProcessor, SeqLenProcessor, ModelProcessor, Index2WordProcessor, SetTargetProcessor, \ + SetInputProcessor, VocabIndexerProcessor from fastNLP.core.dataset import DataSet @@ -53,3 +56,46 @@ def test_SeqLenProcessor(self): ds = proc(ds) for data in ds.field_arrays["len"].content: self.assertEqual(data, 30) + + def test_ModelProcessor(self): + from fastNLP.models.cnn_text_classification import CNNText + model = CNNText(100, 100, 5) + ins_list = [] + for _ in range(64): + seq_len = np.random.randint(5, 30) + ins_list.append(Instance(word_seq=[np.random.randint(0, 100) for _ in range(seq_len)], seq_lens=seq_len)) + data_set = DataSet(ins_list) + data_set.set_input("word_seq", "seq_lens") + proc = ModelProcessor(model) + data_set = proc(data_set) + self.assertTrue("pred" in data_set) + + def test_Index2WordProcessor(self): + vocab = Vocabulary() + vocab.add_word_lst(["a", "b", "c", "d", "e"]) + proc = Index2WordProcessor(vocab, "tag_id", "tag") + data_set = DataSet([Instance(tag_id=[np.random.randint(0, 7) for _ in range(32)])]) + data_set = proc(data_set) + self.assertTrue("tag" in data_set) + + def test_SetTargetProcessor(self): + proc = SetTargetProcessor("a", "b", "c") + data_set = DataSet({"a": [1, 2, 3], "b": [1, 2, 3], "c": [1, 2, 3]}) + data_set = proc(data_set) + self.assertTrue(data_set["a"].is_target) + self.assertTrue(data_set["b"].is_target) + self.assertTrue(data_set["c"].is_target) + + def test_SetInputProcessor(self): + proc = SetInputProcessor("a", "b", "c") + data_set = DataSet({"a": [1, 2, 3], "b": [1, 2, 3], "c": [1, 2, 3]}) + data_set = proc(data_set) + self.assertTrue(data_set["a"].is_input) + self.assertTrue(data_set["b"].is_input) + self.assertTrue(data_set["c"].is_input) + + def test_VocabIndexerProcessor(self): + proc = VocabIndexerProcessor("word_seq", "word_ids") + data_set = DataSet([Instance(word_seq=["a", "b", "c", "d", "e"])]) + data_set = proc(data_set) + self.assertTrue("word_ids" in data_set) diff --git a/test/data_for_tests/charlm.txt b/test/data_for_tests/charlm.txt deleted file mode 100644 index 7b3d9d9f..00000000 --- a/test/data_for_tests/charlm.txt +++ /dev/null @@ -1,3370 +0,0 @@ - consumers may want to move their telephones a little closer to the tv set - watching abc 's monday night football can now vote during for the greatest play in N years from among four or five - two weeks ago viewers of several nbc consumer segments started calling a N number for advice on various issues - and the new syndicated reality show hard copy records viewers ' opinions for possible airing on the next day 's show - interactive telephone technology has taken a new leap in and television programmers are racing to exploit the possibilities - eventually viewers may grow with the technology and the cost - but right now programmers are figuring that viewers who are busy dialing up a range of services may put down their control and stay - we 've been spending a lot of time in los angeles talking to tv production people says mike parks president of call interactive which supplied technology for both abc sports and nbc 's consumer minutes - with the competitiveness of the television market these days everyone is looking for a way to get viewers more excited - one of the leaders behind the expanded use of N numbers is call interactive a joint venture of giants american express co. and american telephone & telegraph co - formed in august the venture at&t 's newly expanded N service with N computers in american express 's omaha neb. service center - other long-distance carriers have also begun marketing enhanced N service and special consultants are up to exploit the new tool - blair entertainment a new york firm that advises tv stations and sells ads for them has just formed a subsidiary N blair to apply the technology to television - the use of N toll numbers has been expanding rapidly in recent years - for a while lines and services that children to dial and movie or music information earned the service a somewhat image but new legal restrictions are aimed at trimming excesses - the cost of a N call is set by the abc sports for example with the cheapest starting at N cents - billing is included in a caller 's regular phone bill - from the fee the local phone company and the long-distance carrier extract their costs to carry the call passing the rest of the money to the which must cover advertising and other costs - in recent months the technology has become more flexible and able to handle much more volume - before callers of N numbers would just listen and not talk or they 'd vote yes or no by calling one of two numbers - people in the phone business call this technology N - now callers are led through complex of choices to retrieve information they want and the hardware can process N calls in N seconds - up to now N numbers have mainly been used on local tv stations and cable channels - used one to give away the house that rock star jon grew up in - for several years turner broadcasting system 's cable news network has invited viewers to respond to issues should the u.s. military intervene in panama but even the hottest on only about N calls - the newest uses of the technology demonstrate the growing variety of applications - capital cities\/abc inc. cbs inc. and general electric co. 's national broadcasting co. unit are expected to announce soon a joint campaign to raise awareness about - the subject will be written into the of prime-time shows and viewers will be given a N number to call - callers will be sent educational booklets and the call 's modest cost will be an immediate method of raising money - other network applications have very different goals - abc sports was looking for ways to lift ratings for monday night football - kurt abc sports 's marketing director says that now tens of thousands of fans call its N number each week to vote for the best return etc - profit from the calls goes to charity but abc sports also uses the calls as a sales tool after callers for voting frank offers a football for $ N and N N of callers stay on the line to order it - jackets may be sold next - meanwhile nbc sports recently began scores plus a 24-hour N line providing a complex array of scores analysis and fan news - a spokesman said its purpose is to bolster the impression that nbc sports is always there for people - nbc 's consumer minutes have increased advertiser spending during the day the network 's weakest period - each matches a sponsor and a topic on unilever n.v. 's bros. sponsors tips on diet and exercise followed by a bros. commercial - viewers can call a N number for additional advice which will be tailored to their needs based on the numbers they press one if you 're pregnant etc - if the caller stays on the line and leaves a name and address for the sponsor coupons and a newsletter will be and the sponsor will be able to gather a list of desirable potential customers - an vice president says nbc has been able to charge premium rates for this ad time - she would n't say what the premium is but it 's believed to be about N N above regular rates - we were able to get advertisers to use their promotion budget for this because they get a chance to do says ms. - and we were able to attract some new advertisers because this is something new - mr. parks of call interactive says tv executives are considering the use of N numbers for talk shows game shows news and opinion surveys - experts are predicting a big influx of new shows in N when a service called automatic number information will become widely available - this service each caller 's phone number and it can be used to generate instant mailing lists - hard copy the new syndicated tabloid show from paramount pictures will use its N number for additional purposes that include research says executive producer mark b. von s. - for a piece on local heroes of world war ii we can ask people to leave the name and number of anyone they know who won a he says - that 'll save us time and get people involved - but mr. sees much bigger changes ahead - these are just baby steps toward real interactive video which i believe will be the biggest thing yet to affect television he says - although it would be costly to shoot multiple versions tv programmers could let audiences vote on different for a movie - fox broadcasting with this concept last year when viewers of married with children voted on whether al should say i love you to on 's day - someday viewers may also choose different of news coverage - a by phone could let you decide i 'm interested in just the beginning of story no. N and i want story no. N in mr. says - you 'll start to see shows where viewers program the program - integrated resources inc. the troubled financial-services company that has been trying to sell its core companies to restructure debt said talks with a potential buyer ended - integrated did n't identify the party or say why the talks failed - last week another potential buyer financial group which had agreed in august to purchase most of integrated 's core companies for $ N million ended talks with integrated - integrated said that it would continue to pursue other alternatives to sell the five core companies and that a group of senior executives plans to make a proposal to purchase three of the companies integrated resources equity corp. resources trust co. and integrated resources asset management corp - a price was n't disclosed - integrated also said it expects to report a second-quarter loss wider than the earlier estimate of about $ N million - the company did n't disclose the new estimate but said the change was related to integrated 's failure to sell its core businesses as well as other events which it did n't detail that occurred after its announcement last week that it was in talks with the unidentified prospective buyer - meanwhile a number of top sales producers from integrated resources equity will meet this afternoon in chicago to discuss their options - the unit is a constructed group of about N independent brokers and financial planners who sell insurance annuities limited partnerships mutual funds and other investments for integrated and other firms - the sales force is viewed as a critical asset in integrated 's attempt to sell its core companies - cited concerns about how long integrated would be able to hold together the sales force as one reason its talks with integrated failed - in composite trading on the new york stock exchange yesterday integrated closed at $ N a share down N cents - integrated has been struggling to avoid a bankruptcy-law filing since june when it failed to make interest payments on nearly $ N billion of debt - integrated senior and junior creditors are owed a total of about $ N billion - an earthquake struck northern california killing more than N people - the violent temblor which lasted about N seconds and registered N on the richter scale also caused the collapse of a section of the san bay bridge and shook candlestick park - the tremor was centered near southeast of san francisco and was felt as far as N miles away - numerous injuries were reported - some buildings collapsed gas and water lines and fires - the quake which also caused damage in san jose and berkeley knocked out electricity and telephones roadways and disrupted subway service in the bay area - major injuries were n't reported at candlestick park where the third game of baseball 's world series was canceled and fans from the stadium - bush vowed to veto a bill allowing federal financing for abortions in cases of rape and incest saying tax dollars should n't be used to compound a violent act with the taking of an life - his pledge in a letter to democratic sen. byrd came ahead of an expected senate vote on spending legislation containing the provision - east germany 's politburo met amid speculation that the ruling body would oust hard-line leader honecker whose rule has been challenged by mass emigration and calls for democratic freedoms - meanwhile about N refugees flew to west germany from warsaw the first in east germany 's exodus - the world psychiatric association voted at an to the soviet union - moscow which left the group in N to avoid over allegations that political were being certified as could be suspended if the of against is discovered during a review within a year - nasa postponed the of the space shuttle atlantis because of rain near the site of the launch in fla - the flight was for today - the spacecraft 's five are to the galileo space probe on an mission to jupiter - senate democratic leaders said they had enough votes to defeat a proposed constitutional amendment to ban flag burning - the amendment is aimed at a supreme court ruling that threw out the conviction of a texas on grounds that his freedom of speech was violated - federal researchers said lung-cancer mortality rates for people under N years of age have begun to decline particularly for white males - the national cancer institute also projected that overall u.s. mortality rates from lung cancer should begin to drop in several years if cigarette smoking continues to - bush met with south korean president roh who indicated that seoul plans to further ease trade rules to ensure that its economy becomes as open as the other industrialized nations by the mid-1990s - bush assured roh that the u.s. would stand by its security commitments as long as there is a threat from communist north korea - the bush administration is seeking an understanding with congress to ease restrictions on u.s. involvement in foreign coups that might result in the death of a country 's leader - a white house spokesman said that while bush would n't alter a longstanding ban on such involvement there 's a needed on its interpretation - india 's gandhi called for parliamentary elections next month - the balloting considered a test for the prime minister and the ruling congress i party comes amid charges of leadership and government corruption - gandhi 's family has ruled independent india for all but five years of its history - the soviet union from a u.n. general assembly vote to reject israel 's credentials - it was the first time in seven years that moscow has n't joined efforts led by nations to israel from the world body and was viewed as a sign of improving ties - israel was by a vote of N with N - black activist walter sisulu said the african national congress would n't reject violence as a way to pressure the south african government into concessions that might lead to negotiations over apartheid - the sisulu was among eight black political activists freed sunday from prison - london has concluded that president was n't responsible for the execution of six british in world war ii although he probably was aware of the - the report by the defense ministry also rejected allegations that britain covered up evidence of 's activities as a german army officer - an international group approved a formal ban on ivory trade despite objections from southern african governments which threatened to find alternative channels for selling elephant - the move by the convention on trade in endangered meeting in switzerland places the elephant on the list - an in colombia killed a federal judge on a street - an caller to a local radio station said cocaine traffickers had the in for the of wanted on drug charges in the u.s. - leader met with egypt 's president and the two officials pledged to respect each other 's laws security and stability - they stopped short of diplomatic ties in N - the reconciliation talks in the desert town of followed a meeting monday in the egyptian resort of - group inc. revised its exchange offer for $ N million face amount of N N senior subordinated debt due N and extended the offer to oct. N from oct. N - the n.j. company said holders would receive for each $ N face amount $ N face amount of a new issue of secured senior subordinated notes convertible into common stock at an initial rate of $ N a share and N common shares - the new notes will bear interest at N N through july N N and thereafter at N N - under the original proposal the maker of specialty coatings and a developer of technologies offered $ N of notes due N N common shares and $ N in cash for each $ N face amount - completion of the exchange offer is subject to the tender of at least N N of the debt among other things - which said it does n't plan to further extend the offer said it received $ N face amount of debt under the original offer - the stock of ual corp. continued to be amid signs that british airways may at any of the aborted $ N billion buy-out of united airlines ' parent - ual stock plummeted a further $ N to $ N on volume of more than N million shares in new york stock exchange composite trading - the plunge followed a drop of $ N monday amid indications the takeover may take weeks to be revived - the stock has fallen $ N or N N in the three trading days since announcement of the collapse of the $ 300-a-share takeover jolted the entire stock market into its plunge ever - this is a total for takeover-stock traders one investment banker said - los angeles financier marvin davis who put united in play with a $ N billion bid two months ago last night both a ray of hope and an extra element of uncertainty by saying he remains interested in acquiring ual - but he dropped his earlier $ 300-a-share bid saying he must first explore bank financing - even as citicorp and chase manhattan corp. scrambled to line up bank financing for a revised version of the labor-management bid british airways a N N partner in the buying group indicated it wants to start from - its partners are united 's pilots who were to own N N and ual management at N N - adding to injury united 's machinists ' union which helped scuttle financing for the first bid yesterday asked ual chairman stephen wolf and other ual directors to resign - a similar demand was made by a group that represents some of united 's N employees - john machinists union general vice president attacked mr. wolf as greedy and irresponsible for pursuing the buy-out - although mr. wolf and john pope ual 's chief financial officer stood to $ N million for stock and options in the buy-out ual executives planned to reinvest only $ N million in the new company - the blue-collar machinists longtime rivals of the white-collar pilots say the would load the company with debt and weaken its finances - confusion about the two banks ' efforts to round up financing for a new bid that the ual board has n't even seen yet helped send ual stock downward - and rumors of forced selling by takeover-stock traders triggered a in the dow jones industrial average around N a.m. edt yesterday - yesterday 's selling began after a japanese news agency reported that japanese banks which balked at the first bid were ready to reject a revised version at around $ N a share or $ N billion - several reports as the day gave vague or indications about whether banks would sign up - citicorp for example said only that it had of interest of a transaction from both the borrowers and the banks but did n't have an agreement - late in the day mr. wolf issued a statement calling mr. 's blast divisive and for - but he gave few details on the progress toward a new bid saying only we are working toward a revised proposal for majority employee ownership - meanwhile in another sign that a new bid is n't imminent it was learned that the ual board held a telephone meeting monday to hear an update on the situation but that a formal board meeting is n't likely to be until early next week - in london british airways chairman lord king was quoted in the times as declaring he is not prepared to take my shareholders into a deal - observers said it appeared that british air was angered at the way the bid has into confusion as well as by the banks ' effort to round up financing for what one called a deal that is n't a deal - the effort to revive the bid was complicated by the nature of the buying group - the pilots were meeting outside chicago yesterday - but british air which was to have supplied $ N million out of $ N million in equity financing apparently was n't involved in the second proposal and could well reject it even if banks obtain financing - a group of united 's employees said in a statement the fact that wolf and other officers were going to line their pockets with literally millions of dollars while severe pay cuts on the employees of united is not only but - the machinists also asked for an investigation by the securities and exchange commission into possible violations in the original bid for ual by mr. davis as well as in the response by ual - last week just before the bank commitments were due the union asked the u.s. labor department to study whether the bid violated legal standards of fairness governing employee investment funds - in his statement mr. wolf said we continue to believe our approach is sound and that it is far better for all employees than the alternative of having an outsider own the company with employees paying for it just the same - mr. wolf has merger advice from a major wall street securities firm relying instead only on a takeover lawyer peter of slate & flom - the huge drop in ual stock prompted one takeover stock trader george managing partner of & co. to deny publicly rumors that his firm was going out of business - mr. said that despite losses on ual stock his firm 's health is excellent - the stock 's decline also has left the ual board in a - although it may not be legally obligated to sell the company if the buy-out group ca n't revive its bid it may have to explore alternatives if the buyers come back with a bid much lower than the group 's original $ 300-a-share proposal - at a meeting sept. N to consider the labor-management bid the board also was informed by its investment adviser first boston corp. of interest expressed by buy-out funds including kohlberg kravis roberts & co. and little & co. as well as by robert bass morgan stanley 's buy-out fund and pan am corp - the takeover-stock traders were hoping that mr. davis or one of the other interested parties might with the situation in disarray or that the board might consider a recapitalization - meanwhile japanese bankers said they were still about accepting citicorp 's latest proposal - macmillan inc. said it plans a public offering of N million shares of its berlitz international inc. unit at $ N to $ N a share - the offering for the language school unit was announced by robert maxwell chairman and chief executive officer of london-based maxwell communication corp. which owns macmillan - after the offering is completed macmillan will own about N N of the berlitz common stock outstanding - five million shares will be offered in the u.s. and N million additional shares will be offered in international offerings outside the u.s. - goldman sachs & co. will manage the offering - macmillan said berlitz intends to pay quarterly dividends on the stock - the company said it expects to pay the first dividend of N cents a share in the N first quarter - berlitz will borrow an amount equal to its expected net proceeds from the offerings plus $ N million in connection with a credit agreement with lenders - the total borrowing will be about $ N million the company said - proceeds from the borrowings under the credit agreement will be used to pay an $ N million cash dividend to macmillan and to lend the remainder of about $ N million to maxwell communications in connection with a note - proceeds from the offering will be used to repay borrowings under the short-term parts of a credit agreement - berlitz which is based in princeton n.j. provides language instruction and translation services through more than N language centers in N countries - in the past five years more than N N of its sales have been outside the u.s. - macmillan has owned berlitz since N - in the first six months of this year berlitz posted net income of $ N million on sales of $ N million compared with net income of $ N million on sales of $ N million - right away you notice the following things about a philip glass concert - it attracts people with funny hair or with no hair in front of me a girl with sat a boy who had his - whoever constitute the local left bank come out in force dressed in black along with a of who want to be on the cutting edge - people in glass houses tend to look - and if still at the evening 's end you notice something else the audience at first and by the music releases its feelings in collective - currently in the middle of a tour as a solo mr. glass has left behind his equipment and in favor of going it alone - he sits down at the piano and plays - and plays - either one likes it or one does n't - the typical glass audience which is more likely to be composed of music students than their teachers certainly does - the work though sounds like for - philip glass is the and his music the new clothes of the - his success is easy to understand - introducing and explaining his pieces mr. glass looks and sounds more like a describing his work than a classical playing a recital - the piano which have been labeled as cyclical and are therefore therefore and but therefore both pretty and - it is music for people who want to hear something different but do n't want to work especially hard at the task - it is listening for the now generation - mr. glass has the famous less is more - his more is always less - far from being the music us with apparent not so in the of N time and or - but the music has its and mr. glass has constructed his solo program around a move from the simple to the relatively complex - opening N from the audience to the glass technique never too far from the piano 's center mr. glass works in the two on either side of middle c and his fingers seldom leave the - there is a musical style here but not a particular performance style - the music is not especially indeed it 's hard to imagine a bad performance of it - nothing no no problems challenge the performer - we hear we may think inner voices but they all seem to be saying the same thing - with planet news music meant to of allen 's wichita mr. glass gets going - his hands sit apart on the - seventh make you feel as though he may break into a very slow - the but there is little even though his fingers begin to over more of the - contrasts predictably first the music is loud then it becomes soft then you realize it becomes again - the fourth play an from on the beach is like a but it does n't seem to move much beyond its ground in three blind mice - when mr. glass decides to get really fancy he his hands and hits a bass note with his right hand - he does this in at least three of his solo pieces - you might call it a or a - in mad rush which came from a commission to write a piece of length mr. glass and confessed that this was no problem for me an a with a b section several times before the piece ends - not only is the typical it is also often multiple in its context s - mad rush began its life as the to the lama 's first public address in the u.s. when mr. glass played it on the at new york 's of st. john the - later it was performed on radio in germany and then took it for one of her dance pieces - the point is that any piece can be used as background music for virtually anything - the evening ended with mr. glass 's another multiple work - parts N N and N come from the of morris 's film the thin blue line and the two other parts from music to two separate of the story of the same name - when used as background in this way the music has an appropriate as when a phrase a minor third the seemingly endless of reports interviews and of witnesses in the morris film - served up as a solo however the music lacks the provided by a context within another medium - of mr. glass may agree with the critic richard 's sense that the N music in twelve parts is as and as the - but while making the obvious point that both develop variations from themes this comparison the intensely nature of mr. glass 's music - its supposedly a that makes one for the of the radical of and and what in even seems like in - mr. is professor of english at southern university and editor of the southwest review - honeywell inc. said it hopes to complete shortly the first of two sales of shares in its japanese joint venture for about $ N million - the company would n't disclose the buyer of the initial N N stake - proceeds of the sale expected to be completed next week would be used to repurchase as many as N million shares of honeywell stock the company said - honeywell said it is negotiating the sale of a second stake in but indicated it intends to hold at least N N of the joint venture 's stock long term - a N N stake would allow honeywell to include earnings in its results - honeywell previously said it intended to reduce its holding in the japanese concern as part of a restructuring plan which also calls for a reduction of on weapons sales - yesterday a spokeswoman said the company was pleased with our progress in that regard and hopes to provide additional details soon - honeywell said its defense and marine systems group incurred delays in shipping some undisclosed contracts during the third quarter resulting in lower operating profit for that business - overall honeywell reported earnings of $ N million or $ N a share for the three months ended oct. N compared with a loss of $ N million or N cents a share a year earlier - the previous period 's results included a $ N million pretax charge related to contract costs and a $ N million pretax gain on real estate sales - sales for the latest quarter were flat at $ N billion - for the nine months honeywell reported earnings of $ N million or $ N a share compared with earnings of $ N million or $ N a share a year earlier - sales declined slightly to $ N billion - once again your editorial page the law to conform to your almost - in an of little to his central point about private enforcement suits by environmental groups michael s. your readers the clean water act is written upon the the rather that nothing but zero risk will do it a legal standard of zero environmental sept. N - this statement surely your editorial viewpoint that environmental protection is generally silly or excessive but it is simply wrong - the clean water act contains no legal standard of zero - it requires that of into the waters of the united states be authorized by permits that reflect the limitations developed under section N - whatever may be the problems with this system it reflects zero risk or zero - perhaps mr. was confused by congress 's statement of the national goal in section N which indeed calls for the elimination of by N no less - this statement was not taken seriously when enacted in N and should not now be confused with the provisions of the statute - thus you do the public a great when mr. suggests even that the clean water act prohibits the preparation of a and water your readers may be led to believe that nothing but chance or oversight protects them as they in the night with their and waters from the knock of the sierra club at their doors - robert j. - national geographic the u.s. magazine is attracting more readers than ever and offers the glossy pages that upscale advertisers love - so why did advertising pages plunge by almost N N and ad revenue by N N in the first half - to hear advertisers tell it the magazine just has n't kept up with the times - despite renewed interest by the public in such topics as the environment and the third world it has n't been able to shake its reputation as a magazine boys like to through in search of tribe women - worse it lagged behind competitors in offering from regional editions to discounts for frequent advertisers - but now the magazine is attempting to fight back with an ambitious plan including a revamped sales strategy and a surprisingly aggressive ad campaign - advertisers do n't think of the magazine first says joan who joined in april as national advertising director - what we want to do is take a more aggressive stance - people did n't believe we were in tune with the marketplace and in many ways we were n't - the magazine has never had to woo advertisers with quite so much before - it largely on its N million subscribers in the first half up from N million a year ago an average age of N for readers at the of their years loyalty to the tune of an N N average subscription renewal rate - the magazine had its best year yet in N when it its centennial and racked up a N N gain in ad pages to N - but this year when the surrounding its centennial died so too did some advertiser interest - the reason ad executives say is that the entire magazine business has been soft and national geographic has some that make it especially during a soft market - perhaps the biggest of those factors is its high ad prices $ N for a page vs. $ N for the a comparable publication with a far smaller circulation - when ad dollars are tight the high page cost is a major for advertisers who generally want to appear regularly in a publication or not at all - even though national geographic offers far more readers than does a magazine like the page costs you an arm and a leg to develop any frequency says harry glass new york media manager for bozell inc - to combat that problem national geographic like other magazines began offering regional editions allowing advertisers to appear in only a portion of its magazines for example ads can run only in the magazines sent to subscribers in the largest N markets - but the magazine was slower than its competitors to come up with its regional editions and until last year offered fewer of them than did competitors - time magazine for example has more than N separate editions going to different regions top management and other groups - another sticking point for advertisers was national geographic 's tradition of its ads together usually at the beginning or end of the magazine rather than spreading ads out among its articles as most magazines do - and national geographic 's size means extra production costs for advertisers - but ms. says the magazine is fighting back - it now offers N regional editions it very recently began running ads adjacent to articles and it has been up its sales force - and it just launched a promotional campaign to tell chief executives marketing directors and media executives just that - the centerpiece of the promotion is its new ad campaign into which the magazine will pour about $ N mostly in the next few weeks - the campaign created by group 's ddb needham agency takes advantage of the photography that national geographic is known for - in one ad a photo of the interior of the in paris is with the headline the only book more respected than does n't accept advertising - another ad pictures a tree magnified N times with the headline for impact far beyond your size consider our regional editions - ms. says she wants the campaign to help attract advertisers in N categories including corporate financial services consumer electronics insurance and food - her goal to top N ad pages in N up from about N this year - whether she can meet that ambitious goal is still far from certain - the ad campaign is meant to the thought of national geographic she says - we want it to be a kind of image - wcrs plans sale - wcrs group hopes to announce perhaps today an agreement to sell the majority of its ad unit to eurocom a european ad executive said - wcrs has been in discussions with eurocom for several months - however when negotiations down recently wcrs 's chief executive peter scott met in paris with another french firm or - according to the executive 's involvement prompted renewed in the talks and the two agencies were hoping to out details by today - executives of the two agencies could n't be reached last night - ad notes - new account procter & gamble co. cincinnati awarded the ad accounts for its line of professional and oil products to cincinnati - billings were n't disclosed - professional products are specially made for the industry - who 's news stephen N was named executive vice president deputy creative director at grey advertising new york - he was executive vice president director of broadcast production - the commodity futures trading commission plans to restrict dual trading on commodity exchanges a move almost certain to exchange officials and traders - the cftc said it will propose the restrictions after the release of a study that shows little economic benefit resulting from dual trading and cites problems associated with the practice - dual trading gives an exchange trader the right to trade both for his own account and for customers - the issue exploded this year after a federal bureau of investigation operation led to charges of widespread trading abuses at the chicago board of trade and chicago mercantile exchange - while not specifically mentioned in the fbi charges dual trading became a focus of attempts to tighten industry regulations - critics contend that traders were putting buying or selling for their own accounts ahead of other traders ' customer orders - traders are likely to oppose such restrictions because dual trading provides a way to make money in slower markets where there is a shortage of customer orders - the exchanges contend that dual trading improves liquidity in the markets because traders can buy or sell even when they do n't have a customer order in hand - the exchanges say liquidity becomes a severe problem for traded contracts such as those with a long time remaining before expiration - the cftc may take those arguments into account by allowing exceptions to its restrictions - the agency did n't cite specific situations where dual trading might be allowed but smaller exchanges or contracts that need additional liquidity are expected to be among them - wendy the agency 's chairman told the senate agriculture committee that she expects the study to be released within two weeks and the rule changes to be completed by - the study by the cftc 's division of economic analysis shows that a trade is a trade a member of the study team said - whether a trade is done on a dual or basis the member said does n't seem to have much economic impact - currently most traders on commodity exchanges specialize in trading either for customer accounts which makes them brokers or for their own accounts as - the tests indicate that dual and traders are similar in terms of the trade executions and liquidity they provide to the market mrs. told the senate panel - members of congress have proposed restricting dual trading in bills to cftc operations - the house 's bill would prohibit dual trading in markets with daily average volume of N contracts or more those considered too difficult to track without a sophisticated computer system - the senate bill would force the cftc to suspend dual trading if an exchange ca n't show that its oversight system can detect abuses - so far one test of restricting dual trading has worked well - the chicago merc banned dual trading in its standard & poor 's 500-stock index futures pit in N - under the rules traders decide before a session begins whether they will trade for their own account or for customers - traders who stand on the pit 's top step where most customer orders are executed ca n't trade for themselves - a merc spokesman said the plan has n't made much difference in liquidity in the pit - it 's too soon to tell but people do n't seem to be unhappy with it he said - he said he would n't comment on the cftc plan until the exchange has seen the full proposal - but at a meeting last week tom the board of trade 's president told commodity lawyers dual trading is definitely worth saving - it adds something to the market - japanese firms push car - japanese luxury-car makers are trying to set strict design standards for their dealerships - but some dealers are negotiating terms while others decline to deal at all - nissan motor co. 's infiniti division likes to insist that every dealer construct and a building in a japanese style - specifications include a at the center of each showroom and a bridge a stream that flows into the building from outside - infiniti has it down to the says jay a partner at power & associates an auto research firm - toyota motor corp. 's lexus division also provides specifications - but only two-thirds of lexus dealers are new buildings according to the lexus - some are even coming up with their own novel designs - in louisville ky. for example david peterson has built a lexus dealership with the showroom on the second floor - yet some dealers have turned down infiniti or lexus because they were unwilling or unable to meet the design requirements - lee seidman of cleveland says infiniti was a bear on but at least let him an existing building without the stream - mr. seidman says he turned down a lexus franchise in part because the building was but very expensive - to head off arguments infiniti offers dealers cash bonuses and construction loans - device 's plays back a lesson - products have to be first to be winners - that 's the lesson offered through one case study featured in a design exhibit - dictaphone corp. was caught off guard in N when its main competitor office products of japan introduced a recorder half the size of standard devices - blocked by patent protection from following suit dictaphone decided to go a step further and cut the in half again down to the length of a - by N designers and engineers at dictaphone a pitney bowes subsidiary had produced a working model of a recorder - by N however the patent status of the had changed permitting dictaphone to develop its own competitive micro system which it did - marketing and sales departments then urged of the project - but others said should proceed - both were right - dictaphone went ahead and introduced the in N but it has n't sold well - to date says a dictaphone vice president it has broken even or shown a small loss - nevertheless the device has been successful in other ways - it helped dictaphone attract better engineers and it provided new technology for other company products - the recorder also helped transform the company 's reputation from to - it gave me great pride to see the inventor of the in japan look at the and shake his head and say says mr. - dictaphone 's recorder is one of N case studies in the design project sponsored by the design management institute of boston and harvard business school - the studies are on exhibit at harvard this month and will travel to chicago 's institute of design and the university of california at berkeley - a rake 's progress means out - one day carl barrett of mobile ala. was some leaves but the rake kept riding up over the - the harder he tried to push them into large the closer he came to breaking the rake and his back - so mr. barrett then vice president of the alabama association took a garden rake and taped it to the of a rake about nine inches up - his crude device worked the lower teeth gathered the leaves into a pile while the higher harder teeth moved the top of the pile - now incorporated into a rake the or also are supposed to aid in picking up leaves - one customer donald of mobile says the barrett rake allowed him to do his lawn in N N hours two hours less than usual - but other rake makers have their doubts - richard mason president of co. in w. va. says the barrett rake makes sense but it would be tough to explain to consumers - john marketing director for true corp. a subsidiary of black & decker says people do n't want to move a pile - they either pick it up he says or they start pulling from a fresh direction - odds and ends - no more or promises corp. of ind. the designer of a bed support to replace traditional - four steel each roughly in the shape of a are attached to the bottom of the box spring in a position - nearly half of u.s. consumers say they 'll pay up to N N more for packaging that can be recycled or is according to a survey commissioned by the michael peters group a design consultant - the pentagon is a house - living there for six years was really scary - the ghosts of the past are everywhere they are kept at bay only by feeding them vast quantities of our defense budget - some can be bought off relatively - during the korean war gen. douglas demanded and got in addition to his u.n. command in korea his own naval command in japan - those operations cost less than $ N billion a year and keep mac 's ghost quiet - that 's about all it costs to adm. erich 's ghost - in N and the german navy threatened to attack the panama so we created the southern command in panama - the southern command has grown even bigger since the war because 's ghost sometimes runs through the e ring dressed like gen. noriega - the command 's huge bureaucracy is needed to analyze whether leaders of coups against gen. noriega meet the war powers act 's six points cap 's seven points the intelligence committee 's N points and wilson 's N points necessary to justify u.s. support - so far no one has - the ghost of the soviet discovered in cuba back in the costs just a few hundred million the price of the caribbean command in key west that president carter created in N - the has n't been heard from since but we keep the staff around just in case - george marshall 's ghost is much more difficult to keep happy - we keep a lot of to him around the pentagon and such - the army headquarters on the third deck of the pentagon used to a lot of to him but the navy headquarters on the fourth deck made them stop it - you see marshall had this thing about the navy and the he wanted to make them part of the army but secretary of the navy james blocked him - now his ghost wo n't let up till it 's done - to keep him quiet we a new unified command every year or so run by the army or the air force and put more of the navy and under it - but we still hear him at night because the navy has a few ships left and to satisfy him the navy 's sea lift forces were given to a new air force bureaucracy in illinois its space operations to another command in colorado the to a new army bureaucracy in fort and the navy 's indian ocean and persian gulf forces to an army bureaucracy in florida - which brings up the worst and ghost of all the ghost of the shah of iran - when the shah died president carter was so scared that the shah 's ghost would blame him for him out to make way for the that he declared the carter doctrine - mr. carter said he would go to war to stop anyone from trying to grab iran - but that ghost would n't settle for words he wanted money and people lots - so mr. carter formed three new army divisions and gave them to a new bureaucracy in tampa called the rapid force - but that ghost was n't he knew the was neither rapid nor nor a force even though it cost $ N billion or $ N billion a year - after mr. carter was defeated in N the shah 's ghost claimed the credit and then went after president reagan and cap - i saw what he did to them - it made my dance with - why he used to lay in wait for cap suddenly he 'd leap from behind some of marshall onto cap 's and grab him by the and him till he up an additional $ N billion or so - cap added four more divisions to the army two active and two reserve two carrier groups to the navy a division equivalent to the and the and a thousand tactical aircraft to the air force - he bought $ N billion in ships and $ N billion in and equipment to fill them and them at a new $ N billion base at diego garcia in the middle of the indian ocean - he dedicated all these new forces to the persian gulf - one night both marshall 's ghost and the shah 's ghost together caught cap and threw him to the ground - before they let him go he added a thousand bureaucrats to the in tampa and renamed it central command - he gave those bureaucrats charge of all naval operations in the persian gulf and indian ocean - marshall figured it would be good training for those soldiers someday maybe they would get the whole navy - they had fun moving the carriers around but it turned out that they had forgotten all about mine - but the shah still kept leaping out at cap so cap bought a hundred merchant ships more and $ N billion of etc. in order that those seven new army divisions and three marine could unload from all those new ships and aircraft and go to war in the - then suddenly 's ghost came to visit and said what the hell are you doing planning for a land war in asia N miles away - we 'd get our kicked - lucky for cap was and soon went away while the shah he kept coming back - so the u.s. found itself paying about $ N billion in to various arab for rights around the indian ocean - we had great success in somalia - but then it turned out that president was not at all a nice person and the navy pointed out that the base he promised us in had up about a hundred years ago and anyway was N miles from the mouth of the gulf - but who 's counting - still was the best we could get so we stay in bed with president - all these reports about him committing are probably anyway - but would n't you know now that we are spending of dollars and have built those new divisions and new air wings and have positioned all these ships and supplies to fight the russians in iran the russians seem to have lost interest in the whole subject - meanwhile congress is cutting huge chunks out of the rest of the defense budget - predictably some navy guys said do we still need to keep all N army divisions on active duty and all those extra aircraft without bases and all those army guys playing in tampa - could n't we save $ N billion or $ N billion a year by shifting that stuff to the reserves - and why not save the costs of a thousand bureaucrats by central command and putting responsibility for gulf naval operations back where it belongs afloat with the task force in the gulf - and where were all our paid indian ocean allies last year when our were being attacked - questions like that really stir up marshall 's ghost - he appeared late one night in the of the new defense secretary dick cheney - marshall came in like 's ghost dragging those chains of and air wings and links with arab - he would n't leave until mr. cheney promised to do whatever the pentagon systems analysts told him - so next day mr. cheney went out and did just that he canceled the navy and cut back one carrier and N - then he canceled production of the navy 's most important carrier aircraft the f-14 and the - on the other hand mr. cheney retained all those new land forces - marshall 's ghost is satisfied for now but he 'll be back - what with halloween coming and bigger defense cuts looming more and more pentagon bureaucrats are under their desks - they know that they can hold off the ghosts only a little while longer by cutting carriers and ships - then the whole thing will start to collapse just as it did in the 1970s and the ghosts and will be through the place turning people 's hair white - gives me the just thinking about it - mr. lehman a reagan navy secretary is a managing director of painewebber - the metal and marble lobby of centrust bank 's headquarters is than your average savings and loan - for one thing there is an old master on the wall samuel david a big painted by a - at the moment however the painting is a nagging reminder of the problems that have centrust and its flamboyant chairman and chief executive david l. paul - in an international buying spree that began barely two years ago mr. paul a collection of about N works including the at a total cost of $ N million - by midnight oct. N all of the paintings were supposed to have been sold off under orders from florida 's comptroller whose office the state 's s&ls - centrust did n't meet the deadline - the collection was at the heart of a plan mr. paul had in which the art was to do double duty as an investment for centrust and as for the s&l 's new office tower designed by - the is that the $ N million was from the funds of this federally insured institution even as centrust was losing money hand over - mr. paul had no right to buy art for the s&l in the first place it is n't on the comptroller 's permissible list without seeking a special which he did not do - besides that some of the paintings that were to grace the walls of centrust actually ended up hanging in the chairman 's estate on la off miami beach - last spring the comptroller 's office called a halt to mr. paul 's giving him six months to sell the paintings - the acquisitions officials said in a letter to mr. paul were and unauthorized - so far mr. paul has but three of his he wo n't say to whom - the comptroller 's office says it is monitoring the situation - though the agency could remove mr. paul it has no current intention to do that - it 's not like selling mr. paul says as he takes a drag on a st. cigarette - the last six months has established the quality of the collection - there 's no fire sale here - despite mr. paul 's characteristic the is finding that getting centrust florida 's largest thrift institution out of its investments is much tougher than getting into them had been - paintings are just part of the picture - although mr. paul has a $ N billion junk-bond portfolio to less than $ N million since april the high-yield debt market has plummeted - itself of what is left as is required of all thrift institutions by july N under the new federal s&l bailout law may well prove difficult - and centrust has other problems - late last week federal regulators ordered the thrift institution to stop paying dividends on its preferred stock a move that suggests deep concern about an institution - mr. paul has a plan to bring in $ N million by selling off N of centrust 's N branches but it has yet to be approved by regulators - it is mr. paul 's art venture however that has drawn the most attention from investors and regulators not to mention throughout the world - shareholders some of whom are suing say the chairman and his collection the excesses of speculation that set off the national s&l crisis - centrust shares have fallen sharply in price from a high of $ N in N to close yesterday at $ N - gallery directors meanwhile say mr. paul and others of his have left an mark on the art world and not for the better - collectors do n't say it 's a van anymore harry brooks the president of & co. a new york gallery - they say got $ N million for his so certainly $ N million is n't too much for mine - the great collectors we depended on such as paul mellon or norton simon have stopped buying and the new buyers are brilliant men who made money in the stock market or in takeovers and rushed into collecting - mr. an art dealer and sold vincent van 's at a sotheby 's auction in november N to australian businessman alan bond - trouble is mr. bond has yet to pay up and until he does sotheby 's has the painting under lock and key - when mr. paul moved in on the art market he let it be known that virtually no piece was too costly to be considered by centrust - he established his reputation as a in january last year at sotheby 's auction of the linda and gerald guterman collection in new york - there on one of his first shopping trips mr. paul picked up several paintings at stunning prices - he paid $ N million for instance for a still life by jan that was expected to fetch perhaps $ N - the price paid was a record for the artist - some N N of items offered at the guterman auction were sold at an average price of $ N - the rest were withdrawn for lack of acceptable bids - afterward mr. paul is said by mr. guterman to have mr. guterman the new york developer selling the collection and - he says he them recalls mr. guterman - and he tells me if you want to see your paintings you 'll have to come to my house in florida - mr. paul denies and - it 's just not true he says - mr. paul quickly became more aggressive in his collecting with the help of george wachter a sotheby 's expert in old masters whom he met at an exhibition of the guterman items - mr. wachter who became his principal adviser searched in london paris and - and according to one dealer mr. wachter had a for introducing mr. paul with the phrase he can buy anything - nicholas hall the president of the u.s.a. ltd. gallery in new york sold mr. paul and in the by giovanni - mr. hall says mr. paul was known to spend a lot of money - people were interested in seeing him but it was recognized that the route was through sotheby 's and particularly george wachter - mr. paul thus developed a close relationship with sotheby 's - mr. paul was eager to a collection for the headquarters centrust has been moving into for the greater part of a year - sotheby 's the auction house founded in london N and now under the of sotheby 's holdings inc. was hoping to stir up interest in old masters as it to build its u.s. business - european dealers continued to dominate the action in old masters which sotheby 's north america had lately been touting in this country - for several months there was optimism all around - last october mr. paul paid out $ N million of centrust 's cash plus a $ N million commission for portrait of a man as - the painting attributed to artist peter paul rubens was purchased privately through sotheby 's not at auction - in march N just N months into his campaign mr. paul was named by art & magazine as one of the top N individual collectors in the u.s. - an unknown quantity to most of the art world paul is no to spending the magazine said noting that he does n't stop at on but also spends big on art you can eat - he recently bid $ N at a paris charity auction for a dinner by six of the world 's great chefs but the final party cost closer to $ N - mr. paul says it was n't that high - the art collection might have come to rival the ' had the florida comptroller 's office not got wind of mr. paul 's - in its letter to him dated march N and shared with reporters alex the chief of the bureau in the comptroller 's office expressed that the s&l could be so when it had reported losses of more than $ N million in its two preceding quarters - the state gave centrust N days to sell the rubens - the comptroller 's office eventually extended the deadline to six months but its demands ordering that the book value of the collection be reduced to zero - in other words get rid of all the pictures - the state noted that banking practices are grounds for removing an officer or director and closed with the to mr. paul govern yourself - the state agency was particularly to learn that the rubens and a other paintings listed among the bank 's furniture and were actually hanging in the chairman 's house - mr. paul says that at one point he did indeed have eight or nine of the paintings at home and that the rest were in storage at sotheby 's - he explains that he was merely the paintings at home with some display because of the special environment required for their until centrust 's new building was ready for them - still the incident was embarrassing - it came on the heels of a number of local newspaper articles suggesting that mr. paul has benefited from his association with centrust - for instance he got a $ N million loan from the s&l negotiated at a rate - he owns N N of centrust 's shares - adding to mr. paul 's problems dealers some with vested interests insist that he relying rather too heavily on sotheby 's advice paid much too much for several pieces in the centrust collection - the $ N million on the rubens for example was a record price for the artist and maybe twice its value given a dispute among scholars about its - david the president of david inc. a new york gallery says scholars question the of the rubens - it may have been painted instead by a rubens associate - the feeling among many experts on the commercial side is that the price paid at the time was excessive in any event mr. says - it sounds like with the rubens he got absolutely taken to the - victor the executive director of the association of america agrees that mr. paul paid very for the rubens and adds that getting rid of it any time soon for a similar sum would be quite a feat - it 's not beyond credibility the rubens will someday be worth $ N million but whether it could be sold for that amount tomorrow remains to be seen - still predicting is tricky - i 'm forever by what i see making these high prices - jonathan h. the son of the painting 's former owner mrs. rush the price talk as sour - dealers of the purchase price he says were themselves interested in buying the rubens but lost out - mr. paul for his part the rubens price saying a lot of the experts have never seen the thing itself - most of them were n't even born the last time the painting was displayed publicly he says - art prices are but a good deal of is involved in statistics on sales - salomon brothers inc. the investment-banking firm in its annual tally of investment returns reported that old masters N N in the year ended june N the greatest return of any of N assets it tracked - and modern paintings not tracked by salomon are ranked even higher at N N by sotheby 's - salomon moreover gets its data on art appreciation from sotheby 's whose prices go up with clients like mr. paul in its - the from consideration the many paintings that go at auction - art indexes track winners not losers - but art that has fallen sharply in value is rarely put up for sale - also at any of sotheby 's auctions of old masters roughly one-third to of what is offered does n't sell at any price - it 's not that there are n't any bids but the bids do n't meet the minimum reserve prices set by the sellers - in january the painting that now hangs at centrust was expected to bring no more than $ N at auction until mr. paul came along with his $ N million - mr. hall of the gallery says $ N million would have been an impossible price for anyone to ask for a four years ago - but from his point it is n't that mr. paul a customer of his too for the work a painting by an artist who is not a household word - the painting is N feet wide seven feet high - rather it just shows things have changed - mr. paul boasts that he spotted bargains in old masters just before they took an upward turn - they went up N N last year and they 'll do it again this year he declares - they were a - everybody was out buying - sotheby 's vice president says the auction house has been mr. paul in selling the paintings - and while sotheby 's chief rivals in the art world private art dealers wo n't be happy to hear it she adds a number of the have already been sold and at a substantial profit - mr. paul claims to have sold three paintings at more than a N N profit - that is n't N N and the claim is n't - he furthermore denies that he relied too heavily on sotheby 's or mr. wachter - mr. paul says he had not one but four advisers and that he never bid - after all he had the counsel of from the most reputable in the world - he says he expects to sell the collection including the controversial rubens carefully and just as it was put together - but in mr. paul 's holdings are - that is he is being to put them on the market too soon and has already gotten offers that are less than he paid for some of the art works - after a few years you can argue there has been natural appreciation says susan the publisher of leonard 's annual price index of art auctions - but quick turnover in is like your jewelry you end up with N N - people hold out and try to get a bargain - sotheby 's itself and mr. paul in the matter - mr. wachter says mr. paul was a quick study who worked intensely and bought the best pictures available at the moment - on occasion he paid a high price mr. wachter concedes but he says those who bid less and dropped out were dealers who would then have marked up the paintings to them at a profit to collectors - a at associates in san francisco considers it conflict of interest for an auction house to both advise a client on purchases and to set price estimates on the paintings to be purchased - sotheby 's she says is wearing both hats - i ca n't see why there would be a conflict of interest says sotheby 's ms. - estimates are based on the previous price of similar works sold at auction and current market conditions and are not affected by any knowledge of who the potential buyer could be - frequently clients express interest in paintings but do n't end up bidding she adds so we do n't know who the potential buyer will be - mr. paul in selling off his paintings is seeking at least a N N return on the bank 's investment so as to prove that the venture was sound - mr. paul says that he has out over much of the globe and that potential buyers from as far away as japan and italy have examined the collection - because of the pressure on centrust to sell dealers and collectors have been trying to get the paintings at prices - but so far mr. paul and his advisers are holding fast - one dealer martin of french & co. in new york says he would have loved to buy a jan de painting from the bank - i tried to steal the picture to buy it and sotheby 's would n't do it - they were protecting his interests - meanwhile mr. paul and centrust executives are getting about - mr. paul has been characterized as the great or something complains karen e. an executive vice president of centrust - the media she says have distorted his personal life - mr. paul in agreement - i do n't think i have a life style that is frankly so flamboyant he says - but at just that moment he is interrupted in his office by a in who coffee from silver into a cup of china and the with - mr. paul says yes the ceiling in his executive is - the offices are done in and books and of course a $ N million rubens - but he that the be played down - do n't say it 's a gold ceiling - just say the offices are appointed he says - otherwise the regulators will take it for and everything 's got to be - figures do n't include taxes or transaction costs - companies listed below reported quarterly profit substantially different from the average of analysts ' estimates - the companies are followed by at least three analysts and had a minimum five-cent change in actual earnings per share - estimated and actual results involving losses are omitted - the percent difference compares actual profit with the 30-day estimate where at least three analysts have issues forecasts in the past N days - otherwise actual profit is compared with the 300-day estimate - during its centennial year the wall street journal will report events of the past century that stand as milestones of american business history - creative accounting mostly by forced to change their way of setting standards to be followed by corporations reporting financial results standards that had become all too flexible - the new financial accounting standards board fasb was created in N to replace the accounting principles board of the american institute of certified public accountants - all of the former board 's members were criticism because they were writing rules while handling clients ' books at the same time - the new board 's structure kept four but the others were from industry and - francis m. wheat a former securities and exchange commission member headed the panel that had studied the issues for a year and proposed the fasb on march N N - the former board had produced N opinions and N critics in its 12-year life its chairman had conceded - the climate was right for the new fasb - in the late 1960s some failed to correct such abuses as clients picking rules that earnings and stock prices - and in november N congress had passed a special act to one board rule - also james needham an sec commissioner in april N had warned that the industry might face a federal agency writing accounting rules if they rejected the fasb idea - of the books dubbed figure the threat - the fasb had its initial meeting on march N N - on dec. N N it issued its first rule it required companies to disclose foreign currency in u.s. dollars - the fasb since then has issued N rules and some still industry - since late N for example it has put off a rule dealing with deferred income taxes because of the continuing controversy over the issue - industrial corp. said it plans to repurchase N shares or about N N of its shares outstanding in open market transactions - the metal products concern currently has N million common shares outstanding - previously had said it planned to repurchase shares but did n't disclose when or how many shares it intended to buy back - the company named dillon read & co. as its exclusive agent for the stock buy-back program - a seat on the chicago board of trade was sold for $ N down $ N from the previous sale last tuesday - seats currently are quoted at $ N bid $ N asked - the record price for a full membership on the exchange is $ N set aug. N N - an associate member seat was sold for $ N up $ N from the previous sale oct. N - associate member seats currently are quoted at $ N bid $ N asked - the record price for associate membership is $ N set aug. N N - industries ltd. said its link flight division was awarded a contract by the u.s. army for two helicopter which the company valued at as much as N million canadian dollars us$ N million - said the fixed price for the first of the combat mission is c$ N million - it is scheduled for delivery in late N - the price of the second ranges between c$ N million and c$ N million said depending on when the army exercises its option - is a toronto-based maker of commercial and military aircraft and training equipment - inc. said it agreed to team with a unit of honeywell inc. to provide power for a new military system being proposed by honeywell - total value of the contract could be $ N million said and work on the project would be about evenly divided - as previously reported emerged from chapter N bankruptcy-law protection in february - this los angeles company and its union federal savings bank subsidiary said more than N N of their N N N convertible subordinated debentures due N were tendered for conversion into common stock - the conversion increased total equity capital by about $ N million to a total of $ N million - union federal a federally insured savings bank has $ N billion in assets - david d. lung was appointed president and chief operating officer of this maker of building materials for manufactured homes and recreational vehicles - as president mr. lung N years old succeeds his father d. lung N who founded the company in N - lung remains chairman and chief executive officer - david lung has been with patrick since N and has served as vice president for administration and purchasing since N - general dynamics services co. a unit of general dynamics corp. won a $ N million army contract to establish maintenance facilities for tracked vehicles in pakistan - grumman corp. was given a $ N million navy contract for improvements - hughes aircraft co. a unit of general motors corp. got a $ N million air force contract for equipment - reynolds metals co. said third-quarter net income dropped nearly N N to $ N million or $ N a share from $ N million or $ N a share a year earlier - the latest earnings reflect an increase of about N million in common shares outstanding - revenue rose N N to $ N billion from $ N billion - reynolds is the third big aluminum company since friday to report disappointing earnings - the no. N domestic aluminum producer aluminum co. of america friday said its earnings fell N N to $ N million or $ N a share - and ltd. yesterday reported net income slid N N to $ N million or N cents a share from $ N million or $ N a share - analysts on average had been expecting about $ N for and $ N for - it 's a good indication that level of profitability has peaked for the industry says metals analyst with ball & inc. who had estimated reynolds would earn about $ N a share - the nation 's no. N aluminum company said earnings were hurt by lower prices for certain aluminum products which typically follow price fluctuations of primary - the base metal price has dropped N N from a year earlier to N cents a pound - much of the price decline has been blamed on a slowing economy and the third quarter is typically the industry 's period - but william o. chairman and chief executive officer said the price appears to have out - he said shipments are continuing at a healthy pace and the company has no excess inventory - aluminum shipments of N metric tons were nearly equal to the year-earlier period the company said - nevertheless the company said that in the latest quarter there were increased material and labor costs including a new employee plan - in composite trading on the new york stock exchange reynolds closed at $ N up $ N - no but certainly no home run - that 's how the game is shaping up for the months ahead according to money managers and a few brokers - yesterday 's recovery from friday 's in the dow jones industrials had many brokerage houses that stocks are a good bargain again - but quite a few money managers are n't buying it - weakening corporate earnings they say are no prescription for a bull market - the stock market ai n't going to do much of anything for a while says john of wellington management who runs the $ N billion windsor fund - he suspects that friday 's market decline may have a second leg perhaps a N N to N N drop later on - mr. says the stock market has lost some powerful driving forces namely earnings growth and the lbo buy-out fever that investors to bid up whole groups of stocks such as media and airlines - after sitting with N N of his fund in cash before friday 's sell-off mr. says he bought a narrow list of stocks yesterday - with flat corporate profits on the horizon for N money managers say price-earnings multiples that look cheap today might go on being cheap for a long time - this is not a grossly market but it 's not cheap either says george collins president of the mutual fund company t. rowe price associates in baltimore - according to institutional brokers estimate system wall street market strategists see only a N N jump in company profits in N unlike in N when profits a year out looked good they did soar N N in N - bulls say the market is an incredible bargain priced at only about N times estimated N earnings for stocks in the standard & poor 's N index - before the N crash the was more than N - the common view says cohen strategist for drexel burnham lambert is that there will be mild economic growth modest profit expansion and things are going to be - our view is that we may see a profit decline - some think investors should sell into rallies - the market is going to wind down says gerald w. a chicago money manager - things are a little less after friday 's jolt in the market - he expects stocks to decline an additional N N to N N with the dow perhaps out between N and N between now and june - after friday 's decline mr. 's firm ran statistical tests on N high-quality stocks using old-fashioned value criteria devised by benjamin graham an analyst and author in the 1930s and who is widely considered to be the father of modern securities analysis - he found N still and N fairly valued - nicholas parks a new york money manager expects the market to decline about N N - i 've been two-thirds in cash since july and i continue to think that having a defensive position is appropriate he says - companies that on debt in leveraged buy-outs during the past two years will continue to surface as business problems - about value are n't useful says new york money manager john of delta capital management - for instance he says international business machines and unisys might look cheap but investors might continue to do better with stocks like walt disney procter & gamble and coca-cola strong performers in recent years - money manager robert ross head of ross associates ltd. in vancouver british columbia says stocks would have to fall N N to N N before they are competitive with less risky investment alternatives - russell a money manager in okla. says friday 's is going to have more of a permanent impact on the of many investors than wall street would want to admit - there are still bulls out there - i still think we will have a N dow whether it 's six months or N months from now i do n't know says david managing partner of value management in new york - we 're doing a little buying in some stocks that have really been down - many brokerage house officials also are optimistic - yesterday goldman sachs merrill lynch and dean witter all increased the proportion of assets they recommend investors commit to stocks - dean witter now recommends N N goldman N N and merrill lynch N N - some investors say friday 's sell-off was a good thing because it a lot of crazy takeover speculation - it was a healthy says michael who runs salomon brothers asset management in new york - from here out these investors see a return to old-fashioned investing based on a company 's ability to show profit growth - the fundamentals are pretty strong mr. says - i do n't see this as a bear market at all - it 's a recognition that there was much too much in the lbo market - friday 's big fall was just a by the stock market says john connolly chief strategist for dean witter - it was an to an event the failure of a management and union group to get bank financing for a takeover of ual that does n't mean that much to lots of stocks - many investors have nagging worries however - newspapers are full of about companies on their debts and banks writing off real estate loans - that investors ' confidence in the economy and stocks - not even all the brokerage firms see clear sailing ahead - disappointing profits are likely to get worse in the next two quarters says mary farrell a market strategist at painewebber - she thinks the market could drop about N N in the next few months then recover and go higher - companies with steady earnings growth could do well she says while others with high debt or poor earnings could see their shares decline far more than N N - the turmoil on wall street may benefit some retailers attempting to lead leveraged buy-outs of their specialty and department-store chains investment bankers and retailers said - managers at five chains have said in recent weeks that they intend to bid for their companies - the chains include bloomingdale 's owned by campeau corp. toronto saks fifth avenue and marshall field 's owned by b.a.t industries plc london and b. altman & co. and inc. owned by hooker corp. which is now being managed by a court-appointed provisional - hooker is based in sydney australia - the combination of so many chains available for sale the recent failures of such retailing lbo 's as miller & inc. and declining investor confidence will drive down prices retailing observers said - the pricing will become more realistic which should help management said bruce rosenthal a new york investment banker with nathan s. & co - investors are n't going to be throwing money at any of the proposed lbos but doing deals on the basis of ridiculous assumptions never made sense either - earlier this year bankers and other investors were willing to provide financing because they assumed there would be major gains in both profitability and sales mr. rosenthal added - those days are over now he believes - competition from third parties who have cash and are prepared to buy has always existed and will continue added mr. rosenthal - but when prices were crazy it was even harder to do an lbo - bankers believed in the theory that says somebody else is always willing to pay more - this is no longer true today - at saks fifth avenue paul senior vice president marketing agreed that lower prices will help his management team in their proposed lbo - having to take on less debt would certainly be an advantage said mr. - it would also help us in our search for equity partners - to make an lbo work now we are going to need more than just junk bonds - none believe the proposed management lbos will be easy to complete especially at b. altman & co. which is under chapter N bankruptcy protection - not only could the wall street gyrations damp christmas sales if consumers lose confidence in the economy but potential junk-bond buyers are sure to demand even stronger and greater management equity participation - further many institutions today holding troubled retailers ' debt securities will be to consider additional retailing investments - it 's called bad money driving out good money said one retailing - institutions that usually buy retail paper have to be more concerned - however the lower prices these retail chains are now expected to bring should make it easier for managers to raise the necessary capital and pay back the resulting debt - in addition the fall selling season has generally been a good one especially for those retailers dependent on apparel sales for the majority of their revenues - what 's encouraging about this is that retail chains will be sold on the basis of their sales and earnings not liquidation values said joseph e. brooks chairman and chief executive officer of ann taylor inc. a specialty chain - retailers who had good track records of producing profits will have a better chance to buy back their companies - still most retailing observers expect that all the proposed retailing lbos will depend partly on the sale of junk bonds a market already in in part because of concerns associated with bonds issued by the federated and allied units of campeau - prices for retail chains are lower today than they were last week which will help management said harrison chairman of inc. an investment-banking firm specializing in retailing acquisitions - but the hurdle of financing still has to be resolved - potential bondholders will either look for greater equity participation on behalf of management or insist the equity component of the deals be substantially greater than in the past - sony corp. won a pretrial order blocking u.s. sales of justin products inc. 's my own line of portable audio players for children - judge john e. issued the order in manhattan federal court where sony has accused the tiny company of illegally knocking off the my first sony line - the judge held that the combination of colors used for the sony products is distinctive and subject to protection under new york state law rather than federal law - the legal fight was the subject of a wall street journal story yesterday - justin 's attorney charles e. said justin would ask an appeals court to set aside the order temporarily pending an appeal - he also repeated justin 's of sony 's charges - their likelihood of us is very slim said lewis h. sony 's attorney who said he doubts justin will go ahead with a trial - continental mortgage & equity trust said it will resume dividend payments with a payout on nov. N to shares of record oct. N - the dallas real estate investment trust last paid a dividend on dec. N N when shareholders received $ N a share - despite continuing troubles with problem assets and nonperforming loans the trust said it expects to be able to maintain or increase the rate of distributions because of operations of joint-venture properties - a federal appeals court struck down a natural-gas regulation that had prevented pipeline companies from passing to customers part of $ N billion in costs from controversial contracts - the court in a N ruling threw out a deadline set by the federal energy regulatory commission for settling old contract disputes over gas that the pipeline companies reserved but did n't use - ferc 's regulation had given pipelines until march N N to pass on to customers as much as N N of the costs of buying out their broken contracts which were made with producers when gas prices were high and supplies short - a majority of old contracts were by the deadline and settled at steep discounts - but pipeline companies estimate they still face $ N billion in liabilities from disputes including $ N billion they fear they wo n't be able to pass on to customers - according to industry lawyers the ruling gives pipeline companies an important second chance to resolve remaining disputes and take advantage of the cost-sharing mechanism - the court left open whether ferc could a new deadline later - the court agreeing with pipeline companies found the march N deadline was and and highly to the bargaining power of pipelines that were forced to negotiate settlement of the old contracts to meet the deadline - a report last month by the interstate natural gas association of america found that pipelines ' settlement costs had jumped in the three months before the deadline to N cents on the dollar from N cents on the dollar in N - the court ordered ferc to justify within N days not only its cost-sharing deadline but other major elements of its proposed regulation for introducing more competition into natural-gas transportation - the court also questioned a mechanism that could be used to resolve liabilities - the complex regulation known in the industry as order N has been contested by all sides including natural-gas producers pipelines local distribution companies and consumers - the court 's decision would allow ferc to change some of its provisions but it will be reviewed again quickly by the court - corp. said it voluntarily prepaid $ N million on its original $ N million term loan bringing the total debt reduction for the year to $ N million - after the payment the cleveland company owes $ N million on the loan - the cement producer said the payment was made from excess cash flow - national income realty trust said it will resume dividend payments with a dividend to be paid nov. N to shares of record oct. N - the mortgage and equity real estate investment trust last paid a dividend on aug. N N when holders received N cents a share - despite continuing troubles with problem properties and nonperforming loans the dallas trust said it has reserves abandoned properties with little potential and experienced improved operating results from joint ventures - mlx corp. said it reached a preliminary agreement with senior lenders to its and group to restructure the $ N million of credit facilities the lenders provide to the group - mlx which also makes aircraft and truck parts said the debt was accumulated during its acquisition of nine businesses that make up the group the biggest portion of which was related to the N purchase of a co. unit - among other things the restructured facilities will substantially reduce the group 's required amortization of the term loan portion of the credit facilities through september N mlx said - certain details of the restructured facilities remain to be negotiated - the agreement is subject to completion of a definitive amendment and appropriate approvals - william p. mlx chairman and chief executive said the pact will provide mlx with the additional time and flexibility necessary to complete the restructuring of the company 's capital structure - mlx has filed a registration statement with the securities and exchange commission covering a proposed offering of $ N million in long-term senior subordinated notes and warrants - dow jones & co. said it acquired a N N interest in corp. a subsidiary of oklahoma publishing co. oklahoma city that provides electronic research services - terms were n't disclosed - customers of either or dow jones are able to access the information on both services - dow jones is the publisher of the wall street journal - flowers industries inc. said it will report a charge of eight cents to N cents a share for its fiscal first quarter ended sept. N from the sale of two in high point n.c. and - the company said it sold the to mills family for an undisclosed amount - it said the sales were part of a N federal trade commission consent order - a year earlier flowers had fiscal first-quarter net income of $ N million or N cents a share on revenue of $ N million - production by the nation 's mills decreased N N last week to N tons from N tons the previous week the american iron and steel institute said - last week 's output rose N N from the N tons produced a year earlier - the industry used N N of its capability last week compared with N N the previous week and N N a year ago - the capability utilization rate is a designed to indicate at what percent of its production capability the industry is operating in a given week - b. was named executive director of the commission effective early november - mr. N years old succeeds N who resigned to join hong kong 's securities and futures commission - mr. was vice president and director corporate finance of thomson inc. a toronto investment dealer - dun & bradstreet corp. 's market data unit said it acquired school and college construction reports service from intelligence for education inc - terms were n't disclosed - the service supplies weekly reports on school and college construction plans - market data is a of educational information and provides related services - closely held intelligence in education of n.y. is an educational publisher and consultant - a battle is in venice over plans to have the italian city be the site for a universal in N - the plans include a subway system a congress center floating trees and as many as N additional tourists a day - enthusiasts argue that holding the fair would attract businesses create jobs and help abandoned sections of town - but opponents fear - this city already has too many tourists and it ca n't hold them all says the president of the venice association - about N italian businesses including fiat s.p a. and c. olivetti & co. have formed a consortium to lobby for holding the in venice - three gambling casinos have opened in poland - the three two in warsaw and one in accept only foreign currency and are joint ventures between polish firms and western companies - not all poles are pleased - what do we want casinos for when we have n't got anything in the shops one asked - but who runs the casino at warsaw 's hotel said the ventures would help poland service its $ N billion foreign debt by pouring dollars into the state firms in the joint ventures the lot airline and tourist organization - plans to increase natural-gas sales to europe and the u.s. - according to the middle east economic survey the north african nation is holding talks with italy for adding a fourth pipe to a section of the pipeline expanding capacity by up to six billion cubic a year from N billion - also wants to build a pipeline through and across the of to supply spain france and west germany with up to N billion cubic a year by the late 1990s - south africa 's national union of agreed to suspend the strike by diamond workers and resume negotiations with de beers consolidated mines ltd. over their wage dispute de beers said - it also said the union had agreed to meet the company for further talks tomorrow - the strike at five de beers mines began last thursday with N out of a total N members employed on de beers mines participating according to the union while de beers said there were N participants - the union has demanded a N N increase in the minimum wage while de beers 's final offer was an increase of N N - a environmental conference opened in - the gathering is expected to focus on curbing the of and limiting damage from industrial and improving the handling of harmful chemicals - west german environment minister said bonn is convinced of the need for cooperation especially with our neighbors in the east because we are directly affected by their ecological progress or lack of it - the u.s. and canada joined every european country except at the meeting - the swedish publishers of a new newspaper rushed an extra edition across the on oct. N after the first run sold out in one day - editor said plans had called for N copies of the monthly are business paper to be sold at and an additional N promotion issues to be sent by direct mail - he said N more copies were sent to because of strong sales - the swedish publishing company owns N N of are and the management company minor owns N N - mexico 's top debt negotiator said the country 's creditor banks are responding to mexico 's package - mr. 's optimism contrasts with some bankers ' views that the deal may require a lot of arm by the u.s. treasury in order to succeed - mr. mexico 's of the ministry of finance met yesterday with european bankers in london at the point on a so-called road show to market the package around the world - an increasing number of banks appear to be considering the option under the deal they can swap their mexican loans for 30-year bonds with a face value discounted by N N mr. said - the other two options consist of loans for bonds with N N interest rates or providing fresh loans - the accord which covers $ N billion of mexico 's medium and long-term debt is expected to go into effect in early - china 's top film actress paid $ N in back taxes and fines in province the people 's daily reported - the amount is equal to about N years earnings for the average peasant who makes $ N a year - china will spend $ N million for maintenance on 's palace former home of the lama the china news service said - the lama who was just awarded the nobel peace prize lives in in india - george w. koch N years old president and chief executive officer of grocery manufacturers of america inc. was elected a director of this maker of and specialty foods succeeding n. white jr. N who resigned - american business computer corp. said it privately placed N common shares at $ N a share - the placement was made through gray securities new york to institutional investors - proceeds will be used to recently technology and support the company 's international expansion - the company develops and markets products for the food service industry - the r.h. macy & co department-store chain is n't for sale - in yesterday 's edition it was incorrectly included with a list of new york chains up for sale - korean car exports have slid about N N so far this year but auto makers here are n't - they are enjoying domestic sales that are more than making up for lost overseas sales - south korean consumers are expected to buy almost N passenger cars this year up N N from N - in fact some auto executives suggest that demand for their cars in the u.s. and canada is a blessing otherwise they would n't be able to keep up with demand in the more profitable local market - we are very lucky to easily change an export loss to domestic plus says hong managing director of domestic marketing for hyundai motor co - as it is waiting lists of a month are n't unusual for popular models - demand is so strong that all of the domestic makers hyundai kia motors corp. daewoo motor co. and even ssangyong motor co. plan to build more factories - industry analysts predict that by N south korea will be building three million cars a year about half of that for export - it 's an optimistic move in a industry already facing world-wide overcapacity - but south korean auto makers are confident that the export market will bounce back and that demand in korea will stay strong - currently only one in N south koreans owns a car up from one in N a decade ago - in the year N it will be one car per family - at that point domestic sales will slow down says kim director of marketing for daewoo motor - the reason for the tremendous demand is simple south koreans suddenly have a lot more money - we never thought we 'd own a car says ok who just bought a daewoo on a five-year loan - she and her husband started a small printing business and need the car for work as well as for weekend - pay raises of N N over the past three years have given many south koreans the money to enjoy the things they were supplying the rest of the world - the success of ssangyong motor shows the strength of the auto market and its growing diversity - a part of the conglomerate ssangyong group it took over the dying motor co. in N - ssangyong began making variations of the vehicle - had had a technology agreement with jeep maker american motors corp. now a part of chrysler corp - the most popular style is the stretched family which resembles a ford or chevy - the vehicles start at $ N a family can cost over $ N - ssangyong which has only about N N of the domestic market will sell about N of its models this year twice as many as last year - it sees sales rising N N to N units next year - the company plans to expand plant capacity N N by N - by then it also hopes to begin producing a passenger car based on the N and selling for about $ N - hyundai and daewoo seem about the ssangyong threat but kia the auto maker is selling vehicles through its asia unit - it plans to sell N units in N - kia the only korean car maker that has seen its overseas sales grow in N aims at korea 's common man - its advantage has been the little pride sold as the ford in the u.s. - at N million won or $ N the is the car in south korea - along with two larger models the company claims N N of the domestic market - ford motor co. and japan 's mazda motor corp. have equity interests in kia - kia is the most aggressive of the korean big three in offering financing - loans for as long as five years make the cars very accessible with monthly payments as low as N won or $ N - daewoo motor a N joint venture with general motors corp. and the daewoo group conglomerate is the only auto maker that appears to be hurting - shipments of its to gm 's division are off about N N from a year ago a N N decline for hyundai and an N N increase for kia - moreover daewoo 's domestic sales have grown half as fast as sales of its rivals - the big problem for daewoo which holds about N N of the market is the long series of labor disruptions it suffered this year - but daewoo is expanding too - in fact a sister company daewoo shipbuilding and heavy machinery plans to build N by the mid-1990s - hyundai the korean market leader with a N N share also plans to jump into at the same time - it has a similar project for N cars a year - kia is reportedly also considering such a plan - even giant group is rumored in the korean press to be considering getting into the business a company spokesman had no comment - robert p. N years old was named president and chief administrative officer of this regional commercial bank - both posts had been vacant - robert N was named to the new positions of vice chairman and chief credit officer - many mutual fund investors picked up the phone yesterday but decided not to cash in their chips after all - as the stock market bounced back withdrawals of money from stock funds amounted to a mere compared with black monday when investors dumped $ N billion or about N N of assets - fidelity investments the nation 's largest fund company said phone volume was more than double its typical level but still half that of oct. N N - net outflows from fidelity 's stock funds stood at less than $ N million or below N N of the $ N billion cash position of the firm 's stock portfolios - much of the money was switched into the firm 's money market funds - outflows since the close of trading friday remain below one-third their level of two years ago fidelity said - other mutual fund companies reported even lighter withdrawal requests - and some investors at fidelity and elsewhere even began buying stock funds during the day - two years ago there was a lot of redemption activity and trouble with people getting through on the phone said head of the investment management division of the securities and exchange commission - this time we do n't have that at all - of course the relative calm could be jolted if the market again - and any strong surge in redemptions could force some funds to dump stocks to raise cash as some did during black monday - but funds generally are better prepared this time around - as a group their cash position of N N of assets in august the latest figure available is N N higher than two years earlier - many fund managers have boosted their cash levels in recent weeks - the biggest flurry of investor activity came early in the day - vanguard group inc. saw heavy exchanges from stock funds into money market funds after the telephone lines opened at N a.m - in the first hour the real nervous folks came along a spokesman said - but the pace of call volume in the first half-hour slowed considerably - at stevens & clark inc. phone calls came in at N N more than the normal pace through early afternoon - most of that increase came in the first hour after the phone lines opened at N a.m - as stocks rose in fact some investors changed course and reversed their sell orders - many funds allow investors to orders before the close of trading - at and at the smaller ivy funds group in mass. for instance some shareholders called early in the morning to switch money from stock funds to money market funds but later called back to reverse the switches - because mutual fund trades do n't take effect until the market close in this case at N p.m. these shareholders effectively stayed put - at fidelity 's office in downtown boston gerald sherman walked in shortly after N a.m. and placed an order to switch his retirement accounts out of three stock funds and into a money market fund - but by N p.m. with the market ahead for the day mr. sherman was preparing to undo his switch - it 's a nice feeling to know that things stabilized said mr. sherman the of a discount department store - but some investors continued to switch out of high-risk high-yield junk funds despite yesterday 's rebound from that market 's recent price declines - shareholders have been steadily out of several big junk funds the past several weeks as the $ N billion market was jolted by a cash crunch at campeau corp. and steadily declining prices - much of the money has been switched into money market funds fund executives say - instead of selling bonds to meet redemptions however some funds have borrowed from banks to meet withdrawal requests - this knocking down prices further - the $ N billion t. rowe price high yield fund was among the funds that borrowed during the campeau crisis says george j. collins president of t. rowe price associates inc - that way mr. collins says we did n't have to sell securities in a sloppy market - when the market stabilized he added the firm sold the bonds and quickly paid the loans back - tom contributed to this article - financial inc. said it agreed to acquire central of illinois inc. in a stock swap - shareholders of central a bank holding company based in sterling ill. will receive stock equal to N times central 's N earnings said - for the first nine months of N central earned $ N million - also a bank holding company has assets of $ N billion - central 's assets are $ N million - during its centennial year the wall street journal will report events of the past century that stand as milestones of american business history - soft contact lenses won federal blessing on march N N and quickly became eye for their makers - the food and drug administration that day said bausch & could start selling them in the u.s. - the product was more comfortable and less prone to falling out than hard contact lenses which had been around since N - bausch & sold the under a from national patent development which had gained the rights from the czechoslovakia academy of sciences - a invented them in N - the plastic lens itself over the eye while permitting to pass through - but the new lens became the eye of a storm - in september N california officials seized lenses made by companies after some showed of bacteria - in october doctors were the product 's safety some claiming it caused - and there were senate hearings on the questions in july N - the product the bad publicity and kept - the early soft lenses which cost $ N a set were expected to last for a year - in N extended wear versions designed to be for N days at a time offered - months ago a disposable seven-day model bowed a year 's supply costs about $ N - last month the fda and contact lens institute cautioned users that serious eye could result from wearing lenses more than seven days at a stretch - today N million of the N million americans using contact lenses are using the soft type - including the eye care products contacts account for $ N billion in annual retail sales - although bausch remains the leader among the six johnson & johnson with its new is coming on fast - the roller-coaster stock market is making life tougher for small companies trying to raise money - in the wake of friday 's plunge and yesterday 's rebound some companies are already deals and others wish they could - as in other jittery times many small businesses expect a particularly rough time raising funds as investors risky deals seeking safety in bigger companies - even if stock prices fully recover from friday 's sharp decline the unsettled conditions will many investors - the implication of an unsettled situation is that the thing could drop dramatically says henry jr. chairman of corp. a four-year-old biotechnology company that is planning a private placement of stock - the more that indicate risk the more the investor is going to drive a hard bargain - earlier this month inc. a mass. said it would accelerate expansion plans nationwide and offer more of its stock to the public - at the time its shares were selling above their initial offering price of $ N and bankers believed would sell new stock without a - but with the company 's shares standing at $ N yesterday a new offering seems unlikely company officials say - business however continues to be robust and the stock market has n't affected the concern 's expansion plans says a senior executive - other companies figure they ca n't avoid the market - we have capital requirements says mr. so we have to go ahead with a planned $ N billion private placement - unless the market goes right back up he says it may take us six to nine months to find the money instead of three - and the columbia md. company may have to settle for a lower price he adds - life is particularly for companies that had planned to go public this week - is becoming an investment-banking job requirement - robertson & co. a san francisco investment banking concern has a client that looked forward to making its initial public offering yesterday - officers of the company a health-care concern were very discouraged on friday and felt they should n't go public we felt they should says sanford robertson partner in the banking concern - as the market dropped friday robertson slashed the value of the offering by N N - yesterday when similar securities rebounded it the valuation up again - as of late yesterday the ipo was still on - for many the situation is especially discouraging because the market for was showing signs of strengthening after several years of weakness - we were just beginning to look at the increase in seeing the light at the end of the tunnel says frank jr. partner in funds a beverly hills calif. venture capital concern - but the tunnel 's just gotten longer - companies planning to go public are definitely taking a second look says allen senior analyst at the institute for research fort fla. which publishes the new issues newsletter on - he that the recent market slide translated into a N N to N N reduction in ipo proceeds to companies - many companies are - corp. had been planning to sell N N of its stock this week in an ipo that would raise up to $ N million - but now peter president says we 're making decisions on a day-to-day basis - and profitable the colo. concern could borrow funds if it decides against an ipo now he says - inc. an atlanta concern says it is still planning to go ahead with its ipo this week or next unless conditions change - it 's a situation right now says terry president - delayed financings also would affect the operations of many companies - sierra tucson cos. a tucson ariz. operator of centers has a planned doubling of capacity riding on an ipo scheduled for next week - william president says he still thinks the ipo will succeed - if it does n't he says the company would have to change its expansion timetable - but the market turmoil could be partially beneficial for some small businesses - in a sagging market the federal reserve system might flood the market with funds and that should bring interest rates down says leonard t. vice president of the bank of new england boston - james g. president of savings bank mass. says the market turmoil is an for small business - for small companies he says interest rates are far more important than what happens on stock exchanges - mr. thinks rates are heading down helping small companies - peter biotechnology analyst for securities international chicago thinks market uncertainty may encourage small companies to form more strategic alliances with big corporations - partly because the N market crash made it harder for them to find financing many high-technology concerns have made such alliances recently - some even see a silver in the dark clouds - alan wells president of wells & co. a new york merger specialist thinks investors may lose their enthusiasm for leveraged buy-out and giant takeover deals - instead they could turn to investing in smaller deals involving smaller companies he says - and william e. jr. a university of new hampshire management professor and director of venture capital network inc. says the market 's gyrations will the investors ' lack of control in big stock investments - this will add to the appeal of small business he says where investors often have a degree of influence - bay financial corp. hurt by high debts and deteriorating real estate investments reported a wider loss for the fourth quarter and said it might be forced to seek a bankruptcy-court reorganization if it ca n't its borrowings - bay said a substantial part of its debt outstanding is in default as a result of inability to sell certain properties quickly and lower-than-expected prices for sales made - the company said its real estate portfolio is highly leveraged while about two-thirds of its investments are n't - thus it is coming up short on a big bet that quick sales at higher prices would enable it to keep up with mortgage and other debt payments - according to its latest annual report about a quarter of the company 's holdings are in massachusetts in the midst of a real-estate slump - the company said it had a net loss in its fourth quarter ended june N of $ N million or $ N a share on revenue of $ N million - a year earlier the company had a loss of $ N million or $ N a share on revenue of $ N million - for the year it had a net loss of $ N million or $ N a share on revenue of $ N million - in the previous year it had a loss of $ N million or $ N a share on revenue of $ N million - although it is having serious problems bay said the value of its holdings minus debt was equal to $ N a share at june N based on a recent - book value per share which is based on investments at cost was a negative $ N a share - a year earlier value per share was $ N and book value was $ N a share - annualized interest rates on certain investments as reported by the federal reserve board on a basis N and wednesday october N N - adjusted for constant maturity - inc. reported a N N decline in third-quarter net income but the company said that excluding unusual gains in both quarters operating profit rose N N - the electronics automotive and aerospace concern said third-quarter net was $ N million or N cents a share down from $ N million or $ N a share a year earlier - share earnings are reported on a fully diluted basis by company tradition - results for the N quarter included a gain of $ N a share from sale of the pump and cable units partly offset by a charge of N cents a share for recall of truck steering systems - the latest quarter included a gain of N cents a share as a partial reversal of the recall charge because the reserve established last year exceeded the actual recall costs - sales for the quarter rose N N to $ N billion from $ N billion with all three major product groups reporting gains - the company said aerospace and defense sales were up N N for the quarter to $ N million and operating profit climbed N N to $ N million mainly because of improved program performance in spacecraft and contracts - automotive sales jumped N N to $ N million mainly because of higher sales of air bags and other passenger restraint systems said - the group had an operating profit of $ N million against a loss of $ N million a year earlier - however excluding the year-earlier charge for recall of steering gear operating profit in the latest quarter declined N N reflecting higher start-up and product development expenses in systems - materials and production costs also rose said - the information systems segment had a N N jump sales to $ N million - an acquisition accounted for half the sales rise said - operating profit rose to $ N million from $ N million - for the nine months 's net was $ N million or $ N a share down N N from $ N million or $ N a share a year earlier - sales rose N N to $ N billion from $ N billion - a by an not though english butler in his proceeds as if the realistic english novel of like herself still ruled the waves - in fact 's the remains of the day N pages $ N is both an to traditional english forms and a dramatic of them - it implies that the british empire was rooted in its subjects ' minds and and argues that its flaws were in the defensive willful and especially the of its domestic - as the stevens the butler of hall over such terms as dignity service and loyalty we see how the soul - stevens 's of the public and private like his master 's all it was designed to preserve - such the - the cuts to the quick - it 's N the year the suez crisis marked the final end of empire - as he stands on a hill at the beginning of a motor from to where a former perhaps the victim of an unhappy 20-year marriage perhaps he hopes with more than he will ever acknowledge not to return to domestic service stevens surveys the view and thereby provides a a and the author 's for the of the novel we 're reading - we call this land of great britain and there may be those who believe this a somewhat practice - yet i would venture that the landscape of our country alone would justify the use of this - it is the very lack of obvious drama or that sets the beauty of our land apart - what is is the of that beauty its sense of restraint - it is as though the land knows of its own beauty of its own and feels no need to it - in comparison the sorts of sights offered in such places as africa and america though undoubtedly very exciting would i am sure strike the objective as on account of their - an landscape - an mountain - but let stevens continue in his comic manner his efforts at always fail most this whole question is very to the question that has caused much debate in our profession over the years what is a great butler - his answer is one of a dignity in keeping with his position - such dignity has to do with a butler 's ability not to abandon the professional being he - he will not be shaken out by external events however surprising or - are unable to be because they are as a breed of the emotional restraint which only the english race are capable of - despite his racial advantage to be a great butler is a calling one 's is not unlike general 's headquarters during a battle - if for example in the midst of a great social occasion such as an international conference on the treaty in N one 's father himself a great butler once should happen to die of a one must continue to serve the port please do n't think me improper in not to see my father in his condition just at this moment - you see i know my father would have me to carry on just now - it is this kind of dignity and restraint that allows stevens to declare for all its sad associations whenever i recall that evening today i find i do so with a large sense of - we note the imperial public word used to deny private rage and - that stevens himself is not or but funny and sad and is entirely the author 's - mr. 's ability to create a voice that permits him to explore such domestic cultural and political themes was clear in his previous novel an artist of the floating world set in japan after the war - now shifting his scene from the country he left at five to the england he has lived in for nearly N years he has a novel in the mode of henry james and - with great he considers not only and utterly sexual love but british the 's with democracy and support of and the moral of loyalty it is in practice simply not possible to adopt such a critical attitude an employer and at the same time provide good service - this employer all that i find noble and - i will devote myself to serving him - this is loyalty - in the end after meeting with the former stevens sits by the at thinking of her and of his employer and declares i trusted - i trusted in his 's wisdom - i ca n't even say i made my own mistakes - really one has to ask what dignity is there in that - the loyal has come full circle - what is - what is dignity - we understand such wisdom must be the of only spreads her wings at - but as the remains of the day so demonstrates with quiet such wisdom can be in art - mr. teaches english and literature at columbia university - corp. said its subsidiary completed the previously announced sale of its air separation plant and related assets in wis. to aga gas inc. cleveland - the price was n't disclosed - the transaction is part of 's continuing program to shed 's industrial gas interests and expand the subsidiary 's propane business - since june has more than $ N million from industrial gas and reinvested more than $ N million to acquire three propane distributors - is a gas and electric utility and distributes propane nationally through its subsidiary - who represents the soviet airline aeroflot here has some that are wild even by the current standards of perestroika - in his office the runway of shannon airport mr. throws out what he calls just ideas - first he suggests group ltd. the international aircraft leasing company based in ireland could lease some of its boeing to the soviet airline - then aer the irish flag carrier could teach aeroflot pilots to fly the and the fleet could be based here at shannon airport - that 's not all he says - aer the irish airport authority could build a cargo terminal in the soviet union - aeroflot could lease some of its cargo planes to aer through for a joint-venture cargo airline - and then there is his notion of an charter airline to ferry to los angeles via shannon - have the freedoms of glasnost gone to mr. 's head - hardly - the aviation connection is alive and well here at shannon airport - is indeed talking about leasing western planes to aeroflot and even about buying - aer is in discussions with the soviet carrier about a cargo venture and other possibilities - aer already has so many ventures with aeroflot that its chief executive is studying russian - unlikely as it may seem tiny politically neutral ireland has the mighty soviet airline bureaucracy - and as aeroflot struggles to boost its service standards upgrade its fleet and pursue commercial opportunities the irish aviation industry seems poised to benefit - irish and soviet people are similar says mr. - they look the same - they 're very friendly - moreover he says irish companies are small but - we have to study their experience very well he says - we must find any way to get business - the two groups have been working together since the late 1970s long before soviet joint ventures were the rage in the west - aeroflot carried about N million passengers last year and shannon airport the airline 's largest transit airport outside the soviet union saw N aeroflot flights and N passengers pass through - an apartment complex down the road is the and staging area for more than N aeroflot pilots and flight attendants - the airport 's biggest supplier of aircraft fuel is the soviet union - from the port of each year unload N million gallons of fuel into a special tank farm at the airport - what aeroflot does n't pour into its own is to the airport authority which it to N western carriers including air france trans world airlines and pakistan international airlines - aeroflot thus pays its landing fees and bills with fuel preserving its hard currency - that is n't all - last year the irish airport authority in a joint venture with aeroflot opened four duty-free shops at moscow 's airport - aer now manages duty-free sales on all aeroflot international flights out of moscow - duty-free shops in 's airport opened in july and shops in hotels and on the are coming soon - aer is talking about similar joint ventures in and in a black sea resort and even has a project cooking with the city of - aeroflot 's international fleet of N planes is being and at shannon airport - thanks to a new agreement and the ability of irish travel agents to issue aeroflot tickets tourists here are taking advantage of aeroflot 's reasonable prices to board flights in shannon for holidays in and mexico city - the fare to is N irish punts $ N - jamaica costs N punts - a formal blessing of sorts was on this friendship in april when mikhail and gorbachev stopped here for talks with irish prime minister charles - new trade accords were signed - it all started with geography - when it opened in N shannon was the first in europe for airplanes flying from north america - advances in aircraft fuel efficiency over the years made a shannon stop unnecessary for most western air fleets but aeroflot still flies inefficient that ca n't make it from moscow to managua on one - as a result ireland did n't the soviets after they shot down a korean air lines jetliner over the sea of japan in N though it suspended direct flights for two months - in fact aer started russians from shannon to new york when washington stripped aeroflot of its u.s. landing rights - today aer is making a of money from its soviet friendship - and with those contacts in place it could be relatively simple to add aer and to the team - then perhaps mr. 's ideas would n't sound like so much - britain 's industrial production rose N N in august from july and was up N N from august N according to provisional data from the central statistical office - output in the energy sector which can vary greatly with swings in the oil market rose N N in august from may but was down N N from a year earlier - the latest figures compare with july 's N N rise and N N year-to-year fall - when corp. begins shipping steel from the world 's first plant this month it will begin testing the competitive of its giant competitors - the new technology which creates a very thin piece of steel reduces the costs of making flat-rolled sheets - an kenneth iverson 's chairman says the company 's plant eventually will make a ton of steel in N man hours compared with four to six man hours at a conventional mill - we 've had the russians and chinese and people from india visiting us mr. iverson - everyone in the world is watching us very closely - especially his neighbors the major u.s. steelmakers - already usx corp. and armco inc. are studying 's technology to see if they can adopt it - says the chief executive officer of a major midwest steel company it 's damn worrisome - the steel industry is about to be turned by a 1990s technology revolution - new efficient and sophisticated processes make it easier for smaller less companies to make steel at a fraction of what big steel paid decades ago - it also enables minimills finally to get a in the flat-rolled steel market the major steelmakers ' largest most and until now market - but such technology is only the beginning - eager engineers and direct casting which by the end of the 1990s will enable production without coke and blast - those massive structures while cost and environmental headaches effectively locked out all but giants from - there 's a revolution ahead of us that will ultimately change the way we market and distribute steel says william dennis vice president manufacturing and technology for the american iron and steel institute - it is n't that major steelmakers have ignored high technology - in fact they 've spent billions of dollars to boost the percentage of cast steel to N N in N from N N five years before - moreover their balance sheets are rich with diversity their old plants and work forces lean - but that wo n't - it 's no longer enough to beat the guy down the street - you have to beat everyone around the world says mr. dennis - he wants to see steelmakers more involved in computers and intelligence - the problem they 're with huge plants that require costly maintenance - and try new dollars free in a market that is softening hurt by a strong dollar and concerned about overcapacity the industry 's - the technology revolution is going to be very threatening to established producers says peter marcus an analyst with painewebber inc - they 've got too much invested in the old stuff and they ca n't get their workers to be flexible - no one expects minimills to major integrated steelmakers who remain the of steel used for autos and refrigerators - 's plant in ind. ultimately will produce only one million tons annually a drop in the flat-rolled steel and it will be years before such plants can compete in the market - still flat-rolled is the steel industry 's bread and butter representing about half of the N million tons of steel expected to be shipped this year - moreover the process is n't without its headaches - because all operations are connected one equipment failure forces a complete plant shutdown - on some days the plant does n't produce anything - at this point the capacity wo n't make a great in the integrated market but it does challenge them to develop new markets says james mccall vice president materials at a technology and giant based in columbus ohio - indeed with demand for steel not growing fast enough to absorb capacity steelmakers will have to change the way they do business - in the past says armco 's chief economist john steelmakers made a product and set it out on the - we said we 've got a product if you want it you can buy it he says adding now we 're figuring out what people need and are going back to make it - armco 's sales representatives visit the general motors corp. 's assembly plant in kansas city mo. two or three days a week - when they determined that gm needed parts more quickly armco convinced a steel service center to build a processing plant nearby so shipments could be delivered within N minutes - such relationships with major clients car and makers is a means of survival especially when those key clients are relying on a smaller pool of producers and with plastic and aluminum makers - for example when detroit began talking about cars the american iron and steel institute began a major lobbying effort to show auto makers how they could use steel more efficiently by simply how a car door is assembled - but steelmakers must also find new markets - after letting take the recycling lead a group of the nation 's largest steelmakers started a recycling institute to promote steel cans to an environmentally nation - 's mr. mccall thinks steelmakers should concentrate more on construction - weirton steel corp. weirton w. va. for example is touting to homeowners fashionable steel doors with glass as a secure and alternative to wooden or aluminum ones - other steelmakers steel covering - still others are looking at overseas markets - usx is drilling pipe to soviet union - this year the nation 's largest steelmaker its overseas sales operation - producers also are trying to by concentrating on output such as coated and products which remain beyond the reach of minimills - almost all programs announced by major steelmakers within the past year involve building lines used to produce steel for such products as household appliances and car doors - but unfortunately that segment is much smaller than the bread-and-butter flat-rolled steel - it 's like everyone climbing out of the ii and getting into a says john jacobson an analyst with consultants - after a while someone has to go over the side - although he does n't expect any he does see more plants being sold or closed - robert crandall with the institute agrees - unless there is an enormous rate of economic growth or a further drop in the dollar it 's unlikely that consumption of u.s. produced steel will grow sufficiently to offset the growth of minimills - not to mention the of imports - japanese and european steelmakers which have led the recent technology developments are awaiting the lifting of trade restraints in N - moreover the u.s. can expect more competition from low-cost producing pacific and latin american countries - a taiwanese steelmaker recently announced plans to build a plant - people think of the steel business as an old and mundane business says mr. iverson - they 're dead wrong - \* usx ltv bethlehem inland armco national steel - \*\* projected - polaroid corp. 's damages case against eastman kodak co. one of the highest stakes corporate trials ever is getting attention on wall street - after N days of testimony in federal court in boston the trial is being all but ignored by analysts and patent attorneys - most have read the pre-trial documents however and estimate kodak will be ordered to pay $ N billion to $ N billion for on seven polaroid patents - that may be the largest patent award ever but it is well below the $ N billion polaroid seeks - the highest patent damage award to date was in N when smith international inc. was ordered to pay $ N million to baker hughes inc. for on a patent on an oil drilling bit seal - the two companies later agreed to settle for $ N million - few analysts think it is worth their time to through the polaroid trial testimony - it 's like for gold outside of grand central station - you might find something but the chances are low said michael an analyst at wertheim schroder & co - and eugene glazer an analyst at dean witter reynolds inc. said if you hired an attorney to be there all the time and give you a prediction of the eventual award i would be willing to bet that he would be off by a lot - a trial in the early 1980s determined that kodak based in rochester n.y. infringed on patents of polaroid of cambridge mass - the main issues remaining are how to calculate damages and whether the infringement was willful and - if so the damages could be tripled - two analysts who have read the david nelson of shearson lehman hutton inc. and d. a litigation analyst at simpson & co. think judge a. david will decide in kodak 's favor on the willful and issue - mr. said testimony by kodak 's patent counsel francis t. carr of & showed that he worked with kodak from the outset of the project in an effort to avoid infringement - carr told kodak on many occasions to avoid various features because of polaroid 's patent positions and kodak followed his advice in every instance mr. said - but irving a patent expert at george mason university school of law who is familiar with the case said the fact that seven patents were infringed suggests that infringement was willful - it 's difficult to be that consistently wrong - observers also wonder whether judge will use the method of determining damages which polaroid favors because it would result in a larger award or the reasonable royalty method - polaroid claims it could have manufactured and sold all the instant cameras and film sold by kodak if kodak had n't entered the market - moreover polaroid contends it could have sold them at a higher price and thus made higher profits because it would n't have been forced to match kodak 's lower prices - each side has called a harvard business school professor to testify on that issue - kodak hired robert and polaroid brought in robert j. - there 's nothing that says that people at harvard business school have to agree with each other said mr. - testimony is expected to continue until early december - a decision is n't expected until some time next year - international business machines corp. said earnings tumbled N N in the third quarter even a bit further than expected the outlook doubtful for the next few quarters - the main reason was a delay in shipment of new high-end disk drives a business that accounts for some N N of ibm 's $ N billion of annual revenue - ibm which the poor results three weeks ago also cited an increase in its leasing business which tends to lock in business long-term but cut revenue in the near term - in addition ibm noted that the stronger dollar has cut the value of overseas revenue and earnings when they are translated into dollars - earnings fell to $ N million or $ N a share somewhat below securities analysts ' revised expectations of around $ N a share - that compared with the year-earlier $ N billion or $ N a share which was inflated by a gain from the sale of some mci communications corp. stock and by an unspecified amount from a payment by fujitsu ltd. relating to a software dispute - revenue climbed N N to $ N billion from $ N billion - ibm armonk n.y. remained upbeat - the computer giant whose u.s. results have been dismal for years noted that revenue rose again in the u.s. in the third quarter following an increase in the second period - the company said in a statement that demand for ibm products and services continues to be good world-wide - we do not see anything in the fundamentals of our business that would cause us to change our strategy of investing for profitable growth - securities analysts however remained - i think N will be another year said steve of first boston - jay stevens of dean witter actually cut his per-share earnings estimate to $ N from $ N for N and to $ N from $ N in N because he decided sales would be even weaker than he had expected - both estimates would mark declines from the N net of $ N billion or $ N a share which itself was well below the record ibm set in N - mr. stevens said he kept a recommendation on the stock only because all the damage has been done - he said the stock has n't traded below N N times book value over the past N years which at the moment to a stock price of $ N - the stock closed yesterday at $ N a share up just $ N in composite trading on the new york stock exchange as the market surged - analysts worry that the disk-drive and leasing problems will last at least through the first quarter - a key part of the question is how soon does this disk-drive come and how soon does production up said steve cohen at financial group - and the input i 've had from customers is that it still could be a while - on leasing bob at research said he thinks ibm has hurt itself - he said ibm has priced its leases aggressively thinking that would help win business - but he said ibm would have won the business anyway as a sale to a third party that would have then leased the equipment to the customer - he said ibm has not only hurt its short-term revenue outlook but has also been losing money on its leases - bob executive vice president of marketing at inc. a huge leasing firm said to put it mildly ibm credit has been doing some of the worst economic deals of any leasing company we have ever seen - ibm is expected to get a boost soon when it some new versions of its mainframes - but the basic technology in the line is almost five years old which means it is long in the and competitors are rolling out strong products of their own - ibm is gaining momentum in the personal-computer market and is expected to introduce some impressive workstations early next year - but it 's hard to squeeze much profit out of the personal-computer business these days and the workstation market while important is too small to rely on for much growth - the disk drives will sell well when they finally become available - but the ibm 's highly successful line is losing its momentum and some analysts said sales could even decline in the fourth quarter - in addition ibm 's growth in software in the third quarter was just N N well below historical levels even when adjusted to reflect last year 's payment from fujitsu and the stronger dollar - and expenses up N N in the quarter have stayed high - in the nine months ibm earned $ N billion or $ N a share down N N from the year-earlier $ N billion or $ N a share - revenue increased N N to $ N billion from $ N billion - pepsico inc. 's chairman said he is more than comfortable with analysts ' estimates that third-quarter earnings rose to at least N cents to $ N a share from N cents the year earlier - d. wayne calloway also chief executive officer of the company indicated that he expects analysts to raise their forecasts for N after the company releases its earnings today - so far analysts have said they are looking for $ N to $ N a share - after today 's announcement that range could increase to $ N to $ N a share - the official said he also would be comfortable with that new range - in N the soft-drink giant earned $ N a share - results for N will include about N cents a share from the effects of snack-food and bottling company acquisitions - in composite trading on the new york stock exchange the company closed yesterday at $ N a share up $ N - the company said third-quarter sales are expected to increase N N from $ N billion of last year 's third quarter - domestic soft-drink case sales are estimated to have risen only N N in the third quarter well below the N N to N N growth of recent years but about in line with the rest of the soft-drink industry - mr. calloway blamed the slower volume on weather a of new products in the industry and to a much lesser extent pricing - pepsico said its soft-drink prices were about N N higher in the quarter - mr. calloway also noted that soft-drink volume rose a hefty N N in last year 's third quarter making the comparison more difficult - international soft-drink volume was up about N N - snack-food increased a strong N N in the third quarter while domestic profit increased in double mr. calloway said - excluding the british snack-food business acquired in july snack-food international jumped N N with sales strong in spain mexico and brazil - total snack-food profit rose N N - led by pizza hut and bell restaurant earnings increased about N N in the third quarter on a N N sales increase - sales for pizza hut rose about N N while bell 's increased N N as the chain continues to benefit from its strategy - bell has turned around declining customer counts by permanently lowering the price of its - same for kentucky fried chicken which has struggled with increased competition in the fast-food chicken market and a lack of new products rose only N N - the operation which has been slow to respond to consumers ' shifting away from fried foods has been developing a product that may be introduced nationally at the end of next year - the new product has performed well in a market test in las vegas nev. mr. calloway said - after a four-year $ N billion acquisition binge that brought a major soft-drink company soda a fast-food chain and an overseas snack-food giant to pepsi mr. calloway said he does n't expect any major acquisition in the next year or so - but you never can tell he added you have to take advantage of opportunities - president bush chose martin a longtime friend from texas to be chairman of the federal energy regulatory commission - mr. would succeed who is resigning - the white house said ms. a chicago who previously held posts at the energy department and ferc is leaving to become a vice president of first chicago corp - mr. an attorney in midland texas has been at the interior department - he met mr. bush in the 1950s when the president was a young oil man in midland and mr. was a lawyer for an oil firm - the ferc is a commission that billions of dollars of interstate wholesale energy transactions - mr. 's appointment is subject to confirmation by the senate - administration officials said a date for ms. 's departure has n't been set - california real estate investment corp. said its directors declared a dividend of five cents per class a common stock payable nov. N to stock of record oct. N - the dividend represents the balance of its regular quarterly payout of N cents a share of which half was paid july N in a final distribution prior to its merger with real estate investment corp. also in july - the company said it hopes to resume its schedule of regular quarterly dividends at the end of this year - hydro-quebec said it notified central maine power co. it will cancel a $ N billion contract to supply electricity to the maine utility - the owned utility said it is up the deal because the contract 's objectives ca n't be - hydro-quebec said maine regulators ' refusal to approve the contract earlier this year halted work on transmission lines and stopped negotiations for resale of electricity carried through maine to other utilities - it would now be impossible to begin deliveries in N a hydro-quebec official said - the contract was to run from N to N - under the contract hydro-quebec was to supply N of power to central maine power starting in N N starting in N and N starting in - hydro-quebec said maine regulators ' refusal to approve the contract means central maine power has lost its place in line - we wo n't sign any new contracts with deliveries beginning earlier than N the hydro-quebec official said - he said hydro-quebec already has some customers in mind for the power that was to be delivered to maine - nothing has happened since we signed the contract to undermine our conviction that hydro-quebec was the most environmentally acceptable choice for meeting a part of our customers ' energy needs through the year N said central maine senior vice president donald f. kelly - central maine said it is evaluating many energy options to make up for the lost future power including new energy generation and management proposals from new england and possibly new canadian purchases - chicago options traders were among the big victims of friday 's plunging stock market including one small firm that required an emergency $ N million bailout - while monday 's rebounding markets helped other investors recoup losses many options customers and professional traders in stock-index options and the options on takeover stocks were left with multimillion-dollar losses traders here and in new york said - options traders were hurt worse than others on friday because of the highly volatile nature of options which often rise or fall in value several times the amount of the price change in the individual stock or index of stocks on which they are based - thus options traders friday were stuck with losses that also were several times larger than those suffered by many stock traders in new york - jeffrey miller of miller & co. said that given the high degree of leverage in the options market it is very easy for these guys to get wiped out - that may just be the nature of these highly leveraged little creatures - an options contract gives the holder the right to buy call or sell put a specific amount of stock or in this case the value of a stock index based on a price within a given time period - options traders who in return for a small fee or premium had previously sold put options on stocks or stock indexes were forced on friday to buy those contracts back at the previously agreed prices which were substantially above those in the market as it was falling - they then had no choice in many cases but to sell the contracts at prevailing prices in most cases at a substantial loss - the latest round of losses is likely to be a serious blow to the chicago board options exchange which has never fully recovered from the of black monday when investors fled the market because of huge losses - making matters worse was the fact that late friday afternoon the cboe halted stock-index options trading in step with the chicago mercantile exchange 's halt in stock-index futures - but while the merc reopened a half hour later the cboe remained closed leaving many options traders unable to make trades that might have reduced the losses - cboe chairman duke said that unlike the futures market the options exchange has to open in a that allows each different options series to trade - exchange officials that they would n't have been able to make such a with the time remaining friday afternoon and with the stock-index futures on the verge of closing for a second and final time the cboe that its best course was to remain closed - the damage was so bad at fossett corp. an options trading firm here that it was forced to transfer its accounts to first options of chicago a unit of continental bank corp. as a result of options trading losses - so far is the only member of a financial exchange to be forced to be taken over by another firm as a result of friday 's rout - fossett still had several million dollars in capital left after friday 's close of trading but not enough that regulators worried about another potential market plunge yesterday would let it reopen for trading options exchange officials said - thus in an unprecedented arrangement the of the transfer the cboe the american stock exchange and the options clearing corp. as well as the firm 's owner stephen fossett put up a total of $ N million to guarantee the customer positions being transferred to the bank holding company subsidiary in case the market plunged again yesterday - s. iii vice chairman of continental bank first options ' parent company said the firm took on about N accounts formerly held by fossett almost all of them to professional floor traders - steve and his firm were still worth a lot of money mr. said - a package of credit support was put together including the assets of steve and his firm - the bailout was together over the weekend with officials from the federal reserve board securities and exchange commission comptroller of the currency and treasury as well as the options exchanges - it was great to have the luxury of time mr. said - at one point an options industry official had to talk the federal reserve bank of chicago 's night into giving him the home phone number of chicago fed president - first options did n't have to put any money into the bailout - yesterday 's rally in the stock futures and options markets led cboe and amex officials to conclude that the $ N million in guarantees almost certainly wo n't need to be tapped by first options - the fossett firm had some losses and liquidity problems during the october N crash as well mr. said - a federal official said that continental bank worked with securities and banking regulators over the weekend to fashion the fossett bailout but that conditions were n't by those agencies - it was their business decision the official said - officials at options clearing corp. which processes all options trades for u.s. exchanges said that the $ N million guarantee was unprecedented but was necessary to help insure the integrity of the options markets - it was an extraordinary situation that needed extraordinary steps said paul stevens president and chief operating officer - mr. stevens declined to give the specific contributions to the $ N million guarantee from each participant - but cboe and amex officials said that options clearing corp. contributed $ N million to the guarantee the cboe put up $ N million the amex added $ N million and $ N million came from mr. fossett 's own assets - mr. fossett could n't be reached to comment - foster takes off her herself on a chair and gently forward - with a tape playing in the background the hands of begin to work on ms. foster 's neck and - it 's like an in this room ms. foster - the room in question is the directors ' of co. N floors above the of pittsburgh - there amid oil paintings and marble tables massages are every wednesday - on days that i 'm really busy says ms. foster who works in public relations for the company it seems to take time off for a massage - although such sessions may never replace coffee breaks on-site massage as it is known in the trade is certainly corporate america - in some companies middle managers massage into the office fearful that executives wo n't approve - ms. foster 's is nothing like the enjoyed by visitors - nor does it at all resemble despite what some executives think the more intimate variety offered at specialty in bad parts of town - on the contrary office usually take place in conference rooms where employees relax in specially designed chairs fully - the massages last N minutes and typically cost about $ N - some companies including even pay part of the fee - ms. has been seeing some N clients a visit since the program was started at last year - anthony the company 's chairman by her firm touch saying regular massages are a for his old football injuries - massage advocates say that the head neck and back can go a long way toward easing tension and improving morale - they also insist that is a basic need as powerful as the need for food or sleep and that the office is as good a place as any to do it - the blood flows to your head you feel and you do n't feel tension around the head or neck says an operations supervisor at the social security office in grand mich. where massages began last month - when you leave the room after your massage people say you look like you 're - adds the who her trade in the grand office they fall in love with my hands - not everyone however is at ease with office massage - three years ago the internal revenue service 's office in san jose calif. opened its doors to on-site massage - and even though employees paid the bill taxpayers - sometimes with the release of stress you hear and coming out of the room explains morgan banks the agency 's health specialist - and you ca n't have taxpayers coming into an audit hearing and - last month the complaints and the massages ended - now we 're looking for a room with walls ms. banks says - massage also has an image problem to contend with - some have tried to get around this by calling themselves and describing their office visits as breaks - but massage no matter how is still associated in many minds with fronts for and that makes some executives nervous - last year the research and development division of weyerhaeuser co. the large concern invited a to its wash. offices - phil a software engineer was an eager customer - you build up a lot of tension working at a terminal all day he says - but after about eight months the vice president of the division ed learned about the sessions and brought them to a halt - mr. says his only beef was that the massages were being given in a company conference room the department 's health facility would have been fine - in my view massages should be managed with an appropriate of males and around he says - given such attitudes some corporate prefer to go about their business quietly - russell of park n.j. says he has been working for the past year at a huge chemical and manufacturing concern in new york to the company 's executives - he visits the same department every two or three weeks - his massage chair is kept in a and a secretary him past security - this is common with a lot of large companies says mr. who worked for american telephone & telegraph co. for N years before choosing his current trade - managers he contends are afraid how they 're going to look in the eyes of their - my vision is to change human touch - my attitude is let 's come out of the - occasionally all that 's needed is a little - a st. louis won over officials at emerson electric co. a maker of electrical and electronic equipment by providing documents and other articles the benefits of massage - she notes that she also during her weekly visits - i pull my hair back wear a little makeup and look corporate says ms. who has been visiting emerson since january - if i go in there as i normally dress they 'd ask who is this - the father of on-site massage is david palmer a san francisco whose mission is to save the - to help do this mr. palmer developed a portable massage chair three years ago that he hopes will bring structured into mainstream america - the culture is not ready to take off its clothes lie down and be touched for an hour for $ N he says - the idea is to keep the clothes on and to keep people - the chair is a way to package massage - sitting in one of mr. palmer 's chairs which cost $ N and have since been by others is a bit like a - customers lean forward rest their on side supports and their face in on the back of the chair - ms. the grand says she has heard the compared to something out of the spanish - mr. palmer who serves as president of the on-site massage association and writes an industry newsletter says some N practitioners out of about N certified across the country now use massage chairs in the workplace as well as on street corners in airports and and at and other where people can be found - a in colo. had a scary experience while a man in a supermarket as part of a store promotion - three minutes into the massage the man up began shaking and turned red - were called - a week later the man told mr. he had suffered a mild heart attack unrelated to the massage - it was a powerful point in my career says the mr. who has since taken out a $ N million liability policy for his business - but he pulled through and after the left there were still six people in line waiting for a massage - the next woman was older and i was afraid to touch her - but it 's like falling off a horse and getting back on - despite the number of fans that office massage has won some look down on it arguing that naked are the only way to go - linda who does work in pittsburgh says that while on-site massage is better than nothing tired workers should realize it is only the tip of the - whole areas of their bodies are neglected she says adding that clothes the experience - there 's nothing like skin to skin - in what is believed to be the first cancellation of a loan to china since the june N killings in beijing an international bank syndicate has terminated a $ N million credit for a shanghai property project - the syndicate led by asia ltd. agreed last november to provide the loan to asia development corp. a u.s. property developer - but several weeks ago in the wake of the beijing killings the loan was canceled according to bankers and executives close to the project - asia development and declined to comment on the move - lenders had doubts about the project even before june N but the harsh crackdown which caused many businesses to their china transactions gave the banks the out they wanted says an official close to the shanghai venture - the decision to cancel the loan the tough attitude bankers have taken toward china since june N - while some commercial lending has resumed international lenders remain nervous about china 's economic troubles and foreign debt $ N billion at the end of N - many loans are being especially those tied to the hotel sector which has been hit hard by a N tourism slump - many bankers view loans as particularly risky - the canceled shanghai loan leaves asia development a small concern with a apartment building and heavy debts - the company owes $ N million to the on group the project 's hong kong contractor and a significant though unspecified amount in legal fees to brothers a u.s. law firm the sources say - the project known as lotus mansion has been mired in controversy - when the loan agreement was announced it was hailed as one of the first western-style financing transactions ever used in china - unlike most loans to china there was no chinese - instead the banks secured a promise from state-owned bank of communications that it would lend asia development the entire $ N million at maturity to finance repayment of the original borrowing - the loan was to have in just two to three years as soon as construction was completed - but in a letter sent in august to asia development said the loan was terminated because the developer had failed to deliver adequate financial data and pay certain fees to the committee on time according to officials close to the project - creditors involved in the project contend however that the termination actually had nothing to do with these technical violations - instead the creditors say the loan fell victim to nervousness about china 's political turmoil as well as to concern about the loan 's security - the bank syndicate is made up mostly of european banks but it includes china 's state-owned industrial bank - the N banks in the syndicate sustained no monetary losses because none of the credit facility had been drawn down - k mart corp. agreed to acquire pace membership warehouse inc. for $ N a share or $ N million in a move to expand its presence in the rapidly growing business - the proposed merger comes as k mart 's profit is declining and sales at its core discount stores are rising more slowly than at such competitors as stores inc - k mart based in mich. recently said net income would fall for the third consecutive quarter after a N N drop in the first half of its current fiscal year - the membership concept has great potential the company 's chairman joseph e. said in a statement - warehouse clubs typically carry general merchandise and food products which they sell for close to wholesale prices in stores - shoppers many of whom operate small businesses pay annual membership fees which provide an income base for the stores - k mart tested the sector last year with its acquisition of a N N interest in inc - but the chain which operates as a joint venture between k mart and shv holdings n.v. of the netherlands has only six stores and annual sales that one analyst estimated at about $ N million - pace based in colo. operates N stores - the company had losses for several years before turning profitable in fiscal N - in the year ended jan. N pace up profit of $ N million or N cents a share after a tax-loss carry-forward on sales of $ N billion and analysts expect its results to continue to improve - the company turned the corner fairly recently in profitability said of painewebber inc. who had been forecasting a N N jump in pace 's net income from operations this year and another N N increase next year - warehouse productivity is really beginning to take off - but some analysts contend k mart has agreed to pay too much for pace - even if you look at it as a turnaround situation it 's expensive said wayne of prudential-bache securities inc - in my opinion you would only pay that kind of price if you were getting a premier player in the industry - ms. of painewebber raised a more fundamental question about the deal - if k mart ca n't get its act together in discounting why is it spending time worrying about other growing markets - she said i would say k mart 's number one job is to address its market-share loss in discount stores which longer-term will lead to improved profit margins - at that point perhaps diversification would be appropriate - but k mart 's mr. is intent on pushing the company into new retail businesses - for instance k mart is opening big food and general merchandise stores called and stores specializing in office products and sporting goods - it also operates pay less drug stores and builders square home improvement stores - in composite trading on the new york stock exchange k mart closed yesterday at $ N a share up N cents - pace rose $ N to close at $ N a share in national over-the-counter trading - a k mart spokesman said the acquisition would be financed with short-term borrowings - under terms of the agreement a k mart subsidiary will soon make a tender offer for pace shares - among the conditions of the offer is that pace shareholders tender a majority of the company 's shares outstanding - the companies said pace would ill continue to operate under its present management - g. william president of stations was named chief executive officer of the unit of this media company effective jan. N - he will succeed joel who will remain a vice president of the company and continue to represent stations in several industry organizations the company said - literally - traders nervously watching their quotron machines yesterday morning were stunned to see the dow jones industrial average plummet N points in seconds - a minute later it soared N points then back down N points N below friday 's close - it was crazy said neil general partner of capital corp - it was like flying without a pilot in the front of the plane - but those who said this ca n't be happening were right - the were wrong - quotron systems inc. a citicorp unit blamed the on a timing problem in our software caused by the enormous early volume about N million shares in the first hour of new york stock exchange trading - the prices of the individual stocks that make up the average were correct quotron said but the average was wrong - meanwhile there was an awful lot of confusion - at about N a.m. on the over-the-counter trading desk at a major brokerage firm a veteran trader who buys and sells some of the most active stocks looked at a senior official and asked what 's going on - is the market up or down - at the time quotron was reporting that the industrial average was down N points - in fact it was up N - stark a vice president who heads the trading desk at dillon read capital corp. said that once she figured out the quotron numbers were wrong she called brokers to tell them - it 's been kind of to say the least she said - to matters further when ual corp. stock finally opened on the new york stock exchange at N a.m. the price was listed at $ N a share up about $ N from friday in fact its true price was $ N down $ N - that was the new york stock exchange 's - a spokesman cited a technical error and declined to elaborate - and there were other - when the market opened at N a.m. est a reporter for the reuters the industrial average 's drop as a N N decline when it really was down N N - it was a case of human error which we found almost immediately and corrected a spokesman for reuter in new york said - meanwhile some currency traders at west german banks in frankfurt said they sold dollars on the news and had to buy them back later at higher prices - but it was the quotron problems that had effects - dillon read 's ms. stark said in early afternoon that she was still prices and other data as subject to and she said portfolio managers continued to question the numbers they saw on the screen - it was the second time in less than a week that quotron has had problems the industrial average - at the start of trading last wednesday the average appeared to plunge more than N points - actually it was down only a few points at the time - quotron said that which lasted nine minutes resulted from a failure to adjust for a stock split at philip morris - a quotron spokeswoman said recent software changes may have contributed to yesterday 's problems - she said quotron switched to a backup system until the problems were corrected - today of all days she - the eyes of the world were watching us - steven f. was named a senior vice president of this graphics equipment company - he retains his current positions as chief strategic officer of am international and president of am ventures - houston attorney dale friend representing a plaintiff in a damage suit says he has negotiated a settlement that will strike a blow for his client - literally - it turns out mr. friend 's client parks of cincinnati did n't like the way defense attorney tom alexander acted during the legal proceedings - so she has agreed to monetary damages against mr. alexander 's client in return for the right to the attorney - ms. parks 's mother also gets to mr. alexander - so does mr. friend and his law partner - the bizarre arrangement grows out of mr. alexander 's representation of construction co. one of several defendants in a death lawsuit brought by ms. parks the widow of a construction worker killed in january N while working on a new houston convention center - last month mr. friend says mr. alexander 's associate agreed that would pay $ N as part of an overall settlement - but mr. alexander the deal at the last minute the plaintiff 's side - i never agreed to it mr. alexander says adding that it 's not necessary to pay these settlements - when ms. parks and her mother heard about what had happened mr. friend says they that they would like to give mr. alexander a good - mr. friend says he passed that along to his adversary and soon they were talking about the ground rules under which could keep its money and the plaintiffs could take a shot at mr. alexander - although time and place have yet to be determined some details are in place - mr. friend says he agreed to strike mr. alexander above the belt - ms. parks and her mother indicated they want to catch him from behind he says - mr. alexander for his part insisted that the ca n't their rights to anyone else ca n't use a blunt instrument and ca n't take a running start - mr. alexander says he the agreement which has n't been submitted to a judge as something of a joke - however he acknowledges they have the option of taking a at me if they really want to - mr. friend says his side is dead serious - although they do n't delivering any he says that mr. alexander will be asked to sign a release from liability just in case - after two years of drought it money in the stock-index futures markets yesterday - as financial markets rebounded trading volume in the chicago mercantile exchange 's huge standard & poor 's N stock-index futures pit soared reaching levels for the first time since october N - the sudden influx of liquidity enabled several traders to reap in a matter of minutes as prices soared traders said - guys were money in there today said john a futures broker for elders futures inc. in chicago - the s&p N futures contract which moves in of an index point under normal conditions jumped two to three points in seconds early yesterday after an initial downturn then moved strongly higher the rest of the day - each index point represents a $ N profit for each s&p N contract held - for the first time since the N crash traders said that they were able to trade several hundred s&p N contracts at a time in a highly liquid market - many institutions and individual investors have away from stock-index futures blaming them for speeding the stock market crash on black monday two years ago - since the crash many futures traders have n't assumed large positions for fear that the s&p N market with much of its customer order flow missing would dry up if prices turned against them - more than N traders the s&p N futures pit to await the opening bell - traders were shouting bids and offers a full five minutes before the start of trading at N am - the contract fell five points at the open to N the maximum opening move allowed under adopted by the merc to stem a market slide - but several traders quickly stepped up and bid for contracts driving prices sharply higher - the market near friday 's closing price of N for about a half hour moving several index points higher or lower in seconds then broke higher and did n't look back - the s&p N contract that expires in december closed up a record N points on volume of nearly N contracts - traders five feet from each other were making bids and offers that were a full point apart said one s&p N broker - you could buy at the bid and sell at the offer and make a fortune he - several of wall street 's largest securities firms including salomon brothers inc. and painewebber inc. were also large buyers traders said - salomon brothers was among the largest sellers of stock-index futures last week traders said - brokerage firms as a rule do n't comment on their market activity - unlike the week following black monday two years ago individual traders in the s&p N pit were also being about their one-day profits - with the fbi around here rights are a thing of the past said one trader referring to the federal investigation of futures trading that so far has resulted in N against individuals on the merc and the chicago board of trade - the market for $ N billion of high-yield junk bonds regained some of its footing as the dow jones industrial average rebounded from friday 's plunge - but the junk recovery led by the bellwether rjr holdings bonds was - no trading existed for the vast majority of junk bonds securities industry officials said - on friday trading in practically every issue ground to a halt as potential buyers fled and brokerage firms were unwilling to provide bid and offer prices for most issues - nothing traded on friday and people were n't really sure where the market should have opened yesterday said raymond of merchant banking at merrill lynch & co - but we had a fairly active day yesterday - at drexel burnham lambert inc. the leading underwriter of junk bonds i was prepared to be in a very bad mood tonight said david a junk bond trader - now i feel maybe there 's a little bit of euphoria - but before the stock market rebounded from a sharp early sell-off yesterday he said you could n't buy junk bonds and you could n't give them away - yesterday 's rally was led by rjr holdings N N N bonds which initially tumbled three points or $ N for each $ N face amount to N N before rebounding to N N - bonds issued by and american standard also showed big gains recovering almost all their losses from friday and early yesterday - but traders said the junk bond market increasingly is into a group in which trades can be executed easily and a larger group of bonds in which liquidity or the ability to trade without too much difficulty has steadily deteriorated this year - liquidity has n't returned to the vast middle ground of the market said mr. of merrill - the are still said mr. of drexel - analysts are concerned that much of the high-yield market will remain for investors - paul associate professor at the massachusetts institute of technology 's sloan school of management citing a pattern of junk-bond default rates that are low in the early years after issuance and rise later says we 're now in a period where we 're starting to see defaults from the big issue years of N to N - mark a senior vice president at standard & poor 's corp. confirms that there is increasing concern about the future liquidity of the junk bond market - junk bonds are a highly market said lewis vice chairman of smith barney harris upham & co - there 's a whole bunch of stuff that 's money good and a whole bunch of stuff that 's not so good - analysts at standard & poor 's say junk bond offerings by tightly stretched issuers seem to be growing - almost $ N billion of junk bonds that are considered include issues from sci tv gillette holdings not related to gillette co. furniture allied stores federated department stores national holdings leaseway transportation and price communications - you could still have some very bad times ahead said mr. - it 's possible to have a N N default rate in one year because we 're already seeing big problems in the midst of a pretty strong economy - i 'm certainly not comfortable saying we 've seen the bottom - but yesterday 's rally among good junk was a badly needed for the market - many issues bounced off the floor mr. said and benchmark junk issues recovered all of their losses from friday and early yesterday - in contrast he says the stock market gained back only about half what it lost friday and the government bond market lost about half what it gained friday - traders said yesterday 's rally was fueled by insurance companies looking for bargains after a drastic slide in prices the past month - in addition mutual funds did n't appear to be major sellers of high-yield securities as was expected - sometimes a is healthy said drexel 's mr. - people will learn to be more - if they do good credit analysis they will avoid the hand - i think the market is in good shape - should you really own stocks - that 's a question a lot of people are asking following the stock market 's stunning display of volatility - financially and by friday 's 190-point drop in the dow jones industrial average and yesterday 's rebound they 're wondering if an individual has any business being in the market - the answer say academic researchers money managers and investment specialists is yes as long as you approach the stock market as an investor - but they say people should n't try to be traders who buy and sell in an effort to ride the latest economic trend or catch the next hot stock - the case for owning stocks over the long-term is compelling - if you look at N years worth of investment history including the great depression and every bear market since stocks have outperformed almost everything an individual could have owned by a long shot says barry berlin vice president at first wachovia capital management - a dollar invested in the stock market in N would have grown to $ N by the end of last june according to laurence managing director at associates inc - but a dollar invested in long-term bonds in N would have grown to only $ N and a dollar put in treasury bills would equal a $ N - the longer the time period the less risk there is of losing money in the stock market - over time the odds increasingly favor the investor with a diversified portfolio - for instance ken gregory a san francisco money manager that if an investor holds a basket of stocks that tracks the standard & poor 's 500-stock index the chance of losing money is N N to N N over a 10-year period compared with N N over three years and N N over one year - if you do n't need the money for N years there 's a case for sticking to a steady core of stocks mr. gregory says - stock-market investments also help balance the other assets an individual owns says john jr. president of the institute of certified financial planners - stocks have a place in an investors ' portfolio along with real estate bonds international securities and cash he says - there are some important before investing in stocks individuals should have at least three to six months of living expenses set aside in the bank most investment advisers say - individuals also should focus on building equity in a home which provides some protection against inflation as well as a that can be in late in life to help cover the cost of retirement living - people also should n't invest money in stocks that they 'll need in the near future for example for college tuition payments or retirement expenses - you may have to sell your stocks at a time when the market takes a plunge says mr. a del calif. financial planner - but once the are covered then i would start to invest even if it 's as little as $ N says michael lipper president of lipper analytical services inc - he says individuals should consider not just stocks but other long-term investments such as high-quality bonds - despite the strong case for stocks however most pros warn that individuals should n't try to profit from short-term developments - it 's very difficult to do says donald holt a market strategist for morgan securities a los angeles brokerage firm - our markets move so fast and they are so volatile there 's no way the average investor can compete with the pros - individual investors face high transaction costs of moving in and out of the market - the cost of executing stock orders from brokerage to brokerage and with the size of the order but N N of the order 's value is an average says stephen boesel manager of t. rowe price 's growth and income mutual fund - and assuming their first investment is successful investors will have to pay taxes on their gains - that can reduce returns by a third or more once local taxes are included mr. lipper says - after that individual traders face the risk that the new investment they choose wo n't perform well so their trading costs could be sustained for nothing - it 's very tough for most individuals to the mutual funds or the market says mr. lipper - you should really think twice if you think you can the system - then too many individual investors lack the emotional makeup professionals say is needed to plunge in and out of the market - so what 's the best way to buy stocks - unless an individual has a minimum of between $ N and $ N to invest in stocks he 's still better off in mutual funds than in individual stocks in terms of getting enough attention from a competent broker says mr. lipper - still he adds i could see owning both given that individuals often have an advantage over big investors in special situations based on their own he adds - george douglas first vice president at drexel burnham lambert inc. says that individuals have a particular edge now in small to niche companies with exciting earnings prospects a traditional ground for small investors - this growth sector which usually carries a multiple about twice that of the standard & poor 's N happens to include some of the market 's most attractive bargains right now - it 's now selling at a multiple about even with the market says mr. douglas - moreover mr. douglas sees a revival of institutional interest in smaller growth stocks that could boost the performance of these stocks in the medium term - many big wall street brokerage firms who eliminated their research effort in stocks of emerging growth companies a few years ago are now coverage of this area he notes - we 're seeing a real turnaround in interest in small growth stocks he says - the pros advise individuals to stay away from the latest investment fad - they say that 's especially important this late in the growth phase of the economic cycle when there 's no robust bull market to bail investors out of their mistakes - friday 's correction presents a pretty good buying opportunity but let 's not speculate at this point in the business cycle says chief equity portfolio strategist at first boston corp - buy stocks on weakness for their long-term fundamentals he says - in the long run investment advisers say most investors will be better off using the averaging method of buying stocks - in this method a person invests a regular amount every month or quarter into the stock market whether the market is up or down - that cuts the risk mr. gregory the san francisco money manager points out - when the market is low you are buying more shares and when it 's high you 're buying fewer shares he says - otherwise if you put all your money in at one time by sheer bad luck you might pick a terrible time and have to wait three years to get even mr. gregory says - a disciplined program will work the best mr. boesel says - one of the hardest things to do is to buy stocks when the market is down he says - but that 's just the time when you should be buying them - compound annual returns including price changes and income from interest and dividends - \* actual performance not annualized - source associates inc - the following issues were recently filed with the securities and exchange commission co. initial public offering of two million shares of common stock of which N shares are being offered by the company and N shares by holders via blunt ellis & inc. and robert w. & co - giant industries inc. initial public offering of N common shares of which N will be sold by the company and the rest by holders via shearson lehman hutton inc. and inc - fund inc. initial offering of five million common shares via smith barney harris upham & co - overseas ltd. initial offering of four million common shares of which N million will be sold in the u.s. and the balance outside the u.s. via smith barney harris upham & co. and & co - donald trump who faced rising doubt about his bid for american airlines parent amr corp. even before a united airlines buy-out came apart friday withdrew his $ N billion offer - separately bankers representing the group trying to buy united 's parent ual corp. met with other banks about that purchase at a lower price possibly around $ N a share or $ N billion - but a lower bid could face rejection by the ual board - mr. trump who vowed wednesday to go forward with the bid said he was dropping it in light of the recent change in market conditions - he said he might now sell his amr stake buy more shares or make another offer at a lower price - the manhattan real-estate developer acted after the ual buyers failed to obtain financing for their earlier $ 300-a-share bid which sparked a selling panic among that into a 190-point drop friday in the dow jones industrial average - news about ual and amr whose shares never reopened after trading was halted friday for the ual announcement sent both stocks in composite trading on the new york stock exchange - ual tumbled $ N to $ N on volume of N million shares and amr declined by $ N to $ N as N million shares changed hands - together the two stocks havoc among takeover stock traders and caused a N N drop in the dow jones transportation average second in size only to the stock-market crash of oct. N N - some said friday 's market debacle had given mr. trump an excuse to bail out of an offer that showed signs of even before problems emerged with the ual deal - after reaching an intraday high of $ N the day mr. trump disclosed his bid oct. N amr 's stock had retreated as low as $ N last week - some takeover stock traders had been betting against mr. trump because he has a record of disclosing stakes in companies that are potential takeover targets then selling at a profit without making a bid - he still has n't proven his as a artist said airline analyst kevin murphy of morgan stanley & co - he 's done this thing where he 'll buy a little bit of a company and then trade out of it - he 's written this book the art of the deal - why does n't he just follow through on one of these things - mr. trump withdrew his bid before the amr board which is due to meet tomorrow ever formally considered it - amr had weighed a wide range of possible responses from flat rejection to and leveraged buy-outs that might have included either employees a buyer such as texas billionaire robert bass or both - amr had also sought to mr. trump in congress by lobbying for legislation that would have bolstered the authority of the transportation department to reject airline buy-outs - yesterday mr. trump tried to put the blame for the collapse of the ual deal on congress saying it was rushing through a bill to protect amr executives - i believe that the perception that legislation in this area may be hastily approved contributed to the collapse of the ual transaction and the resulting disruption in the financial markets experienced this past friday mr. trump wrote members of congress - amr declined to comment and mr. trump did n't respond to requests for interviews - mr. trump never said how much amr stock he had bought only that his holdings were substantial - however he only received federal clearance to buy more than $ N million of the stock on sept. N when the price rose $ N a share to $ N - between then and his bid on oct. N the price between $ N and $ N - in an attempt to persuade investors that his bid was n't just a stock play mr. trump promised last week to notify the market before selling any shares - amr was trading at around $ N yesterday before his withdrawal announcement then immediately fell to about $ N - assuming that he paid a rough average price of $ N a share and assuming he did n't sell before his announcement reached the market mr. trump could be sitting with a modest loss with the stock at $ N - some analysts said amr chairman robert crandall might seize the opportunity presented by the stock price drop to protect the nation 's largest airline with a defensive transaction such as the sale of stock to a friendly holder or company employees - however other knowledgeable observers said they believed mr. crandall and the amr board might well decide to tough it out without taking any extra steps - some analysts said they believed mr. trump whose had been viewed by some as a reason to believe he would n't back out might come back with a lower bid - ray of dillon read & co. said mr. trump is stepping back and waiting for the dust to settle - i 'm sure he still wants amr - but others remained skeptical - i was never sure donald trump really wanted to take amr said john a bond analyst with shearson lehman hutton inc - what happened with united was a way for him to out - mr. trump never obtained financing for his bid - that skepticism would leave him with an even greater credibility problem should he return that would him in any effort to oust the board in a proxy fight - meanwhile citicorp and chase manhattan corp. the two lead lenders on the ual buy-out met with other banks yesterday to determine if they would be willing to finance the buy-out at a lower price - officials familiar with the talks said citicorp had discussed lowering the offer to $ N a share but said that price was a talking point and that no decision has been made - at $ N a share the group would have to borrow about $ N billion from banks - the first ual deal unraveled after citibank and chase could n't raise $ N billion - citibank and chase had agreed to commit $ N billion and said they were highly confident of raising another $ N billion - together citicorp and chase received $ N million in fees to raise the rest of the financing - but other banks balked at the low interest rate and banking fees the ual group was willing to pay them - officials familiar with the bank talks said the ual buy-out group ual pilots management and british airways plc is now willing to pay higher bank fees and interest but is n't likely to boost its $ N million equity contribution - nor is the group likely to come forward with a revised offer within the next N hours despite the hopes of many traders - the group 's advisers want to make certain they have firm bank commitments the second time around - even if the buy-out group is able to obtain financing the transaction still faces obstacles - ual 's board could reject the new price as too low especially since there are n't any competing bids - los angeles investor marvin davis whose $ offer was rejected by ual 's board has n't shown signs of pursuing a $ 300-a-share bid he made last month - in addition the coalition of labor and management longtime enemies who joined forces only under the threat of mr. davis 's bid could break apart now - the group 's resilience gets its first test today when N top pilot union leaders outside chicago in a previously scheduled meeting - union chairman rick faces the tough task of explaining why banks refused to finance a buy-out the members approved last week - the pilot union is to pursue an acquisition whatever the board decides - but if the board a reduced bid and decides to explore other alternatives it could transform what has been a process into an one - the pilots could play by noting they are crucial to any sale or restructuring because they can refuse to fly the airplanes - if they were to insist on a low bid of say $ N a share the board might n't be able to obtain a higher offer from other bidders because banks might hesitate to finance a transaction the pilots oppose - also because ual chairman stephen wolf and other ual executives have joined the pilots ' bid the board might be forced to exclude him from its deliberations in order to be fair to other bidders - that could cost him the chance to influence the outcome and perhaps join the winning bidder - influential members of the house ways and means committee introduced legislation that would restrict how the new savings-and-loan bailout agency can raise capital creating another potential obstacle to the government 's sale of sick thrifts - the bill whose backers include chairman dan d. ill. would prevent the resolution trust corp. from raising temporary working capital by having an bank or thrift issue debt that would n't be counted on the federal budget - the bill intends to restrict the rtc to treasury borrowings only unless the agency receives specific congressional authorization - such agency borrowing is unauthorized and expensive far more expensive than direct treasury borrowing said rep. stark d. calif. the bill 's chief sponsor - the complex financing plan in the s&l bailout law includes raising $ N billion from debt issued by the newly created rtc - this financing system was created in the new law in order to keep the bailout spending from swelling the budget deficit - another $ N billion would be raised through treasury bonds which pay lower interest rates - but the rtc also requires working capital to maintain the bad assets of thrifts that are sold until the assets can be sold separately - that debt would be paid off as the assets are sold leaving the total spending for the bailout at $ N billion or $ N billion including interest over N years - it 's a problem that clearly has to be resolved said david executive director of the rtc - the agency has already spent roughly $ N billion selling N insolvent s&ls and it is likely to sell or merge N by the time the bailout concludes - other working capital he said the rtc would be forced to delay other thrift resolutions until cash could be raised by selling the bad assets - we would have to wait until we have collected on those assets before we can move forward he said - the complicated language in the huge new law has the fight - the law does allow the rtc to borrow from the treasury up to $ N billion at any time - moreover it says the rtc 's total obligations may not exceed $ N billion but that figure is derived after including notes and other debt and from it the market value of the assets the rtc holds - but congress did n't anticipate or intend more public debt say opponents of the rtc 's plan and rep. charles d. n.y said the rtc oversight board has been in not keeping congress informed - that leads to a proposal like the one from ways and means which seems to me sort of he said - the rtc is going to have to pay a price of prior on the hill if they want that kind of flexibility - the ways and means committee will hold a hearing on the bill next tuesday - we 're about to see if advertising works - hard on the heels of friday 's 190-point stock-market plunge and the uncertainty that 's followed a few big brokerage firms are rolling out new ads a familiar message keep on investing the market 's just fine - their mission is to keep clients from the market as individual investors did in after the crash in october - just days after the N crash major brokerage firms rushed out ads to calm investors - this time around they 're moving even faster - painewebber inc. a new television commercial at N p.m. edt yesterday and had it on the air by last night - fidelity investments placed new ads in newspapers yesterday and wrote another new ad appearing today - shearson lehman hutton inc. by yesterday afternoon had already written new tv ads - it considered running them during tomorrow night 's world series broadcast but decided not to when the market recovered yesterday - other brokerage firms including merrill lynch & co. were out potential new ad strategies - the brokerage firms learned a lesson the last time around when frightened investors flooded the phone lines and fled the market in a panic - this time the firms were ready - fidelity for example prepared ads several months ago in case of a market plunge - when the market went into its free fall friday afternoon the investment firm ordered full pages in the monday editions of half a dozen newspapers - the ads touted fidelity 's automated beneath the huge headline fidelity is ready for your call - a fidelity spokesman says the which already was operating but which many clients did n't know about received about double the usual volume of calls over the weekend - a lot of investor confidence comes from the fact that they can speak to us he says - to maintain that dialogue is absolutely crucial - it would have been too late to think about on friday - we had to think about it ahead of time - today 's fidelity ad goes a step further encouraging investors to stay in the market or even to plunge in with fidelity - the headline diversification it based on the events of the past week all investors need to know their portfolios are balanced to help protect them against the market 's volatility - it goes on to plug a few diversified fidelity funds by name - painewebber also was able to gear up quickly thanks to the N crash - in the aftermath of the N debacle the brokerage firm began taping commercials in-house ultimately getting its timing down fast enough to tape a commercial after the market closed and rush it on the air that night - it also negotiated an arrangement with cable news network under which would agree to air its last-minute - the new painewebber commercial created with ad agency saatchi & saatchi co. features mary farrell one of the firm 's most visible investment strategists particularly bullish - taped just as the market closed yesterday it offers ms. farrell advising we view the market here as going through a relatively normal cycle - we continue to feel that the stock market is still the place to be for long-term appreciation - the spot was scheduled to appear three times on last night - painewebber considered an even harder sell recommending specific stocks - instead it settled on just urging the clients who are its to keep that money in the market - we 're saying the worst thing that anyone can do is to see the market go down and dump everything which just drives the prices down further says john painewebber 's director of advertising - if you owned it and liked it friday the true value has n't changed - he adds this is n't N - with the market and then closing up more than N points yesterday investment firms had to constantly revise their approach - at shearson lehman executives created potential new commercials friday night and throughout the weekend then had to yesterday afternoon - the plan had been to make one of shearson 's black-and-white where we stand commercials which have been running occasionally in response to news events since N - the ad would have run during the world series tomorrow replacing the debut commercial of shearson 's new ad campaign leadership by example - but in a meeting after the market closed yesterday shearson executives decided not to go ahead with the stock-market ad - we do n't think at this point anything needs to be said - the market seems to be out we 're taking a attitude says b. stewart executive vice president of marketing - in any case the brokerage firms are clearly moving faster to create new ads than they did in the fall of N - but it remains to be seen whether their ads will be any more effective - in N despite a of ads from most of the major investment firms individuals ran from the market en - now the firms must try their hardest to prove that advertising can work this time around - ad notes - arnold advertising - edward former chairman of della femina mcnamee reached an agreement in principle to acquire a majority stake in arnold advertising a small boston shop - terms were n't disclosed - mr. who resigned his della femina post in september becomes chairman and chief executive of arnold - john the agency 's president and chief executive will retain the title of president - separately mcdonald 's corp. oak ill. named arnold to handle its estimated $ N million cooperative ad account for the hartford conn. area - that account had been handled by della femina mcnamee wcrs - education ads - a ad supplement to business week 's special corporate elite issue calls on business leaders to use their clout to help solve the nation 's education crisis - the supplement the largest ever for the magazine includes ads from N corporate advertisers and off a two-year business week initiative on education - the magazine will distribute N N of the gross revenues from the supplement as grants to innovative teachers - you know what the law of averages is do n't you - it 's what N explains why we are like well ourselves rather than jackson N that it 's possible to in a lake that averages two feet deep and N predicts that N placed before N would produce N rock roll - baseball that game of the long haul is the sport of the mean and the mean law caught up with the san francisco giants in the world series last weekend - the team that dumped runs by the bushel on the chicago cubs in the national league playoffs was held to just one in two games by the oakland a 's the gang that had been done similarly by the los angeles and in last year 's - much of the damage was accomplished by a 's who had some catching up to do - in game two on a cool sunday evening in this land of perpetual autumn a lot of the catching up was done by the a 's terry - he hit a N pitch from rick into the stands in inning four to stretch his team 's lead from N to a decisive N where it stayed - so what if had struck just seven home runs in N regular-season games and in the seventh position of the a 's lineup - if you get your pitch and take a good swing anything can happen he later - on saturday night quite a few of the boys in green and gold away successes to the pain of past and no doubt future - mark the big oakland first had three hits in four at two more than he 'd had in the series in which he 'd gone - the N through N the bottom of the order got seven of their team 's N hits and scored four of its runs in a N decision - dave stewart held the giants to five hits to account for the zero on the other side of the saturday - that he was the a 's during its american league campaign with a N mark plus two wins over toronto in the playoffs indicates he may have some evening up coming but with the way his is that might not be this week - the same goes for mike moore another veteran who early struggles to permit the giants but a run and four hits in seven in sunday 's contest - every guy they put out there had a better than the guy before giant manager roger craig - he 's an who 's one of the leading of the fashionable delivery which looks like a until it beneath the bat - the of the is that the a 's go into san francisco 's candlestick park tonight up two games to none in the - the to with here says that about three of four clubs N of N that took N series leads went on to win it all - that 's not an average to giant - one might think that the home fans in this series of the subway called bart that 's a better name for a public than desire do n't you think would have been over the proceedings but they them in relative calm - of the two sat side by side in the seats of oakland and while they cheered their and the opposition advanced no further at least as far as i could see - a few folks even showed up wearing bearing the colors and of both teams - i 'm for the giants today but only because they lost yesterday - i love both - the only thing i 'm for is for the series to go seven games said david williams a sacramento at the before sunday 's go - the above represents a of either or - i choose to believe it 's the latter although it probably springs from the fact that just about everyone out here including the a 's and giants is originally from somewhere else - it to say that if this were a new york series or one between the chicago cubs and white it 's possible you 'd need police in every other seat to separate opposing fans and only the would their - anyway the a 's gave you a lot of heroes to root for - in the opening game besides and stewart there was walt weiss a who had lost a couple months of the season to surgery - he was in game two moved a along in the a 's second inning and for his team 's final tally - such is his reputation among the east bay that when he hit his first career home run last season the fan who caught it agreed to turn the ball over to him in return for an - not his 's - an a 's of the second game was henderson who the hot side of the equation - he toronto in the playoffs with six hits seven walks and eight stolen bases in N at and continued that by going at the plate sunday along with walking stealing a base and scoring a run - when you 're in the you see every ball he - the cold guys in the set were will clark kevin mitchell and williams the giants ' N - they combined for N hits six home runs and N runs in in the five games against the cubs - they went a collective here with zero and - it 's that last set of numbers as much as anything else that gives the giants hope in the series games to come - i believe in the law of averages declared san francisco coach dusty baker after game two - i 'd rather see a who 's hot come up for the other side than a good who 's cold - but the old offered no prediction about when good times would return to his side - when it goes you never know when you 'll get it back he said - that 's baseball - ncr corp. reported a N N drop in third-quarter net income citing intense competition that caused its gross profit margins to dip - net income for the quarter fell to $ N million from $ N million roughly what analysts had expected - but per-share profit dropped only N N to $ N a share from $ N a share as the company continued its stock buy-back plan - average shares outstanding dropped to N million from N million - revenue fell N N to $ N billion from $ N billion - the computer maker which sells more than half its goods outside the u.s. also said the negative effect of a stronger u.s. dollar will affect its fourth-quarter performance and make it difficult to better N results - ncr said revenue declined both in the u.s. and overseas reflecting a world-wide softening of the computer markets - the company however said orders in the u.s. showed good gains during the latest quarter - analysts estimate those gains at N N to N N a good part of it coming from large orders placed by a few of ncr 's major customers - in addition to a general slowing of the computer industry ncr which sells automated teller machines and computerized cash is also affected by the retail and financial sectors areas of the economy that have generally not been robust notes g. an analyst for salomon brothers inc - these factors combined with a strong dollar should affect the current quarter 's results ncr said - in the year-earlier fourth quarter ncr had profit of $ N million or $ N a share on revenue of $ N billion - mr. said he lowered his full-year estimates for N to $ N a share from $ N a share - revenue projections were slashed to $ N billion from $ N billion - last year ncr had net income of $ N million or $ N a share on $ N billion in revenue - for the nine months the company 's earnings fell N N to $ N million or $ N a share from $ N million or $ N a share - revenues declined N N to $ N billion from $ N billion - in new york stock exchange composite trading yesterday ncr shares fell N cents to close at $ N - concerning your sept. N article wall street firms link analysts ' pay to performance i 'm that wall street is finally in to the hard cold facts of the real working world - if the firms are serious however why limit the practice to the poor analysts whose ability to see into the future is fragile at best - why not extend the same harsh standards to the sales force and pay brokers a base salary with annual bonus based on how much money they made for their clients during the year - that should stop a lot of and produce a stock market driven only by professional concern careful thought and good sense - now would n't that be a - newport news va - steve clark a shearson lehman hutton inc. trader reported for work at N a.m. two and a half hours before the usual monday morning strategy meeting - at jefferies & co. j. francis did n't reach the office until N a.m. but then he had been up most of the night at home - i had calls all night long from the states he said - i was up every hour N N N N - people are looking for possible opportunities to buy but nobody wants to stick their out - for many of london 's securities traders it was a day that started nervously in the small hours - by the selling was at fever - but as the day ended in a wall rally the city a sigh of relief - so it went yesterday in the trading rooms of london 's financial district - in the wake of wall street 's plunge last friday the london market was considered especially vulnerable - and before the opening of trading here yesterday all eyes were on early trading in tokyo for a clue as to how widespread the fallout might be - by the time trading officially got under way at N a.m. the news from asia was in - and it left mixed signals for london - tokyo stocks closed off a significant but N N on thin volume hong kong stocks declined N N in orderly trading - at jefferies ' trading room on circus a circle at the edge of the financial district desktop computer screens displayed the london market 's major barometer the financial times-stock exchange N share index - red figures on the screens indicated falling stocks blue figures rising stocks - right away the outnumbered the blues N to N as the index opened at N off N points or N N - i see concern but i do n't see any panic said mr. a big new york native who runs the office - the jefferies office a branch of the los angeles-based firm played it seeking to avoid risk - this is not the sort of market to have a big position in said david smith who heads trading in all non-u.s. stocks - we tend to run a very tight book - jefferies spent most of its in the morning trying to match buyers and sellers and there were n't many buyers - all the takeover stocks scottish & b.a.t are getting pretty well this morning mr. smith said - seconds later a sell order for scottish & came in - for the third time in N minutes a trader next to mr. smith left the area to have a cigarette - on the screens only two blue figures remained but the index had recovered a few points and was off about N - because tokyo did n't collapse let 's pick up a little stock mr. smith said - he targeted N shares of reuters and a to call up on his screen other dealers ' price quotes - the vivid yellow figures showed the best price at N pence $ N and mr. smith 's traders started putting out - but the market a serious buyer on a day dominated by selling and the quotes immediately jumped to N pence - when i want to buy they run from you they keep changing their prices mr. smith said - it 's very frustrating - he temporarily abandoned his search for the reuters shares - by this time it was N a.m. in new york and mr. smith a call from a new york customer wanting an opinion on the british stock market which had been having troubles of its own even before friday 's new york market break - fundamentally dangerous mr. smith said almost in a fundamentally weak fairly vulnerable still extremely poised - we 're in for a lot of turbulence - he was right - by midday the london market was in full retreat - it 's falling like a stone said danny a pit trader who was standing outside the london international financial futures exchange - only half the usual crowd gathered at the tony & wine bar on old broad street nearby - conversation was subdued as most watched the latest market statistics on television - at N p.m. the index hit its low N off N points - france opened the limit down off at least N N if you could calculate the index which you could n't mr. clark the shearson trader said early in the afternoon - spain is down N N and suspended sweden 's down N N norway N N - this market has been very badly damaged - as N p.m. wall street 's opening time shearson traders and salesmen traded bets on how low the new york market would open - in the center of the trading floor chief trader roger and two colleagues scrambled for the telephones as soon as the new york market opened more than N points in the first few minutes - they saw an opportunity created by the sell-off - as wall street traders dumped american depositary receipts in jaguar plc mr. and trader sam bought them to in the - investors here still expect ford motor co. or general motors corp. to bid for jaguar - suddenly after about N minutes the u.s. markets rallied - the mmi has gone better shouted one trader at about N london time as the u.s. major markets index contract suddenly indicated a - as wall street strengthened the london trading room went wild - traders shouted as their screens posted an loss on wall street - then nine minutes later wall street suddenly rebounded to a gain on the day - rally rally rally shouted shearson trader andy rosen selling more jaguar shares - this is panic buying - as the london market rallied some whether the weekend of worrying and jitters had been worth it - the london index closed at N its high for the day off N or about N N - ambassador paul 's statement notable & sept. N if you have a million people working for you every bad thing that has one chance in a million of going wrong will go wrong at least once a year is a pretty negative way of looking at things - is n't it just as fair to say that if you have a million people working for you every good thing that has one chance in a million of going right will go right at least once a year - do n't be such a mr. ambassador - frank - the house aviation subcommittee approved a bill that would give the transportation secretary authority to review and approve leveraged buy-outs of major u.s. airlines - the collapsed plan to acquire ual corp. parent of united airlines spurred quick action on the legislation introduced wednesday and approved by the subcommittee on a voice vote yesterday - the bill is expected to be taken up by the public works and transportation committee tomorrow and a floor vote by next week will be urged - the measure drew criticism from the bush administration and a shot from financier donald trump who yesterday withdrew his takeover bid for amr corp. the parent of american airlines - in a letter to subcommittee chairman james d. minn. mr. trump criticized the bill as an explicit effort to thwart his bid for amr and said it contributed to the collapse of the deal - deputy transportation secretary also sent a letter to express the administration 's opposition to the bill in its present form - rep. brushed off mr. trump 's allegations as an excuse for his own deal failing - he also said the fact that the other letter had n't come from transportation secretary samuel skinner indicated there is room in the administration 's position - mr. and other committee members repeatedly stressed that the legislation was n't a response to any particular market situation - but they cited the ual and amr examples as reasons to move quickly to enact this legislation - aides both in the house and senate said the withdrawal of the trump bid for amr is n't likely to efforts to push the legislation - it 's still on the fast track and we still want to do it said one senate aide - the bill is aimed at addressing the concern that an airline might sacrifice costly safety measures to pay off the debt incurred in a leveraged buy-out - currently the transportation secretary does n't have clearly established authority to block mergers but can take the drastic step of the operating certificate of any carrier the official considers - supporters of the legislation view the bill as an effort to add stability and to the process and to preserve the safety and fitness of the industry - in general the bill would give the transportation department a 30-day review period before N N or more of the voting stock of a major u.s. air carrier could be acquired - it also would require the acquiring party to notify the transportation secretary and to provide all information relevant to determining the intent of the acquisition - the bill would allow the secretary to reject a buy-out if sufficient information has n't been provided or if the buy-out is likely to weaken the carrier financially result in a substantial reduction in size of the airline through disposal of assets or give control to a foreign interest - if more information is needed the secretary would have authority to extend the review period N days - all the witnesses both congressmen and industry experts expressed support for the bill in order to prevent from in on airline profits at the expense of safe service - but several committee members some backing mr. trump 's claim that the threat of regulation caused the failure of the ual deal and the stock-market plunge - one of the major concerns expressed by the was that large airlines would be prohibited from themselves of smaller entities and producing independent companies - in a possible prelude to the of talks between boeing co. and striking machinists union members a federal mediator said representatives of the two sides will meet with him tomorrow - it could be a long meeting or it could be a short one said doug hammond the mediator who called the agreement to meet a first step toward a of negotiations - we 're encouraged that talks are scheduled again but beyond that we have made no expression of expectations a boeing spokesman said - the machinists union has rejected a three-year contract offer that would have provided a N N wage increase over the life of the pact plus some bonuses - currently average pay for machinists is $ N an hour boeing said - now in its 13th day the strike has about N machinists and has started to delay delivery of some - with a strike fund of about $ N million the union had said it was prepared for a long strike - after the third week on strike union members will begin receiving $ N a week from the fund - work at boeing continues with supervisors and other personnel the lines - and at the company 's wichita kan. plant about N of the N machinists still are working boeing said - under kansas laws contracts can not require workers to be union members - boeing has declined to say how many employees are working at its giant wash. plant - union officials could n't be reached for comment - dpc acquisition partners a hostile suitor for dataproducts corp. said it intends to launch a tender offer for the computer printer maker 's common stock - dpc a group led by the new york investment firm inc. also said it plans to file preliminary materials with the securities and exchange commission regarding a shareholder solicitation to oust dataproducts ' board - dpc holds a N N stake in dataproducts and made a $ bid for the company in may but dataproducts management considered the $ N million proposal - a dpc spokesman declined to elaborate on the group 's new plan - in american stock exchange composite trading yesterday dataproducts shares jumped N cents to close at $ N - dataproducts which had been seeking a buyer for several months announced a restructuring plan in september and took itself off the auction block - the company 's restructuring includes plans to split into three sectors to phase out domestic printer manufacturing operations and to sell its new england subsidiary - as part of the plan dataproducts announced a pact to sell $ N million of its real estate holdings to properties inc. a unit of canada 's corp - jack davis dataproducts ' president chairman and chief executive officer said the company is at a loss to understand dpc 's intentions - he called today 's announcement and and said the company intends to proceed with its restructuring - share prices plummeted across europe yesterday in response to friday 's new york sell-off but some issues staged a late comeback after wall street opened without another rout - european investors have further reason for optimism today after the u.s. rebound - the frankfurt stock exchange which closed before the new york exchanges opened was the hardest hit of the major european markets with the dax index dropping N N - in london prices plummeted in early trading and were off as much as N N before coming back strong after the new york opening to close down only N N - west german economics minister helmut said in my view the stock market will stabilize relatively quickly - there may be one or other psychological or technical reactions but they are n't based on fundamentals - the economy of west germany and the ec european community is highly stable - paris which has been the center of speculation fever in recent weeks also was hard hit - share prices fell in milan amsterdam zurich madrid and stockholm - prices in brussels where a computer breakdown disrupted trading also tumbled - following is a breakdown of major market activity - frankfurt - one of the sharpest declines came in the financial center of europe 's strongest economy - the dax index of N west german blue chips plunged N N a one-day record out the summer 's gains - the index closed at N down N points - by comparison two years ago on black monday the new index would have dropped N N according to a projection by the exchange - investors may have reacted so strongly to friday 's u.s. stock market loss because they had vivid memories of the frankfurt exchange 's losing N N of its value in the N crash and its wake - this time however many small investors may have been hurt by acting so swiftly - they all went in the wrong direction said andreas an investment adviser for the bank in 's frankfurt branch - he said he told clients to buy selected west german blue chips after they fell by about N N - after the opening was delayed N minutes because of the crush of sell orders frankfurt 's normal trading session was extended N minutes to handle the heavy volume - the beginning was chaotic said nigel a broker for commerzbank ag - it took of an hour before enough prices could be worked out to get a reading on the market - institutional investors and bankers many of whom spent the night before in their offices watching far eastern markets were cautiously optimistic after the mild N N decline in tokyo stock prices - everybody was still confident including most institutional investors - that is why everybody was a little surprised by the storm of sell orders from small private investors said a senior trader for - some big institutions including banks began picking up shares late yesterday but most investors wanted to see what would happen in new york before acting - but even if wall street continues to stabilize analysts here say the latest blow to investor confidence could inhibit a swift recovery for the frankfurt exchange which already was showing signs of weakness after the dax had slipped from a N high of N on sept. N - some of west germany 's chips took some of the biggest hits - a N N drop for ag and dresdner bank ag 's N N decline were especially for their respective boards whose plans for major rights issues in november could now be in jeopardy - dresdner bank last month said it hoped to raise N billion marks $ N million by issuing four million shares at N marks each - yet yesterday 's market dresdner 's share price by N marks to N marks a share leaving little incentive for investors to subscribe to the standing price unless the market quickly - london - headed toward a record drop at midday the london stock market two-thirds of its losses in the wake of new york 's early rally - the financial times-stock exchange N share index closed off N points at N its high for the day after having plunged N points at N p.m - it was big institutions such as union insurance group scottish amicable investment managers and standard life assurance co. that the rally - attracted by low prices and encouraged by new york 's performance they up equities across the board - volume was N million shares more than triple recent levels - paris - late buying gave the paris a after its free fall early in the day - the general index ended down N N at N a drop of N points from friday - there was a volatility in the market that i have never seen before said a partner in brokerage firm - when wall street turned around shortly after the opening there was panic buying in paris - brokers said that as the news spread that wall street was moving up traders who had called to place sell orders changed their line in ordering buys instead - trading was driven primarily by small investors and speculators with large institutions waiting on the sidelines until late in the day - when wall street turned however the big boys entered the market looking for bargains - j.p. morgan & co. swung to a loss in the third quarter while ncnb corp. reported net income more than doubled and security pacific corp. net rose N N - j.p. morgan & co - j.p. morgan as expected posted a $ N billion net loss for the quarter reflecting the new york bank 's decision last month to add $ N billion to reserves for losses on loans to less-developed countries - the reserve addition placed the parent of morgan guaranty trust co. among a few major u.s. banks that have covered nearly all their medium and long-term portfolios to less-developed countries with reserves - the latest quarter 's loss $ N a share - in the year-earlier quarter morgan earned $ N million or $ N a share - george m. analyst at prudential-bache securities inc. called the results mildly disappointing - excluding the $ N billion provision and allowing for the taxes morgan paid earnings were about N cents a share mr. said - in new york stock exchange composite trading yesterday morgan climbed $ N a share to $ N - net interest income sank N N in the quarter to $ N million from $ N million - the interest rate on short-term funds which banks borrow to finance longer-term loans to customers was sharply higher morgan said - morgan received $ N million of interest payments on its medium and long-term brazilian loans had they been interest net interest income would have been $ N million higher in the quarter morgan said - such loans to argentina also remain classified as costing the bank $ N million of interest income in the third period - income from sources other than interest climbed N N to $ N million reflecting higher and other fees and gains on sales of investment securities - these increases were partly offset by lower income the bank said - non-interest expenses grew N N to $ N million - ncnb corp - ncnb corp. 's net income more than doubled in the period largely because of continued strong performance by the bank 's texas operations - the charlotte n.c. company said earnings rose to $ N million or $ N a share from $ N million or N cents a share a year earlier - the latest quarter included a gain of $ N million or N cents a share related to the purchase of the remaining N N of ncnb texas national bank from the federal deposit insurance corp - the strong performance however with an unexpectedly large increase in the size of ncnb 's problem loans particularly in the southeast - in the third quarter nonperforming assets jumped to $ N million or N N of net loans and leases from $ N million or N N in the second quarter - totaled $ N million or N N in the year-ago third quarter - included in the increase in the most recent quarter is a $ N million loan which ncnb said it expects to be fully repaid with no loss early in the fourth quarter - the deterioration in credit quality offset strong loan growth of N N in ncnb 's southeast operations as well as a N N growth in deposits resulting from an aggressive marketing campaign - the higher rates paid on deposits also helped squeeze ncnb 's net interest margin in the southeast to N N from N N a year earlier - in big board composite trading yesterday ncnb jumped $ N a share to $ N - results were released after the market closed - ncnb texas national formed from the of of the failed first corp. of dallas contributed $ N million to ncnb 's bottom line in the third quarter - ncnb said its third-quarter results reflect N N of earnings of the texas operation since aug. N - ncnb raised some $ N billion in new capital during the quarter to complete the ncnb texas purchase and to acquire several small failed thrifts to fill out its regional franchise - last week the banking company said it purchased both freedom savings & loan association tampa fla. and university federal savings association of san antonio texas for $ N million - in the first nine months ncnb 's net income climbed N N to $ N million or $ N a share from $ N million or $ N a share a year earlier - security pacific corp - security pacific 's earnings growth slowed in the third quarter but the los angeles bank holding company was still able to post a N N increase in net income because of robust growth in residential real-estate and consumer loans - net rose to $ N million or $ N a share from $ N million or $ N a share a year earlier - the company said the gain resulted mainly from a $ N million increase in net interest income reflecting a N N increase in real estate loans mainly residential and a N N rise in consumer loans - these loans in effect replaced some assets such as loans which were allowed to decrease - as a result security pacific 's net interest margin fell only N basis points a more mild decrease than some major banks outside california which have been reporting more sluggish earnings - security pacific shares closed at $ N down N cents in big board composite trading - the earnings represent a N N return on assets for security pacific and an N N return on equity - the loan growth offset continuing real-estate loan losses in the depressed arizona market - security pacific reported a N N increase in net credit losses for the quarter to $ N million from $ N million in the year-ago period - nonperforming loans grew slightly to $ N billion at sept. N from $ N billion a year ago - security pacific 's loan-loss provision was down N N or $ N million because it added to its reserve the year before - non-interest income fell N N in the quarter mainly because of an unusual gain a year earlier from the sale of hong kong banking operations - non-interest expense grew only N N in the period - for the nine months net rose N N to $ N million or $ N a share from $ N million or $ N a share a year earlier - lin broadcasting corp. said it wo n't take a position on a revised tender offer by mccaw cellular communications inc. to buy lin and has asked for of the offer - the new offer which seeks N N of the cellular and broadcasting concern is for $ N a share for N million lin shares - mccaw 's revised tender offer would require mccaw to begin an auction process in july N that would buy out remaining holders at a per-share price roughly equivalent to what a third party might then have to pay for all of lin - lin is asking mccaw to clarify its tender offer which challenges an agreement between bellsouth corp. and lin to merge their businesses - bellsouth has notified lin that it would shortly respond to the mccaw proposal in as full and effective a manner as is - the lin board said holders may be by the provision in the mccaw proposal that guarantees private market value after five years for the remaining shares - mccaw has no obligation to purchase and the definition of private market value is uncertain the lin board said - the board added that mccaw would be able to control lin 's operations and could therefore operate lin in a manner which could its private market value and to a in five years - in national over-the-counter trading lin closed at $ N down $ N - a group of institutional investors in telerate inc. said that dow jones & co. 's $ offer for the electronic financial information services company is grossly inadequate - in a letter filed with the securities and exchange commission the group which holds about N million telerate shares or about N N of the shares outstanding said at present none of us believes an offer for less than $ N per share would be fair and some believe that $ N is too low - the letter was dated oct. N - in composite trading on the new york stock exchange telerate shares closed yesterday at $ N down N cents a share - dow jones publisher of the wall street journal has launched an $ or $ N million tender offer to acquire the remaining telerate shares outstanding dow jones owns N N of telerate - telerate has rejected the offer which expires nov. N - the group includes cos. and various affiliates based in boston wells fargo bank san francisco the california public employees retirement system sacramento calif. and t. rowe price associates inc. baltimore - among other issues the group 's letter said it has concerns as to whether dow jones 's offer meets the applicable requirements of procedural fairness - a spokesman for dow jones said he had n't seen the group 's filing but added obviously dow jones with their conclusions - our offer is to buy any and all shares tendered at $ N a share - u.s. trade representative carla hills said the first panel set up under the free trade agreement has ruled that canada 's restrictions on exports of pacific and violate the accord - mrs. hills said the u.s. and canada have until nov. N to resolve the dispute - if a solution is n't reached by then she said the u.s. would have the right to suspend some trade concessions to canada equivalent in value to the losses suffered by u.s. companies in alaska and the pacific northwest - however in canadian trade minister john said the panel accepted the legitimacy of canada 's position on the use of these landing requirements to and manage these important - questioned about the in the u.s. and canadian government views of the panel 's report an aide for mrs. hills said the panel had clearly ruled that the canadian trade restrictions are illegal - the u.s. trade representative declined to put a dollar estimate on the losses resulting from the canadian export restrictions - canada initially had an export prohibition that was replaced by regulations requiring that such fish had to be brought in british columbia by commercial prior to export - this action was defended by the canadian government on conservation grounds - mrs. hills said yesterday that the panel rejected this canadian government argument - we fully expect that canada will comply with the panel 's ruling that the landing requirement also must be ended she said - earlier an international panel set up under the rules of the general agreement on tariffs and trade in geneva determined that the original canadian restrictions violated gatt rules - mrs. hills said the u.s. wo n't accept any delays after nov. N because u.s. firms enter into contracts in the fall to purchase the next season 's catch - she said the canadian restrictions must be removed before such contracts are concluded - idle thought - to spend a idle day when duty calls to pay no to while the precious hours away character is what you need - may - - the guy who throws an his receiver should somehow be advised that we at home can read his - dick - corp. said it completed a restructuring agreement previously agreed to by the federal deposit insurance corp. creditor banks and subordinated debenture holders - the plan would permit the bank holding company to retire its bank and debenture obligations through exchanges of cash and equity - the fdic which in N provided $ N million in assistance to 's bank of oklahoma unit will continue to maintain $ N million in preferred stock in the bank unit - in exchange for the other $ N million the fdic will receive additional warrants it to buy N N of 's common stock outstanding up from the N N option the fdic received under terms of the N capital - in exchange for the $ N million they are owed creditor banks will receive N million shares of common stock and the proceeds from the future sales of four subsidiary banks to private buyers the bank holding company said - also under the agreement debenture holders will get one million shares of common stock in exchange for $ N million in debentures and holders of 's series a preferred stock will receive N shares of common stock for every share of preferred they own the company said - bear stearns 's chief economist lawrence in the sept. N issue of the firm 's global - were it true that a weak currency the way for trade surpluses then presumably argentina would be the center of today 's global economy - bsn corp. said it will begin an offer tomorrow to exchange up to one million of its common shares and all of its $ N million in N N N convertible debentures due N for a package of new debt and common stock warrants - under terms of the offer the sporting goods maker will swap $ N face amount of N N N subordinated notes due N and one warrant for each common share - each warrant allows the holder to buy one bsn share for $ N a share at any time over the next seven years - bsn currently has N million common shares outstanding - bsn also is offering $ N face amount of new notes and N warrants for each $ N face amount of its convertible debt outstanding - the company said it can redeem the warrants at its option for $ N each - the offer is n't contingent on a certain amount of debt or stock being exchanged - bsn said it is making the offer to shrink its capital and increase shareholder value - if all the bondholders and holders of one million common shares accept the offer bsn will increase its debt by $ N million but it also will recognize a $ N million gain from retiring the old debt said michael j. blumenfeld president - we have sufficient cash flow to handle that he said - the offers are scheduled to expire in to late november - merrill lynch & co. 's net income dropped N N while bear stearns cos. posted a N N gain in net and painewebber group inc. 's profit fell but would have risen without a special gain a year ago - at merrill lynch net was $ N million or N cents a share down from $ N million or N cents a share a year ago - total revenue reached $ N billion up N N from $ N billion - the firm 's drop in net reflected weaker revenue in transactions for its own account a decline of N N to $ N million on reduced revenue from trading fixed-income securities - investment banking revenue fell N N to $ N million on fewer equity and municipal - merrill lynch 's commission revenue grew N N however to $ N million on higher share prices and volume and on strong sales of mutual funds - revenue derived from interest and dividends jumped N N to $ N billion - fee revenue grew N N to $ N million - the brokerage also reported a loss of $ N million from the discontinued operations and disposal of its fine homes international limited partnership real-estate subsidiary - bear stearns said net in the first quarter ended sept. N reached $ N million or N cents a share from $ N million or N cents a share in the year-earlier quarter - gross revenue rose N N to $ N million from $ N million - profit from trading for its own account dropped the securities firm said - investment banking revenue climbed N N while commission revenue advanced N N on a stronger retail market - bear stearns is the holding company for bear stearns & co. the investment banking and brokerage firm - in new york stock exchange composite trading yesterday bear stearns shares closed at $ N down N cents - separately painewebber posted net income for the third quarter of $ N million or N cents a share reflecting a broad-based improvement in the company 's core businesses - retail profit surged but the company said it was only a modest to third-quarter results - a year ago net at the new york investment banking firm was $ N million or N cents a share including a special pretax gain of $ N million from the sale of the company 's interest in national car rental systems inc - revenue was $ N million including net interest down slightly from $ N million - in big board composite trading yesterday painewebber closed at $ N up N cents - corp. said it signed an agreement with martin to purchase its headquarters building the columbia center for $ N million - purchase of the structure is subject to execution of a definitive agreement approval by the boards of and its parent company bankamerica corp. and approval by regulators - the market upheaval apparently has n't triggered any cash crunch yet - individual investors investment firms and arbitragers who speculate in the stocks of takeover candidates can suffer liquidity and payment problems when stocks dive those investors often borrow heavily to buy their holdings and use the stocks as collateral for loans - but several large banks said yesterday they detected no signs of unusual demand for credit that would signal such difficulties - we 're seeing nothing out of the ordinary said one official at a top N bank - that 's good news because we all in this water - added another executive at a big bank we were all a little over the weekend trying to forecast what would happen monday but it 's been very quiet - now as for tomorrow hell who knows - what happened friday shows that financial markets are not yet sufficiently to handle another in prices - no with systems and procedures will ever prevent markets from suffering a panic wave of selling - but markets can operate with greater or lesser efficiency - after the N plunge markets agreed that it would be to halt trading whenever panic conditions arose - the new york stock exchange adopted two specific circuit breakers if the dow jones index falls N points in a day the exchange will halt trading for one hour if the decline hits N points the exchange will close for an additional two hours - the rationale is that an of trading will allow investors to reconsider their strategies calm sellers and lead buyers to enter the market at indicated new price levels - it is impossible to know whether that theory is realistic - a temporary of trading may indeed discourage a selling panic from feeding on itself - but there is also the possibility that down markets will intensify fears and cause an even more abrupt slide in prices - what happened friday was the worst of all - the futures exchanges followed their own circuit breakers and shut down at about N p.m. for N minutes after the standard & poor 's N stock index had fallen N points or about N points on the dow jones index - options markets stopped trading in many securities - the new york stock exchange under its own rules remained open - with nowhere else to go sellers and particularly program traders focused all their selling on the new york stock exchange - as liquidity on that market weakened prices fell sharply - had the futures and options markets been open additional liquidity would have been provided and the decline most probably would have been less intense - at N after intense telephone negotiations between the trading markets and washington the futures exchanges reopened - futures trading however was halted altogether at N after the futures markets had dropped an additional N points which is the daily limit for price declines - at this point the options markets also shut down and once more left all sales to be handled by the new york stock exchange - it is time to recognize that the new york stock exchange the futures markets and the options markets though separate have actually become so closely as to constitute one market effectively - traders can vary their strategies and execute their orders in any one of them - it therefore makes no sense for each market to adopt different circuit breakers - to achieve maximum liquidity and minimize price volatility either all markets should be open to trading or none - circuit breakers would not have halted the slide in prices on friday but they probably would have made for less volatile executions - it 's time for the exchanges and the securities and exchange commission to agree on joint conditions for trading or staying open - let 's not have one market shut down for N minutes when the dow declines N points and another shut down for an hour after a decline - the need for last-minute telephone negotiations among market officials will disappear once rules are in place that circuit breakers in all markets - the new circuit breakers if they are to be applied at all will require that futures and options trading continue as long as the new york stock exchange remains open - the rules should be established by agreement of the officials of all affected exchanges acting under the oversight and with the approval of the government regulatory agencies - should the sec and the commodities futures trading commission which with the sec the chicago stock-index markets be unable to agree the issue may have to be resolved by decision of the treasury secretary - in many ways our financial markets are better prepared today to handle a decline than they were two years ago - the new york stock exchange now has the capacity to handle a volume of nearly a billion shares a day - telephone service has been improved for customers trying to reach their brokers and specialists who i believe should stay despite the of some post-crash critics have larger capital positions - of course specialists ' actions alone can never prevent a major crack in stock prices - witness the fact that trading in some stocks closed early friday and opened late monday because of an excess of sell orders - but the task of improving market performance remains - mr. former chief economist of the new york stock exchange is a professor of economics at pace university 's business school in new york - a unified europe labor problems and prospects for u.s. firms - the social worker concerns of the european community 's plan to open its internal borders in N could set the effort off the if not done reasonably says general electric senior vice president frank doyle - u.s. companies wanting to expand in europe face tough pressure from unions in nations such as west germany which play a big consulting role in management decisions he says - corp. and international say unions also wo n't like plant and needed restructuring which means layoffs - many employers have already begun moving to southern countries such as spain and italy where wages are low and unions are weaker demand for trained labor and managers will rise there says - pfizer fluor and ge see big ec N a push for job training and ease in moving and finding workers - a fan was n't the baltimore ' fault - so said a federal judge in a case involving two players for the minor league va. a baltimore farm team - the players were by a during a july N N game with the - like its parent that year was not having a good year the judge said - after the game lost N three in the ninth he noted trouble began - more in the parking lot the players said led to a fight - the fan said he was and kicked by one player and that the other broke his with a baseball bat - the judge dismissed the fan 's suit against the team however ruling the innocent of hiring and not responsible for a fight that was outside the players ' employment - proposals arise for coping with the shortage of nurses - an association of academic health centers report urges nurses from duties that do n't require special skills - it also recommends better retirement and benefits and pay on education experience and nurses ' demanding work schedules - but it opposes an american medical association proposal for creating a registered care as potentially divisive it says the job would an unwanted new doctor 's extension - over a third of N hospitals surveyed by consultant associates use a clinical on performance and education - many also use recruiting bonuses tuition loan repayment or child-care help - some give incentives - systems signs up nurses for paid travel promising annual income up to $ N and free or subsidized housing - treating employees with respect is crucial for managers says consultant group after surveys of a million workers - it 's in their top five work values - fully N N of employees who say their bosses treat them with respect but only a third of those who do n't feel respected say they 're satisfied with where they work - up the digs about N employees of the maryland department of economic and employment development for four months painted walls and floors bought plants windows and and hung pictures at the agency 's baltimore office - the N hours of work will save the state $ N - curbing wage boosts will get high priority again in N collective bargaining a bureau of national affairs survey of N companies with next year indicates - despite warnings N N aim for wage increases of under N N and N N say they 'd try to replace workers if struck or would consider it - temporary workers have good the national association of temporary services says its survey of N such employees shows N N with more than a high-school education and N N with college degrees - about N N have retired from a full-time job while N N were asked to stay on full time - losses rise but they 're often covered by employers - but they search for ways to limit the damage - a third of N companies surveyed by the employee relocation council report a rise in N sales losses over N - about N N reimburse for all or some losses - since N more companies give aid as many real-estate values the council says - rjr nabisco pays up to $ N of losses including improvements - wo n't ensure loss coverage but will prevent a catastrophic loss it has given some employees the full purchase price when values fell from concern over dangers posed by a disposal site - federal express dow chemical ford and national city corp. will buy the home or let the worker sell to an outside firm but usually wo n't cover a loss - since N firms offering house to deter rose to N N of those the council polled from N N - the the national academy of engineering gives two of the semiconductor a $ N achievement award - now that 's letter carriers union president vincent philadelphia charles james of century management tactics - yesterday was in the words of new york stock exchange chairman john j. phelan jr. just your reasonably normal N up day - when it was all over and stocks had staged a huge recovery big board officials were about how well the day had gone - they said the exchange 's trading procedures personnel equipment and links with other exchanges could n't have performed better - we had no operating problems at all mr. phelan said after the market closed - all the things that we set up to slow down the process to let people know that the market was in an extreme position worked extremely well - prices for the N million shares that changed hands during the session were carried on the exchange 's trading tape with barely a delay officials said - while reaching blockbuster yesterday the volume was still well within the N capacity that the exchange has said it can handle daily since up its computers after the october N crash - the so-called circuit breakers devised by the big board and the chicago mercantile exchange to free falls in stock and futures prices were n't triggered yesterday because the markets were higher for most of the day - despite traders ' complaints mr. phelan said the links with the chicago futures market worked as planned in friday 's rout to provide a period - of greater help the big board chairman said was the natural circuit breaker of the weekend that provided a breathing period that brought back to the market - chicken chains by loss of customers - fast-food chicken chains faced with a worsening business slump are struggling to hatch some new marketing strategies - the crest report which tracks consumer purchases says customer traffic at chicken restaurants fell N N in the second quarter while the overall fast-food customer count was down N N - chicken business is off largely because of more competition from convenience food pizza and other fare says a spokesman for the report a publication of group a market research firm in port washington n.y - the loss of more customers is the latest in a string of problems - church 's fried chicken inc. and 's famous fried chicken inc. which have merged are still troubled by restaurant locations - chicken chains also are feeling more pressure from mcdonald 's corp. which introduced its this year and recently tested the sale of individual pieces of chicken - new management at kentucky fried chicken a unit of pepsico inc. has fought back with new medium and large chicken for the lunch crowd - and the chain is testing products that are n't fried such as chicken to try to win consumers - kentucky fried chicken also is testing of chicken which could be a hit with - but some fast-food industry analysts say problems with keeping chicken warm and fresh must be solved first - a kentucky fried chicken spokesman however disputed the notion that the delivery service experienced problems in some markets where testing has been discontinued - he says the test is continuing in chicago columbus ohio and a few other cities - the advertising industry is with rumors that kentucky fried chicken will drop young & rubicam and seek a new ad agency - but the company declines to comment - goldman a painewebber inc. analyst predicts kentucky fried chicken will post an N N drop in N net income - they 've been he says but they 'll have to become more aggressive - reluctant advertisers try spots - call it - pittsburgh consultant david bear is selling a soft approach to clients who want exposure yet ads - his radio spots that offer helpful hints - the only plug for the sponsor is a brief mention at the end of the spot - the messages resemble the business a daily of travel tips developed by mr. bear and sponsored by travel agencies in several major cities - new include burt hill associates a butler pa. architectural firm - its radio series features such spots as evening wear for urban structures and building a place to park - a harder sell says john the firm 's president would from the profession - hospitals have signed up to use the messages to promote and equitable gas co. is considering the format to offer energy tips to consumers - but such spots can be too soft - there 's always a risk of lost messages says john chairman of advertising usa which created similar radio spots for pittsburgh national bank - it 's a question of how much credibility you gain for the possible loss of recognition - retailer sees in environmental push - here 's a retailer that 's getting tough in the push for environmentally safe packaging and products - big bear supermarkets inc. a grocery chain based in san diego plans to display shelf cards and distribute recommending products deemed safe for the environment - the choices will be based on research by the san diego environmental health coalition and will include products like murphy 's oil soap and other - but the chain is quickly the of such - for example it recommends detergent and puts on its environmentally safe list - that does n't procter & gamble co. maker of cascade detergent - a company spokesman questioned the of the list noting that is present in all major - in fact bros. confirms that its brand does contain even though it is n't listed on the label for the version - thomas g. big bear 's executive vice president said the chain is still reviewing its product list to avoid such problems - our intent is to promote the best alternative he says - and it 's important that we be accurate - but in the end customers ' wishes are what will prevail - big bear does n't care for disposable which are n't - yet parents demand them - says mr. we 'll still be forced to sell items we might not agree with - odds and ends - does count at least in the grocery store - a study by 's marketing research shows soap sales climbed N N when bars were neatly on shelves instead of dumped in a wire basket - which celebrity are most - for the third year in a row consumers voted bill cosby first and james second in as spokesmen in tv commercials according to video tests new york - michael j. fox replaced bruce in third place placed fourth for the second time - health and human services secretary louis sullivan has chosen novello to be the next surgeon general bush administration officials said - if she is by president bush and confirmed by the senate dr. novello would succeed c. who rattled liberals and conservatives alike with his outspoken views on a range of health issues - dr. novello an expert on pediatric kidney diseases is deputy director of the national institute of child health and human development - she has also served on several task forces on acquired immune deficiency syndrome - dr. novello 's office said she would n't talk with reporters and it refused to release any information about her - the newsletter medicine & health which first disclosed her selection by dr. sullivan said she is N years old and she studied at the university of puerto rico school of medicine - the continuing series of hud scandals is a predictable result of pork-barrel politics - nevertheless such as the national association of home builders nahb continue to pressure capitol hill for more special-interest spending - kent nahb executive vice president argues that the u.s. faces a housing crisis reduced of homes for first-time buyers increased homelessness and lower apartment construction rates that will be very difficult to solve without expanded federal resources - there 's nothing unusual about business groups pushing for more government spending - but the nahb was created in N out of an organization that made its name fighting a administration proposal to take over all defense housing production - through the years the association has been an active member of the taxpayer 's coalition pushing for such initiatives as the amendment - yet on matters close to home - the hud budget has dropped by more than N N since N argues mr. - we 've taken more than our fair share - i would n't have a problem if other programs had taken a similar hit - but nahb support for subsidies is not related to the current housing crunch over the years the nahb has backed a host of public programs - it once pushed for a national housing production goal set by the federal government and has regularly advanced housing measures - moreover explains one hud official the nahb remains susceptible to internal pressure from members that specialize in subsidized production - the association is pushing an extensive and expensive which would substantially boost spending above the current level of more than $ N billion annually - direct federal subsidies for housing construction have proved expensive in the past and inevitably are to the benefit of developers and lobbyists as demonstrated by the ongoing hud scandal or congressmen - indirect subsidies through the fha for instance are little better - though mr. says expanding fha lending would result in no cost to the government the mere diversion of funds from other parts of the economy and from other forms of housing such as low-income to the single-family home market would result in a major expense - more important housing programs run by hud the va and are in red ink - the fha alone lost $ N billion in fiscal N the government 's equity in the agency essentially its reserve fund fell to minus $ N billion - the federal government has had to pump in $ N billion into the va housing program since N to keep the fund afloat and the va requested an additional $ N million for the fiscal year just ended - all told the federal government already guarantees more than $ N billion of mortgages - in its produced publication where will our children live the nahb does acknowledge that of course the full measure of housing can not be provided by the federal government - it points to the impact of local government regulation particularly and building fees which the price of housing out of the reach of and people - but while the nahb has suggested actions that states and should take to reduce regulatory barriers the association has proposed no activist legislative program comparable to say its detailed request for more federal subsidies to eliminate controls - the association a majority of whose N members build fewer than N units a year is like many other business - explains macdonald of the national taxpayers union it in two - the builders like the subsidies but at the same time they tend to be fiscal conservatives in terms of major issues such as the amendment - unfortunately the organization 's desire for pork tends to override its commitment to overall fiscal responsibility - two years ago when the nahb lobbied for the $ N billion omnibus housing bill the organization basically dropped out of the taxpayers ' coalition says ms. macdonald - as mr. of the nahb acknowledges government is not going to solve the problem - the real key is to have the economy working and interest rates down - more money for hud will increase the deficit and the economy more money to municipalities that are their local housing markets will further them from the effects of their policies - is this what the home builders want - mr. is a institute fellow - see related story and bills to make wishes come true wsj oct. N N - in an attempt to give new momentum to european community plans for a single currency ec government leaders are likely to agree to set a date for starting formal talks on the ec 's founding treaty of rome - according to diplomatic sources in brussels most ec leaders agree that talks should begin in the second half of N and will make a declaration on that during a summit meeting in france on dec. N and N - the only strong opposition to changing the ec treaty comes from british prime minister margaret thatcher who is opposed to creating a single ec currency - but the process of the conference does n't require - setting a date to start treaty negotiations has no legal significance in itself but could be viewed as an important psychological push - french president mitterrand fought to set a date for the conference during the ec summit in madrid last june but the move was because of opposition by mrs. thatcher and west german chancellor helmut kohl - diplomatic sources said mr. kohl may now agree to set a date for the conference to make it clear that west germany is still committed to ec unity - the latest in the equities markets me of the joke t. boone pickens tells about the guy who was run over by the parade - when asked what went wrong the unfortunate victim replied it was a combination of things - and so it was on gray friday - the grand of this parade would appear to have been excess leverage - even if that is so however it 's probably the case that no barriers should have been to stop the before the end of the rout e - the began friday afternoon when word spread that the ual buy-out was - although the expects to patch together a substitute offer consisting of less cash the failure to get cash from japanese and american banks confirmed a growing fear among arbitragers that the of takeover deals is ending - lots of other made up the parade of course notably a surprisingly large increase in producer prices federal reserve and the bush administration 's temporary defeat in trying to lower the capital-gains tax - as usual few favorable reviews were heard for that band of program traders although most serious studies suggest they only play the music that others write - what really spooked the along wall street however was the sudden concern that whatever the reason the pool of debt capital is up - gray friday reflects a panic mainly by the takeover arbitragers rather than the small investor as their highly investments in the deal stocks are by the unexpected up of the for deal financing - deal stocks led the market down as they absorbed the heaviest losses - ual which triggered the slide opened monday at $ N down about N N from thursday 's close - amr opened monday at $ N down nearly N N from thursday 's close - both took further hits yesterday - hilton lost N N on friday paramount lost almost N N - a careful look reveals that where deal financing has been secured the target 's stock price was not affected on friday - the multibillion-dollar prospects where the bidder must line up a consortium of banks issue billions in high-yield debt were where the damage was concentrated - the market for so-called junk bonds has been setting the stage for friday 's dramatic march for several weeks - the growing financial difficulties of recent restructurings or takeovers such as resorts international integrated resources and campeau 's retailing empire have cast a pall over the entire market for high-yield securities - investors have reacted by ignoring recent efforts to float junk bonds by ohio and by forcing ramada to postpone indefinitely its planned junk-bond sale and restructuring - as a result high-yield mutual funds have declined across the board and the many firms planning to sell $ N billion in junk bonds before year-end are experiencing anxious times - these are all market excesses putting aside the boosts that the tax code gives to debt over equity and what we 've seen is the market them in - of course washington had n't been silent in the days leading up to the debacle and its tendency to in the leverage equation remains a troublesome prospect but those preliminary steps should n't us from the basic market that was at work on friday - if it is correct to find that concerns over corporate debt and lbos caused gray friday what are the implications for policy makers - after all the stock market 's response to the collapse of the ual deal might be taken to confirm the direction of regulators - is this a case where private markets are of washington 's of wall street - absolutely not - to the extent that friday 's sell-off reflected a sudden of the excesses of leverage the message is that wall street and the private markets are fully capable of imposing the appropriate incentives and sanctions on corporate behavior - the national economic interests are much better served allowing the private interests of bankers and investors be the ultimate judges of the investment quality of various lbo deals and leveraged restructurings - the recent difficulties in the junk-bond markets and the of bank capital for recent deals the wisdom of letting the free markets operate - if takeover premiums become excessive if lbo become too aggressive then the private market will recognize these problems more quickly and accurately than will policy makers and the markets will move with speed to impose appropriate sanctions - yes the broader exchanges got caught up in the but they rode the tiger up all year - not surprisingly he sometimes - the arbitragers and takeover got killed on gray friday while the besieged managers of prospective targets cheered - if you identify with the besieged managers you must concede that and effective relief from the excesses of the takeover market is more likely to come from the marketplace than from washington - if you side with the arbitragers and raiders you clearly have more to fear from private investors than from regulators although the delaware courts should never be underestimated - the truth is washington understands politics better than economics - although the average citizen is probably not too much from washington 's war against wall street regarding excessive financial actual legislation would probably impose considerable harm - any such attempt to good debt from bad debt or to draw the line at a particular industry such as the airlines is likely to blunt the spur that the proper amount of leverage provides both to equity markets and economic efficiency in general - far better for policy makers to concentrate on the war against drugs panama and the deficit all of them that seem never to end - mr. former top economist at the securities and exchange commission teaches at the university of rochester 's simon business school - tokyo share prices rebounded tuesday morning with the nikkei index of N selected stocks rising N points to close the morning session at N - the index slid N points or N N on monday - in the first N minutes of tuesday 's trading the nikkei index soared N points to N - by N a.m. tokyo time the index was up N points to N as investors hailed new york 's overnight rally - monday 's slide came in a relatively calm session that did n't provide much direction for other markets - shares also closed sharply lower across europe particularly in frankfurt although london and a few other markets recovered some ground after stocks began to rebound in new york - other asian and pacific markets had sharper losses than tokyo but the selling wave stopped short of another market crash - all eyes were on tokyo at the opening because it was the first major market to trade since friday 's 190.58-point plunge on wall street - but rather than set the tone for other markets japan 's major institutional investors chose to remain on the sidelines - still despite the sudden of stock-market turbulence managers of japanese investment funds said they were n't planning to unload u.s. or european equities - we did n't trade much today as our policy now is to wait and see said a fund manager at life insurance co - we would like to wait and see until trading goes around through europe and new york - the institutions appeared confident that japanese regulators would step in to ensure orderly trading if necessary and there was considerable speculation during the day that the finance ministry was working behind the scenes to do just that - but in the absence of trading its presence was never felt - at the close the nikkei average of N stocks stood at N down N points or N N - the broader tokyo stock price index sank N or N N to N - the day 's decline was generally in line with analysts ' weekend predictions - declining issues advancers N - but volume was thin at N million shares compared with N million friday - the market opened sharply lower with the nikkei average down nearly N after N minutes - a midmorning rebound brought it back to show a gain of about N at the end of the morning session but the rally failed in the afternoon and the market closed near the day 's low - the smaller stocks in the tokyo market 's second section also posted their biggest decline of the year - the tokyo stock exchange index for the second section fell N or N N to N - many investors trying to outperform the market 's major indexes have to these small issues in recent weeks - japanese investors and traders expressed relief that the tokyo market did n't fall more sharply - but its performance did bear some to events of two years ago during the october N global stock market crash - on oct. N N the friday before the black monday crash the new york market dropped N N and tokyo followed on monday with a N N drop - this time wall street 's plunge of N N friday was followed by yesterday 's N N loss in tokyo - two years ago tokyo 's biggest fall came the day after new york 's N N black monday plunge when the nikkei average fell N N - thus market participants yesterday were looking ahead nervously to wall street 's opening - but in new york yesterday the dow jones industrial average surged N to close at N on heavy volume of N shares although declining issues still outnumbered advancing ones on the broad market - a director at yamaichi investment trust & management co. called yesterday 's session a good scenario for japan - now we are looking for the time to place buy orders he said - for us institutional investors the chance for buying has come - general manager of the investment research department at trust & banking co. also was optimistic - he described friday 's plunge in the u.s. as a fleeting event resulting in part from excessive merger and acquisition activity - unless there is a panic this is the best time to buy as was the case two years ago he said - those shares which had posted gains on speculation were dashed with cold water but as far as major stocks are concerned there is n't much impact - other fund managers were similarly - we have no plans to adjust our asset allocation in foreign equities said chief portfolio manager in the pension fund management department at trust & banking co - he said friday 's wall street decline was well within the range of volatility that trust plans for when it charts its overseas investment strategy - among other asian and pacific markets malaysia and singapore had the biggest losses with the kuala lumpur composite index in malaysia falling N N and singapore 's times industrial index down N N - major indexes declined more than N N in australia and new zealand and N N in hong kong - manila seoul taipei and escaped with slightly smaller losses - brokers and fund managers said the region 's markets were reacting to friday 's wall street plunge even though that decline was due to local factors such as failed corporate buy-outs and a deteriorating junk-bond market - it 's pure psychology said william an account executive for drexel burnham lambert ltd. in hong kong - markets in this region are n't so geared to leveraged buy-outs and their economies generally are in good shape but there 's no doubt that asia is still following america 's lead - several analysts said malaysia and singapore had the biggest losses because they are relatively open to rapid cash flows - hong kong is the region 's next most open market but many foreign investors have been staying away from it since it plunged in june amid political turmoil in china - singapore took the hit because when people want to get out they tend to go where the liquidity is said elizabeth hambrecht a regional analyst with baring securities hong kong ltd - she pointed out that even after monday 's N N decline the times index is up N N this year so investors who out generally did so profitably - similarly kuala lumpur 's composite index yesterday ended N N above its N close - in hong kong the hang seng index fell N to finish at N - trading was heavy at about one billion shares compared with N million friday - but the session was orderly in contrast to the market 's four-day after the N crash - richard a director at hong baring international fund managers ltd. said the market probably has n't hit bottom yet but is close - if new york does n't collapse i see maybe another N N on the downside not counting the risk of bad news out of china he said - in australia sydney 's all index closed at N down N N its biggest drop since october N - but volume rose only to N million shares from N million friday - an analyst at brokerage firm & young ltd. described the market 's performance as as investors fled to australian stocks and entrepreneurial companies they perceived as having any takeover premium built into the price - london 's financial times-stock exchange 100-share index the most closely watched market barometer ended at its intraday high of N down N or N N - at its low shortly before wall street opened it was off more than N points - the financial times 30-share index closed N points lower at N - volume more than doubled to N million shares from N million friday - prices on the frankfurt stock exchange tumbled in heavy trading - the decline in the german stock index of N points or N N to N was the frankfurt market 's fall ever - retail investors dumped holdings on a massive scale pushing some blue-chip shares down as much as N N - analysts cited memories of two years ago when many small investors held on to their shares after the october crash but the west german market continued to decline for the next three months - here are price trends on the world 's major stock markets as calculated by morgan stanley capital international perspective geneva - to make them directly comparable each index is based on the close of N equaling N - the percentage change is since year-end - frank lloyd wright is reported to have said once that if you the world on its side everything loose would end up in california - we 've always thought that mr. wright underestimated california 's but maybe the state 's are starting to the forces that made it such a significant place - what else is one to make of the initiative just proposed by several major environmental groups and organized by the state 's attorney general - if passed by the voters the recently announced initiative would phase out major pesticides reduce carbon dioxide emissions by N N ban new offshore drilling ban chemicals thought to the ozone layer and create a new state environmental officer armed with a $ N million budget to sue any firm or agency he thinks is being too - the initiative is based largely on the of the green lobby the sierra club the league of conservation voters the natural resources defense council the national campaign and the citizens for a better environment - the environmental defense fund is having nothing to do with this one - not only californians but all americans would pay if this thing passed - the initiative bars the sale of any crops in california that do n't meet the initiative 's standards - kansas wheat farmers and florida fruit growers would have to adjust or give up the california market - in other words california is to take control of the nation 's farm policy - as usual the green lobby 's proposal is from scientific reality - consider the provision - the proposed initiative would mandate a reduction of carbon dioxide of N N - even if one buys into the whole greenhouse theory it is that reductions in a single state could have any impact on what is billed as a global problem - but if rational science and economics have nothing to do with the new environment initiative what is going on - the first place to look under these circumstances is at the ways in which the sponsors themselves will benefit - the key here is the of state attorney general john van de - he 's running for governor - mr. van de is the one who collected the plans from the various radical environmental groups and them into a single initiative to be placed on the ballot for election on nov. N N - that 's also the day of the gubernatorial election - the initiative seems to have been to include all the hot issues that set off the wealthy hollywood who money - and it allows mr. van de to get around campaign spending limits - he can spend the legal maximum for his campaign all the spending for the van de initiative on which there are no limits is - this initiative is being labeled the big green but maybe it should be called the big - the republican candidate sen. pete wilson is playing the initiative game too his own crime initiative - while it is possible that the big green initiative will be ruled unconstitutional it is of course that in modern california it could slide through - this is the state that recently passed the N initiative - if this new proposal ever does become law the green lobby will benefit directly - the initiative creates a free floating state environmental officer to sue companies or government agencies that do things he does n't like - that means the and such groups no longer would have to spend as much money on litigation taxpayers would bear the cost - mr. van de and his allies may be hoping that the environment is such a mom and issue among certain segments of california 's population now that almost any collection of nonsense can pass under its - of course the state 's liberals are not yet a nation themselves - george bush for example may decide that he does n't want to be the president who lost control of interstate commerce to an attorney general from california - and some other segments of california 's political and media culture may yet start to point out that the initiative would impose significant costs on the state 's less affluent citizens in the form of higher food prices and lost jobs - this initiative will help california define itself for the future either as a state still to economic and scientific reality or as one being led to wherever its activists want to take it - first there was a death watch - then - spurred by waves of large-scale buying in blue-chip stocks the dow jones industrial average rallied yesterday and erased about a half of friday 's 190.58-point plunge gaining N to N - it was the advance for the average of N blue chips on new york stock exchange volume of N shares the highest since the days after the N crash - while the advance cheered investors who feared a crash would occur yesterday it was strictly a rally fed by huge buying by bargain-hunting institutions and program traders - a troubling sign declining stocks on the big board outnumbered advancers N to N and the over-the-counter market that includes many smaller stocks suffered aftershocks of friday 's late big board plunge - the nasdaq otc index closed down N to N - meanwhile in a divergence in two of the market 's most important indicators the dow industrials ' sister average the dow jones transportation average tumbled N to N its decline next to the fall during the N crash - plunged on takeover disappointments in two airline stocks ual and amr which each fell more than N N when they reopened for trading yesterday after being suspended friday afternoon - ual the takeover stock at the center of friday 's 190.58-point market plunge fell N N to N N on nearly N million shares - overall this is a rally but it 's very selective said arthur jr. a veteran painewebber inc. trader at the big board - everyone was a little concerned about the general of the rally and failure of the otc market to get into plus territory - it 's just a strange feeling - i do n't think anyone left the place - the rally gave at least for now to the declaration of big board chairman john j. phelan jr. that friday 's market debacle was an condition and not a disaster - but to traders it looked like disaster on the N a.m. opening bell - the dow jones industrial average opened down N shortly after N - but most of the N blue-chip stocks in the average including eastman kodak and general motors could n't trade because of the heavy backlog of sell orders left over from friday 's rout - at N procter & gamble one of the most important dow of late opened down N N to N - the dow dropped to a quick loss and to many traders it looked as if stocks were headed for yet another big tumble - more stocks opened over the half hour as the N big board specialist firms in charge of keeping the market orderly to find buy orders from major brokerage firms to match the selling flood - then to make matters worse computerized sell programs kicked in stocks into losses - there was heavy stock-index arbitrage as traders sold big baskets of stock and bought stock-index futures to profit from the price discrepancies between the two markets - this was a from friday when standard & poor 's 500-stock index futures had closed at a sharp discount to stocks - the of the program selling dashed any hopes that some of the big program trading firms would hold off until the market stabilized - they did n't - the dow accelerated its slide losing N in the first N minutes of trading - with program traders seemingly in charge buyers backed away from the market and watched stocks fall - then at N the dow suddenly started to rebound and when it shot upward it did so even faster than the fall - and this time it was n't just the program traders who were responsible - all the selling had pushed stocks to such cheap values that big investment banks and major money management firms started buying stocks heavily - the program traders were in there too of course - but according to one trader the programmers did n't look as dominant on the upside as on the downside because there was also a lot of bargain-hunting by institutions - m. director of the new jersey division of investment which oversees $ N billion in investments said the first thing we did was to double our orders yesterday morning - with the market down like this we 'll probably take another $ N million and put it in the market - trading in walt disney co. particularly caught traders ' eyes - according to big board officials disney had one of the biggest imbalances on friday it was one of the seven stocks that could n't finish trading that day - the stock opened late at N N down N N - but then it shot upward N N as goldman sachs & co. stepped in and bought traders said - however disney specialist robert said i would be surprised if goldman represented N N of the opening volume - around wall street trading desks were relieved that they could at least play the market yesterday in contrast to friday 's gridlock - at donaldson lufkin & jenrette inc. head equity trader said i think the opening was - it was orderly - we put some orders together - there was n't a lot of panic selling either domestically or internationally - not like friday where they just took the market apart - still the market had n't yet crossed into positive territory and traders were - but in another dramatic burst the dow tacked on N points in five minutes and at N the index showed a gain of N - on the big board floor and on trading desks traders their approval - peck a trader in shearson lehman hutton inc. 's otc department i tell you this market acts healthy - around him scores of traders seemed to get a burst of energy their boss broke out bottles of water to cool them off - among big board specialists the cry was pull your offers meaning that specialists soon expected to get higher prices for their shares - it was on the upside said one big board specialist - but not everybody was making money - the on the chicago board options exchange the nation 's major options market was heavy after the trading in s&p N stock-index options was halted friday - many market makers in the s&p N index options contract had bullish positions friday and when the shutdown came they were frozen with huge losses - over the weekend clearing firms told the chicago market makers to get out of their positions at any cost monday morning - they were absolutely killed said one chicago-based options trader - some traders said that the closely watched major market index whose N stocks mimic the dow industrials did n't lead yesterday 's big rally - james a partner at specialist & said the difference between today and two years ago terrible tuesday oct. N N is that then we needed a to go into the major market index spend $ N million and get the program rally started - this time institutions saw the programs coming and backed away and backed away - then when the market was at a technical level to buy they came in with a - however according to one analyst the timing of major market index futures buying just before the turnaround was similar to that of terrible tuesday - futures were pulling the stock market higher said donald head of stock-index futures research at prudential-bache securities inc - although the big board 's specialist firms struggled through another highly volatile trading session their performance yesterday was better than during friday 's chaos according to traders and brokers who work with them - specialists were criticized for their inability to maintain orderly markets during the friday plunge - but yesterday even with halts in such major blue-chip stocks as merck we expected the halts and it was n't too bad said donaldson 's mr. who had been critical of the specialists ' performance on friday - according to a big board official while many stocks opened late there were subsequent trading halts in only three issues amr merck and energy - merck is one of the most important stocks in the major market index - no sector of the market has been during the past two days ' gyrations - yet from the dow industrials ' high on oct. N through friday 's plunge relatively good performances have been turned in by real-estate utilities precious metals and life insurance stocks - and yesterday the top performing industry group was oil field equipment issues - for example jumped N N to N rose N N to N N and baker hughes rose N N to N - because of the ual and amr airlines were the weakest sector of the market yesterday - philip morris was the big board 's most active issue rising N N to N N on nearly eight million shares - among other major issues coca-cola co. closed up N at N N on N million shares and american telephone & telegraph rose N N to N on nearly N million shares - shares of international business machines which reported earnings yesterday finished at N up N after slipping below N during friday 's session for the first time in five years - shares of three brokerage firms rose after they reported earnings - merrill lynch added N N to N painewebber rose N to N N and bear stearns rose N to N N - federal national mortgage association a recently hot stock climbed N to N on nearly N million shares - at a news conference after the close of trading yesterday the big board 's mr. phelan and other exchange officials praised the performance of their computers and personnel - mr. phelan said that program trading strategies were n't responsible for triggering friday 's decline despite a jump in the use of the computer-driven strategies in recent months - some N million of the more than N million shares traded in the final N minutes of friday 's session when the plunge in stock prices was concentrated were he said - program trades make up N N of the exchange 's volume on an average day but despite the increase friday it was certainly not something you would say the market decline mr. phelan said - mr. phelan expressed relief that the market rebounded yesterday - obviously every time we get this kind of reaction it 's going to make everybody nervous including me he said - he said that exchange officials had conversations with wall street firms throughout the weekend and that all the participants behaved very very today - meanwhile peter dapuzzo shearson 's head of retail equity trading praised institutional investors in the otc market who were heavy buyers of the nasdaq 's biggest technology issues yesterday amid a flood of selling by other investors - the institutions ca n't be criticized for their behavior mr. dapuzzo said in an interview - it was the opposite of what happened on oct. N - they used their judgment - they did n't panic during the first round of selling this morning - instead they bought on weakness and sold into the strength which kept the market orderly - maybe they learned from experience - mr. phelan said the performance of specialists during friday 's plunge was because out of N big board common stocks traded during the day only seven were closed and were n't reopened before the close - they did an excellent job mr. phelan said of the specialists - wall street traders on friday had complained about the trading - james a. white and contributed to this article - west germany 's green party joined its ideological and the institute in the legal battle to ground the atlantis shuttle and its galileo probe to jupiter - the greens wanted a washington federal appeals court to block today 's scheduled long enough for them to ask the world court to order a permanent cancellation of the $ N billion flight - a appeals panel yesterday refused to comply though liberal judge pat went out of her way to deny that this was a case - of course it was - nasa should now sue for fines against all three foreign and domestic for bringing this case - a house-senate conference approved a permanent smoking ban on all domestic airline routes within the continental u.s. and on all flights of six hours or less to alaska and hawaii - the restrictions would cover all but a small percentage of domestic air traffic and represent a major expansion of the current smoking ban on flights of two hours or less - the exemption allowed on longer flights to alaska and hawaii appears to be largely a for the traditionally powerful tobacco industry which has found itself increasingly isolated in the face of public pressure in recent years - by a N margin house negotiators initially rejected last night a senate provision covering all domestic flights - but the compromise was soon agreed to in subsequent discussions - as a practical matter flights from the west coast to hawaii would be covered as they are under the time limit but the language would exempt longer routes beginning for example in chicago or on the east coast - within the senate the ban has had aggressive support from sen. frank d. n.j. who has used his position as a senate appropriations subcommittee chairman to votes for the initiative - the measure is attached to the more than $ N billion fiscal N transportation bill within mr. 's jurisdiction and the final compromise is with more than $ N million in road projects earmarked by members as well as funds sought by major airports including denver - from the outset the tobacco industry has been uncertain as to what strategy to follow - but the industry retains support in the house leadership through the influence of grower states such as north carolina - majority whip william gray owes a political debt to southern agriculture lawmakers for his rise in the house and the philadelphia democrat used his position in the conference to salvage the exemption from a total ban - although the smoking provision has attracted the most public interest the underlying bill was the subject of lobbying because of its impact on air transportation and the more mundane but politically important projects of members - in a stark lesson in the power of the appropriations committees the house deliberately killed a handful of projects backed by lawmakers in florida illinois and pennsylvania who had voted against the panel leadership on the house floor - anybody can vote as they want said rep. william lehman d. fla. head of the house conferees - but if you make a request you should support the committee - within the federal aviation administration the final bill promises to increase spending for facilities and equipment by more than N N from last year and total operations would rise to $ N billion a N N boost - the facilities account includes $ N million for denver 's ambitious new airport and the competition for these funds created shifting alliances between urban lawmakers representing established airports in philadelphia and michigan and the major carriers to denver united and continental - leery of the costs and critics say competition the airlines have sought to gain leverage over the city of denver - texas air corp. which owns continental and the air transport association were prominent in the lobbying - the industry sought to impose conditions that would have delayed funds for the project until denver and the airlines had agreed to leases for N N of the gates - but this was rejected in favor of much language the transportation department to review the costs of the first phase expected to cost about $ N billion - though smaller in total dollars the conference agreed to preserve an estimated $ N million in controversial subsidies to carriers serving rural or isolated airports - the sum is more than double what the house had approved for the program but the list of qualified airports would be cut by N under new distance requirements and limits on the level of subsidy - congress previously cut six airports this year - the impact of the changes is to eliminate many of the most excessive cases where the government has been paying more than $ N for each passenger in subsidies - among rail and highway accounts the agreement provides $ N million for including $ N million for capital improvements - and grants for mass transit would be effectively frozen at $ N billion or $ N million more than last fiscal year - enjoying several blockbuster movie hits including batman los angeles-based guber-peters entertainment co. reported earnings for the first quarter ended aug. N of $ N million or N cents a share compared with a year-earlier loss - sony corp. which has offered to acquire the company is seeking to free its top executives peter guber and jon peters from an exclusive agreement with time warner inc. 's warner communications inc. so they can run columbia pictures entertainment inc - sony two weeks ago agreed to acquire columbia for $ N billion or $ N a share - warner sued sony and guber-peters late last week sony and guber-peters have charging warner with attempting to interfere in sony 's acquisition of the two companies - guber-peters 's net income in the latest quarter compared with a net loss of $ N million or N cents a share in the year-earlier period - the company said revenue rose N N to $ N million from $ N million reflecting the success of its movies in the and as well as the batman - a group including jon m. of salt lake city said it boosted its stake in chemical corp. to N N of the the common shares outstanding - as previously reported holdings corp. owned by jon m. and other members of his family proposed that corp. an affiliate of holdings acquire in a friendly transaction for $ in cash or $ N million - in a filing with the securities and exchange commission the group said it controls N common shares including N shares bought from aug. N to oct. N for $ N to $ N per share - officials at based in pittsburgh declined comment - congress has been critical of the bush administration for not sending enough aid to poland so it is getting ready to send its own version of a care package - last month the senate voted to send a delegation of congressional staffers to poland to assist its legislature the in democratic procedures - senator pete calls this effort the first gift of democracy - the poles might do better to view it as a horse - it is the vast shadow government of N congressional staffers that helps create such legislative as the N page reconciliation bill that claimed to be the budget of the united states - maybe after the staffers explain their work to the poles they 'd be willing to come back and do the same for the american people - plc a financially troubled irish maker of fine crystal and china reported that its pretax loss for the first six months widened to N million irish punts $ N million from N million irish punts a year earlier - the results for the half were worse than market expectations which suggested an interim loss of around N million irish punts - in a sharply weaker london market yesterday shares were down N pence at N pence N cents - the company reported a loss after taxation and minority interests of N million irish punts compared with a loss of N million irish punts for the year-earlier period - there were n't any extraordinary items - sales for the total group rose N N to N million irish punts compared with N million irish punts a year ago - has decided against paying an interim dividend - said the appointment of a new management team and the signing of a comprehensive labor agreement are expected to enhance the company 's long-term prospects - the sudden flight to quality that triggered friday 's explosive rally was reversed yesterday in a flight from quality rout - the setback in which treasury bond prices plummeted reflected a rebound in the stock market and profit-taking - it was a pretty wild day - our markets were closely tied to the stock market said joel manager of trading at smith barney harris upham & co - friday 's flight to quality was no longer needed once the stock market found its he said - some fixed-income investors had expected a further drop in stock prices after the nearly drop in the dow jones industrial average on friday - that caused investors to stocks and buy high-quality treasury bonds which are safer than other types of securities - but when stocks began to climb instead prices of treasury bonds declined - contributing to the selling pressure were by several investment firms advising clients to boost their stock holdings and reduce the size of their cash or bond portfolios - among the firms were merrill lynch & co. and dean witter reynolds inc - the bond market seemed to ignore evidence that the federal reserve eased credit conditions slightly by allowing the federal funds rate to as low as N N N - the closely watched rate on federal funds or overnight loans between banks slid to about N N N last week down from its perceived target level of about N N - the rate is considered an early signal of changes in fed policy - traders said yesterday 's modest easing did n't stir much enthusiasm because it had been widely expected - in fact some economists contend that the latest easing started last week - others note that some investors were disappointed because they had expected a more aggressive easing - the treasury 's benchmark 30-year bond ended about N N points lower or down about $ N for each $ N face amount - the reversal was even more evident among treasury securities - after treasury bill rates plummeted as much as N percentage point on friday they gave back of that amount yesterday - the bond-equivalent yield on three-month treasury bills for example was quoted late yesterday at N N compared with N N friday - investment-grade corporate bonds mortgage-backed securities and municipal bonds also fell - but prices of junk bonds which were battered friday in near standstill trading rebounded to post small gains after a volatile trading session - junk bonds opened as much as four points lower but staged a modest comeback as stock prices firmed - some traders said the high-yield market was helped by active institutional buying - in particular they said firms such as first boston corp. and drexel burnham lambert inc. began making a market in junk issues early in the session when prices hit severely depressed levels - i think the willingness of securities companies to make markets for high-yield issues improved the sentiment for junk bonds said john an economist at moody 's investors service inc - u.s. treasury bonds were higher in overnight trading in japan which opened at about N p.m. edt - the benchmark 30-year bond for example rose one point in early japanese trading in reaction to a quick drop in the tokyo stock market - but as japanese stocks rebounded treasurys retreated and ended just modestly higher - many u.s. trading operations wanting to keep a eye on japanese trading as an indication of where u.s. trading would begin were fully during the tokyo trading session - most of the action was during the night session said michael moore trading manager at continental bank - jay who often trades overnight for capital insight inc. beverly hills calif. said trading in tokyo was very active but highly volatile - we went down N point in N minutes right before lunch then after lunch we went up N point in N minutes he said - in tokyo trading is halted during - tokyo 's market turned out to be a bad bellwether for u.s. trading - when the market opened here bonds prices fell as the stock market regained strength - the bond market 's focus on stock activity was so strong yesterday that it today 's slate of economic data which includes the government 's report on august u.s. merchandise trade and september industrial production - industrial production is expected to have declined N N according to a consensus of economists surveyed by dow jones capital markets report - the august trade deficit is expected to have widened to $ N billion from $ N billion in july - a widening of that magnitude said one new york trader is not a favorable number - it could do damage to us - meanwhile agency supply is expected to weigh heavily on the market today when the federal home loan bank prices a $ N billion offering of one-year three-year five-year and 10-year maturities - tomorrow the resolution funding corp. will provide details of its first bond issue which is expected to total between $ N billion and $ N billion and carry a maturity greater than N years - resolution funding is a division of resolution trust corp. the new federal agency created to bail out the nation 's troubled thrifts - and this week the tennessee valley authority plans to price a $ N billion offering its first public debt borrowing in N years - there 's lots of supply the new york trader said - we have a couple or three tough weeks coming - treasury securities - prices of treasury bonds tumbled in moderate to active trading - the benchmark 30-year treasury bond was quoted late at a price of N N compared with a closing price of N N friday - the yield on the benchmark issue rose to N N from N N - the latest 10-year notes were quoted late at N N for a yield of N N compared with N N to yield N N - short-term interest rates fell yesterday at the government 's weekly treasury bill auction - the average discount rate on new three-month treasury bills was N N the lowest since the average of N N at the auction on oct. N N - the average discount rate was N N on new six-month bills the lowest since the average of N N at the auction on july N N - here are auction details - rates are determined by the difference between the purchase price and face value - thus higher bidding narrows the investor 's return while lower bidding widens it - the percentage rates are calculated on a year while the yield is based on a year - both issues are dated oct. N - the 13-week bills mature jan. N N and the 26-week bills mature april N N - corporate issues - investment-grade corporate bonds ended one to N N point lower - there were no new issues - foreign bonds - foreign bonds surged as the dollar weakened against most major currencies - among benchmark issues japan 's no. N N N bond due N ended on brokers screens at N up N point - the yield was N N - west germany 's N N N issue due june N ended at N up N point to yield N N - britain 's N N N bond due N ended N N higher at N N to yield N N while the N N N notes due N rose N to N N to yield N N - mortgage-backed securities - mortgage securities gave up most of friday 's gains as active issues ended N to N point lower - dealers said morning activity was hectic as prices dropped in response to gains in the stock market and losses in treasury securities but trading slowed to moderate levels in the afternoon - government national mortgage association N N securities for november delivery were quoted late yesterday at N N down N from friday N N N securities were down N at N N and N N securities were at N N off N - federal home loan mortgage corp. N N securities were at N N down N - on friday mortgage issues gained as much as N N - late yesterday ginnie mae N N securities were yielding N N to a 12-year average life assumption as the spread above the treasury 10-year note narrowed N percentage point to N - traders said there were some busy dealings in freddie mac and federal national mortgage association securities because underwriters from last week 's heavy slate of real estate mortgage investment issues moved to gather collateral for new deals - offsetting the purchases were continued heavy sales by mortgage which are producing increased amounts of fixed-rate mortgage-backed issues with lower rates - there was no new-issue activity in the derivative market - municipals - rebounding stocks and weaker treasury prices drove municipal bonds N to N point lower in late dealings - the session losses left municipal dollar bonds close to where they were before the 190.58-point drop in the dow jones industrial average friday prompted a capital markets rally - trading was hectic during the morning with players trying to gauge whether equities would continue friday 's free fall or stabilize after a brief spot of weakness - started the session flat to a touch higher on anticipation of further stock market erosion but bond prices rapidly turned south as it became more clear that a repeat of the october N crash was n't at hand - professionals dominated municipal trading throughout the session - traders said retail investors seemed to be the sidelines until a measure of volatility is out of the market - new jersey turnpike authority 's N N issue of N was off N at N N bid yielding N N up N percentage point from late friday - florida board of education 's N N N issue of N was N point weaker at N N bid - the N N N issue of bridge and tunnel authority of new york due N was off N at N N bid - and county va. water authority 's N N N issue of N was down N at N N bid - serial bond yields were up about N percentage point - corp. kansas city mo. said it 's weighing strategic alternatives for its business men 's assurance co. unit and is possible buyers of the life and health insurance operation - a spokesman said runaway medical costs have made health insurance a significant challenge and margins also have been by changes in the mix of life-insurance products consumers now demand - the business men 's assurance unit represented about $ N million of the company 's $ N million in N revenue and the unit 's operating income was about $ N million said the spokesman - 's investment banker alex brown & sons inc. has been authorized to contact possible buyers for the unit - transportation ltd. said it raised its stake in ltd. of to N N from N N - a spokesman for declined to disclose the price the toronto transportation and waste services concern paid for the additional shares which he said were acquired over the last couple of weeks - the spokesman said would n't increase its stake in beyond N N without a great deal of thought because of british takeover regulations that require a company acquiring more than N N to extend an offer to the rest of the company 's shareholders - a security services and auctions company trades on london 's stock exchange - is by canadian pacific ltd. a montreal transportation resources and industrial holding concern - co. a japanese maker of video games electronic information systems and playing cards posted a N N unconsolidated surge in pretax profit to N billion yen $ N million from N billion yen $ N million for the fiscal year ended aug. N - sales surged N N to N billion yen from N billion - net income rose N N to N billion yen from N billion - net fell to N yen from N yen because of expenses and capital adjustments - without detailing specific product credited its bullish in sales including advanced computer games and television entertainment systems to surging sales in foreign markets - export sales for leisure items alone for instance totaled N billion yen in the N months up from N billion in the previous fiscal year - domestic leisure sales however were lower - hertz corp. of park n.j. said it retained merrill lynch capital markets to sell its hertz equipment rental corp. unit - there is no pressing need to sell the unit but we are doing it so we can concentrate on our core business automobiles in the u.s. and abroad said william hertz 's executive vice president - we are only going to sell at the right price - hertz equipment had operating profit before depreciation of $ N million on revenue of $ N million in N - the closely held hertz corp. had annual revenue of close to $ N billion in N of which $ N billion was contributed by its hertz rent a car operations world-wide - hertz equipment is a major supplier of rental equipment in the u.s. france spain and the - it supplies commercial and industrial equipment including and electrical equipment and trucks - inc. reported a net loss of $ N million for the fiscal third quarter ended aug. N - it said the loss resulted from and introduction costs related to a new medical equipment system - in the year-earlier quarter the company reported net income of $ N or N cents a share - the manufacturer of diagnostic systems based in pa. reported a nine-month net loss of $ N million compared with net income of $ N million or N cents a share for the nine-month period a year earlier - in over-the-counter trading fell N cents to $ N - corp. expects to report third-quarter net of about $ N million or $ N a share down from $ N million or $ N a share a year earlier richard p. simmons chairman and chief executive officer told institutional investors in new york - sales for the producer of specialty and other materials fell to about $ N million in the third quarter from $ N million a year earlier he said - he said the third-quarter estimate indicates profit for the nine months of $ N a share almost equal to the full-year N earnings of $ N million or $ N a share - in the first nine months of N net was $ N million or $ N a share - mr. simmons said the third-quarter results reflect continued improvements in productivity and operating margins - he said capital spending next year will rise to about $ N million from about $ N million this year - u.s. banknote co. said it again extended the expiration date of its $ tender offer for international banknote co. to nov. N - u.s. banknote said it is in negotiations to sell certain facilities which it did n't name to a third party and it needs the extension to try to reach a definitive agreement on the sale - u.s. banknote said it believes the sale if completed apparently would satisfy antitrust issues raised by the u.s. justice department about u.s. banknote 's offer to buy international banknote - both of the new york-based companies print stock certificates and currency - u.s. banknote said there can be no assurance a sale agreement would be concluded - it also said the tender offer would probably have to be extended further to complete financing arrangements - u.s. banknote said citibank extended the expiration date of its commitment for senior secured financing to nov. N - the offer made june N has been extended several times - closely held u.s. banknote offered the $ N a share or $ N million for as many as N million shares or N N of international banknote 's shares outstanding - u.s. banknote said that as of oct. N N million shares or about N N of the fully diluted shares outstanding had been tendered - gitano group inc. said it agreed to buy N N of regatta sport ltd. a closely held apparel maker with the assumption of $ N million of contingent debt - under the terms of the contract new york-based gitano has the option to acquire the remaining N N of regatta a maker of men 's and women 's clothes sold primarily in department stores under certain conditions - that N N is now held by clifford parker regatta 's president and chief executive officer who will continue to manage regatta 's operations under gitano - in N regatta will have sales in excess of $ N million and will show a profit mr. parker said - gitano which makes apparel sold mainly through mass like k mart and said the regatta acquisition will enhance its strategy to expand into department stores - this fall gitano began manufacturing moderately priced clothes aimed at department stores under the trademark which gitano recently acquired - enron corp. houston said the sale of preference units of its newly formed enron partners l.p. master limited partnership subsidiary will result in an gain in the fourth quarter - in the year-ago quarter the natural gas concern had net income of $ N million or N cents a share on revenue of about $ N billion - those results included a $ N million charge related to the retirement of debt - in a related move enron said it increased the number of the partnership 's units it will offer to N from N - the old and revised numbers both include provisions - enron said each unit will be priced in the $ range and will represent about N N of the partnership equity - net proceeds from the offering are expected to be close to $ N million - goldman sachs & co. and drexel burnham lambert inc. are lead underwriters - arthur m. goldberg said he extended his unsolicited tender offer of $ N a share tender offer or $ N million for di giorgio corp. to nov. N - dig acquisition corp. the new jersey investor 's acquisition vehicle said that as of the close of business yesterday N shares had been tendered - including the stake dig already held dig holds a total of about N N of di giorgio 's shares on a fully diluted basis - the offer which also includes common and preferred stock purchase rights was to expire last night at midnight - the new expiration date is the date on which dig 's financing commitments which total about $ N million are to expire - dig is a unit of dig holding corp. a unit of rose partners - mr. goldberg is the sole general partner in rose partners - in august di giorgio a san francisco food products and building materials marketing and distribution company rejected mr. goldberg 's offer as inadequate - in new york stock exchange composite trading yesterday di giorgio closed at $ N a share down $ N - what does n't belong here - a. b. black-and-white c. radio shows - if you black-and-white you 're right - after years of into the background photography is coming back - trendy magazine advertisements feature stark black-and-white photos of hollywood pitching jeans shoes and liquor - portrait studios accustomed to shooting only in color report a rush to black-and-white portrait orders - and black-and-white photography classes are crowded with students - what 's happening in photography the popularity of black and white in fashion home and - on seventh avenue designers have been advancing the look with clothing done entirely in black and white - and classic black-and-white movies are enjoying a comeback on videocassette tapes spurred in part by the backlash against of old films - the is back to black and white says richard the general manager of eastman kodak co. 's professional photography division - until two years ago sales of black-and-white film had been declining steadily since the 1960s - but last year buoyed by increased use in advertising and other commercial applications sales increased N N and they are expected to jump at least that much again this year - photographic companies are scrambling to tap the market some black-and-white product lines and developing new ones - at kodak which largely ignored the market for years black-and-white film sales now account for nearly N N of the company 's $ N billion in film and paper sales annually up from N N three years ago - the rochester n.y. photographic giant recently began marketing N one of the fastest and most sensitive films - aimed at commercial the film can be used in very low light without quality says donald of newsletter - also trying to a portion of the $ N industry is corp. a unit of ag - recently signed olympic gold to a new line of black-and-white paper that 's geared to consumers and will compete directly with kodak 's papers - slated for market by the end of the year the paper could have been introduced a long time ago but the market was n't there then says an spokesman - the biggest of the black-and-white revival is likely to be international paper co. 's division known in the industry for its premium products - sales of 's four of black-and-white film this year are growth in the overall market although the company wo n't say by exactly how much - we hope the trend lasts says 's marketing communications director - why all the interest - for baby boomers who grew up being in color black and white seems and exotic - it has an almost quality to it says owen b. butler the chairman of the applied photography department at rochester institute of technology - you can shift out of reality with black and white he adds - such features have been especially attractive to professional and marketing executives who have been steadily increasing their use of black and white in advertising - processing of black-and-white commercial film jumped N N last year to N million rolls - consider gap inc. whose latest ad campaign features black-and-white shots of hollywood stars artists and other well-known the retailer 's jeans and - richard the account manager for the campaign says gap did n't intentionally choose black and white to its ads from the color spreads of competitors - we wanted to highlight the individual not the environment he says and black and white allows you to do that better than color - the campaign won a award as this year 's best ad by a specialty retailer - even food products and automobiles which have long depended on color are making the switch - companies feel black and white will convey a stronger statement says marc l. a chicago who is working on a black-and-white print ad for food corp. 's lean - other companies that are currently using ads include american express co. and america inc - portrait studios have also onto the trend - using black and white we can make look like stars says john - his photography studio in ore. doubled its business last year and he says is booked solid for the next five - one customer says she a color portrait for black and white because it 's more dramatic - i show it to my friends and they all say - it is n't ordinary like color - still most consumers are n't black-and-white film into their cameras to take family - one big obstacle is that few develop the film anymore - typically it must be to a handful of processors and may take a week or more to be processed and returned - black-and-white film costs consumers a little less than color film and processing costs the same - but for developing costs for black-and-white film are higher - some companies are starting to tackle that problem - for example recently introduced a black-and-white film that can be processed quickly by color labs - intent on wooing customers the company is also increasing its of black-and-white photography classes - similarly is scores of photography at high schools and colleges offering free black-and-white film and paper as prizes - and kodak is distributing an video to processors on how to develop its film more efficiently - other companies are introducing related products - charles co. a leading maker of photographic introduced last month a complete targeted at who want to process their own black-and-white photographs - the which has a suggested retail price of $ N and has already become a was introduced after retailers noticed numerous requests from parents for children 's photography equipment - it seems computers as have says ian 's chairman and chief executive officer - but some industry observers believe the of black and white is only a fad - they cite the emergence of still electronic photography more newspapers turning to color on their pages and improvements in the quality of color prints - black and white has n't made the same quantum in technological development as color says mr. butler of the rochester institute - the color print today is far superior to prints of N years ago - you ca n't say the same with black and white - but when popular photography a leading magazine for selected N of the greatest photos ever made for its latest issue celebrating photography 's anniversary all were black and white - it 's got a classic spirit and carries over says alfred of professional of america - that 's the appeal - newspapers inc. said improvements in advertising and subscription revenue led to a N N gain in third-quarter profit to $ N million or N cents a share from $ N million or N cents a share - sales rose more than N N to $ N million from $ N million - the sacramento calif. company also attributed improved performance to a lower effective tax rate and higher interest income - for the nine months the newspaper chain had almost a N N increase in profit to $ N million or N cents a share from $ N million or N cents a share - sales grew almost N N to $ N million from $ N million - publishes the sacramento calif and wash news tribune and other papers in western states - in composite trading on the new york stock exchange the company closed at $ N a share down N cents - agip s.p a. and societe national the state oil companies of italy and france respectively submitted an offer to buy suisse s.a - the price was n't disclosed - a spokesman for said that the swiss oil concern was the offer submitted last friday along with two other offers also submitted last week - those two offers were private and the spokesman refused to identify the bidding companies - the spokesman further said that at least two more offers are expected from other companies within two weeks - suisse owns an oil refinery in switzerland with a capacity of N barrels a day along with a network of gasoline retailing outlets - while friday 's plunging stock market prompted new fears about the economy 's prospects a indicator that has the economy 's ups and by exceptionally long lead times points to a sustained rise in overall business activity - the barometer developed by analysts at columbia university 's center for international business cycle research here reached a record high of N in august the latest month available and the columbia researchers estimate that it has moved even higher since then - the latest reading of N was up from N in july and N as recently as march - the august rise marked the fifth straight monthly gain for the indicator which uses the N average as a base of N - in contrast the commerce department 's widely followed index of leading indicators while up in august has fallen repeatedly since reaching a high early this year - its behavior through much of N has prompted some to anticipate the start of a new recession perhaps before year 's end - but the far stronger showing of the columbia index makes a recession any time soon highly unlikely says h. moore the director of the columbia facility - a leading authority on the business cycle mr. moore also is a member of the business cycle dating group the panel of private economists that decides for the government when and recessions begin and end - the group normally only when a change in the economy 's general course seems likely - no meeting is scheduled because the expansion shows no sign of going off the tracks mr. moore reports - based largely on the recent strength in their index called the long leading indicator the columbia analysts economic growth through the rest of this year and next year as well - they expect a N N rise in N in the gross national product after adjustment for inflation - underlying this optimism is the index 's longstanding ability to signal recessions or as the case may be by substantially greater periods than the commerce department 's index of leading indicators - over the full war ii era the columbia index on the average has entered sustained declines N months before the of recessions and turned up eight months before - the comparable lead times for the commerce index whose components include the stock market are far shorter N months before recessions and only three months before - the columbia economists also have how the long leading index would have behaved had it existed in N before the stock market crash in october that in the great depression - the indicator reached a peak in january N and then fell steadily up to and through the crash - it was an entirely different pattern from what we 're seeing now mr. moore says - a major source of the recent strength in the long leading indicator has been the performance of the dow jones corporate index which is not a part of the commerce index - in august the bond measure was at its highest monthly average since early N - it also rose last friday while the stock market sagged - other components of the long leading indicator include a ratio of prices to unit labor costs in manufacturing industries the version of the money supply adjusted for inflation and the volume of new permits - notably from the columbia index is the stock market which mr. moore says is simply no longer such a good indicator of the economy 's prospects though it still is useful for anticipating some and turns - as recently as N the stock market as reflected in the standard & poor 's index of N common stocks was rated by the national bureau of economic research as the best of the N leading indicators that then made up the commerce index - it was assigned a mark of N out of a possible N compared with scores ranging as low as N for the other components - the stock market has lost some power analysts at the columbia center claim because of the growing impact of international developments - stocks have become more sensitive to factors not directly tied to the domestic economy mr. moore says citing the exchange rate for the dollar on currency markets the balance and inflows of foreign capital - he also feels that the rise of such practices as program trading has diminished the stock market 's to the economic outlook - bsn s.a. a leading french food group said it agreed to acquire g.m.b h. a west german pasta maker - the value of the acquisition was n't disclosed - the move is in line with bsn 's strategy of gradually building its share of the european pasta market through external growth - bsn will initially acquire a N N interest in a closely held concern - the french group has an agreement giving it the right to buy all the shares outstanding and this could be completed within a few months a bsn spokeswoman said - the takeover was submitted for approval by the west german office bsn said - is west germany 's producer of pasta with sales of N million marks $ N million in N - it has N workers at three production units in southwest germany and is that nation 's leading producer of pasta - the acquisition bsn 's position in the european pasta market - the french group currently ranks second after group of italy whose sales are in the italian market - moody 's investors service inc. said it reduced its rating on $ N million of senior and subordinated debt of this thrift holding company to c from ca saying it believes bondholders will recover only negligible principal - the agency said it confirmed american continental 's preferred stock rating at c - american continental 's thrift unit los angeles-based lincoln savings & loan association is in and the parent company has filed for protection from creditor lawsuits under chapter N of the federal bankruptcy code - centrust savings bank miami - moody 's investors service inc. downgraded its ratings on the subordinated debt of centrust to from - the rating agency also reduced the ratings for long-term deposits to from and for preferred stock to ca from - the rating agency said about $ N million in securities was affected - the were prompted moody 's said by the continuing turmoil in the junk bond market and the suspension of dividends on centrust 's preferred stock - moody 's also said it believed the proposed sale of N centrust branches to great western bank could if completed endanger the thrift 's funding and market position - the stock market avoided a repeat of black monday as prices recovered from an early slide spurred by bargain-hunting institutions and program traders - the dow jones industrials closed up N points at N the gain ever after being down as much as N points in the morning - the rally erased about half of friday 's 190.58-point plunge but analysts are cautious about the market 's outlook - the dollar also rebounded while bond prices plummeted and treasury bill rates soared - junk bonds also recovered somewhat though trading remained stalled - gold also rose - tokyo stock prices bounced back in early trading tuesday following a N N plunge on monday - the dollar also moved higher in tokyo - donald trump withdrew his $ N billion offer for american air citing the recent change in market conditions - amr slid $ N to $ N - also a ual group tried to get financing for a lower bid possibly $ N a share - ual fell $ N to $ N - leveraged buy-outs of airlines would be subject to approval by the transportation secretary under a bill passed by a house subcommittee - ibm 's earnings tumbled N N in the third quarter slightly more than expected - the computer giant partly cited a stronger dollar and a delay in shipping a new high-end disk drive - analysts are about ibm 's outlook for the next few quarters - u.s. auto makers plan to decrease car production N N in the fourth quarter with virtually all the decline coming from the big three - output at and managed plants in the u.s. is due to rise N N - budget director darman said he wo n't give federal agencies much in coping with gramm-rudman spending cuts which took effect yesterday - darman hopes to congress to finish a deficit plan - the s&l bailout agency would be restricted by a new bill in how it raises capital - the ways and means plan would create another possible obstacle to selling sick thrifts - a natural gas rule was struck down by a federal appeals court - the regulation had prevented pipeline firms from passing part of $ N billion in costs along to customers - the supreme court agreed to decide whether a federal court may a merger that has won regulatory approval but been ruled in a private suit - merrill lynch 's profit slid N N in the third quarter - bear stearns posted a N N gain while painewebber had a decline due to a year-ago gain - blue arrow of britain plans to return to the name manpower and take a big write-off - the moves may help the firm its dominance of the u.s. market - j.p. morgan posted a $ N billion loss for the third quarter reflecting a big addition to loan-loss reserves - ncnb 's profit more than doubled - k mart agreed to acquire pace membership warehouse for $ N million expanding its presence in the growing business - markets - stocks volume N shares - dow jones industrials N up N transportation N off N utilities N up N - bonds shearson lehman hutton treasury index N off - commodities dow jones futures index N off N spot index N up N - dollar N yen off N N marks off N - monday october N N - the key u.s. and foreign annual interest rates below are a guide to general levels but do n't always represent actual transactions - prime rate N N N - the base rate on corporate loans at large u.s. money center commercial banks - federal funds N N N high N N N low N N N near closing bid N N N offered - reserves traded among commercial banks for overnight use in amounts of $ N million or more - source fulton prebon u.s.a inc - discount rate N N - the charge on loans to depository institutions by the new york federal reserve bank - call money N N N to N N - the charge on loans to brokers on stock exchange collateral - commercial paper placed directly by general motors acceptance corp. N N N to N days N N N to N days N N N to N days N N N to N days N N N to N days N N N to N days N N N to N days - commercial paper high-grade unsecured notes sold through dealers by major corporations in multiples of $ N N N N days N N N days N N N days - certificates of deposit N N one month N N two months N N three months N N six months N N one year - average of top rates paid by major new york banks on primary new issues of negotiable c.d.s usually on amounts of $ N million and more - the minimum unit is $ N - typical rates in the secondary market N N one month N N three months N N six months - bankers acceptances N N N days N N N days N N N days N N N days N N N days N N N days - negotiable bank-backed business credit instruments typically financing an import order - london late eurodollars N N N to N N N one month N N N to N N N two months N N N to N N N three months N N N to N N N four months N N N to N N N five months N N N to N N N six months - london interbank offered rates libor N N N one month N N N three months N N N six months N N N one year - the average of interbank offered rates for dollar deposits in the london market based on quotations at five major banks - foreign prime rates canada N N germany N N japan N N switzerland N N britain N N - these rate indications are n't directly comparable lending practices vary widely by location - treasury bills results of the monday october N N auction of short-term u.s. government bills sold at a discount from face value in units of $ N to $ N million N N N weeks N N N weeks - federal home loan mortgage corp freddie mac posted yields on 30-year mortgage commitments for delivery within N - N N standard conventional mortgages N N N N rate capped one-year adjustable rate mortgages - source telerate systems inc - federal national mortgage association fannie mae posted yields on N year mortgage commitments for delivery within N days priced at par N N standard conventional fixed N N N rate capped one-year adjustable rate mortgages - source telerate systems inc - merrill lynch ready assets trust N N - annualized average rate of return after expenses for the past N days not a forecast of future returns - intel corp. said it reached an agreement with computer systems corp. to develop software standards for intel 's microprocessor - the introduced earlier this year is intel 's entry in the crowded market for reduced instruction set computing or risc computers - intel based in santa clara calif. is the market leader for traditional microprocessors with its N family that forms the heart of personal computers - under the agreement intel will invest $ N million to acquire a N N stake in a maker of for scientists and engineers - based in mass. will license its technologies to intel providing users a way to let many microprocessors in a single computer work on a problem simultaneously - said it plans to use the microprocessor in future products - it declined to discuss its plans for upgrading its current product line - inc. which intends to expand its position in the medical and markets said it acquired a cotton and products division from closely held products corp. for $ N million - said it expects the division to add substantial sales volume and to make a positive contribution to our earnings in N and beyond - in N the cincinnati company earned $ N million or N cents a share on revenue of $ N million - said the division operates under the trade name and supplies the medical and markets - the business based in st. louis had sales of more than $ N million in the fiscal year ended march N said - burmah oil plc a british independent oil and specialty chemicals marketing concern said shv holdings n.v. of the netherlands has built up a N N stake in the company - james alexander a burmah spokesman said shv had previously owned a little under N N of burmah for about two years - the dutch company had n't notified burmah of its reason for increasing the stake he said - shv which last year merged its north sea oil and gas operations with those of group plc has been pegged by speculators as a possible suitor for burmah oil in recent weeks - shv also owns N N of - burmah which owns the brand of oils reported a N N rise in net income to # N million $ N million in the first half - j.p. industries inc. said it signed a definitive agreement to sell its builders ' hardware group to closely held inc. of beverly hills calif - terms were n't disclosed but a j.p. industries spokesman said the amount j.p. industries will get for the group is a little better than expected by the marketplace and the marketplace had been expecting $ N million to $ N million - the group consists of corp. and modern inc - j.p. industries which is based in ann mich. said the sale a previously announced program to itself of its hardware and plumbing supplies operations - the company 's remaining business is the manufacture and sale of engine and products for industrial and transportation applications - citing a $ N million provision for doubtful accounts dallas-based national heritage inc. posted a loss for its fourth quarter ended june N - a unit of troubled southmark corp. the operator of nursing homes and retirement centers said it sustained a net loss of $ N million or nine cents a share compared with net income of $ N million or eight cents a share a year earlier - operating revenue rose N N to $ N million from $ N million in the year-earlier quarter - the company said the $ N million reserve was created to reflect doubt about the of receivables owed to national heritage by some of the real estate partnerships it manages - the company also said expenses incurred by the previous board and management in the recent contest for control were recognized primarily in the first quarter ended sept. N - national heritage stock fell N cents yesterday to close at $ N a share in new york stock exchange composite trading - united biscuits holdings plc a british food producer announced the creation of a european group to bring together its trading interests in the region - the new group will all of united 's manufacturing and marketing operations in the food sector apart from those based in the u.s. - united biscuits said the combined group which will include businesses such as biscuits and terry 's will have annual sales of more than # N billion $ N billion and trading profit of more than # N million $ N million - the new structure will enable united biscuits to focus clearly upon opportunities for planned growth during the 1990s said bob deputy chairman and group chief executive - last month united biscuits agreed to sell its entire restaurant operations to grand metropolitan plc for # N million - an american journalist now is standing trial in namibia - this is the place that world opinion has been celebrating over in the expectation that it 's going to hold an election - the most likely winner will be the swapo rebels - the u.s. journalist 's crime was writing that the head of the commission charged with overseeing the election 's fairness was openly sympathetic to swapo - shortly after that mr. had scott stanley arrested and his confiscated - mr. stanley is on trial over charges that he violated a issued by the south african administrator general earlier this year which made it a crime punishable by two years in prison for any person to or the election commission - the stanley affair does n't well for the future of democracy or freedom of anything in namibia when swapo starts running the government - to the extent mr. stanley has done anything wrong it may be that he is out of step with the consensus of world intellectuals that the guerrillas were above all else the victims of by neighboring south africa - swapo has enjoyed favorable western media treatment ever since the u.n. general assembly declared it the sole representative of namibia 's people in - last year the u.s. a peace settlement to remove cuba 's from and hold free and fair elections that would end south africa 's control of namibia - the elections are set for nov. N - in july mr. stanley editor of american press international a washington conservative wire service visited namibia to report on the election campaign - he interviewed mr. head of a commission charged with investigating electoral and reported that mr. was openly sympathetic to swapo and indeed had defended its leaders in court - after mr. stanley 's article was published in two newspapers mr. had criminal charges brought against their editors publisher and lawyer - mr. stanley was arrested and charged along with the others when he returned to namibia this month - both the state department and the lawyers committee for freedom of the press have mr. stanley 's - mr. stanley 's arrest is the latest in a series of incidents that threaten to namibia 's elections - both south african and swapo are voters - the u.s. is in the habit of arranging peace settlements and then its hands over the results - it now has the chance to that record in namibia - state and the human-rights community should insist that mr. stanley and his fellow defendants be released and that the united nation 's monitors make certain that mr. commission election complaints from all sides - commodity futures prices generally reflected the stability of the stock market following its plunge friday - yesterday the stock market 's influence at first created nervousness - later however it became more of an than a force as individual markets reacted more to their own factors - gold the traditional haven in times of financial crisis continued its with the dollar rising early in the day as the currency fell and then giving up some of its gains as the dollar recovered - copper and crude oil reacted sharply to the concern that a crash yesterday could have a potentially devastating effect on the economy - copper fell and showed little rebound through the day as one of the major supply problems that had been supporting prices appeared to be solved - crude oil declined early but as the stock market retained early gains it too became stronger ending with a small net loss - trading in cotton and sugar was nervous and showed small declines - in chicago grain and soybean prices rose slightly - livestock and meat prices however dropped on concern that a financial crisis would cut consumption of beef and pork - in commodity markets yesterday precious metals futures prices were moderately higher as gold gave up some of its early gains and platinum behaved first falling and then later rising - silver performed quietly - the spot october gold price rose $ N to $ N an ounce - the more active december delivery gold settled with a gain of $ N an ounce at $ N after trading as high as $ N - december silver was up N cents an ounce at $ N - platinum behaved more like an industrial metal easing early on concern over a possible weaker economy but recovering later as the stock market strengthened - gold was nowhere the spectacular performer it was two years ago on black monday - for one thing last friday precious metals markets closed before the stock market went into its nose dive so it could n't react to it - back on friday oct. N the stock market declined during the day and gold prices surged as the stock market fell - the october N contract that day rose as much as $ N to as high as $ N and the more deferred positions due to mature as late as march N rose as much as $ N - on black monday oct. N N the october contract tacked on further gains rising to as high as $ N for a gain of almost $ N on top of the friday advances before giving up almost $ N of that at the close - yesterday 's october gain of $ N was compared with that - one analyst peter of & co. new york said the gold market already had some good technical factors that would have caused prices to rise with or without the stock market - what the stock market did was cause the rise to take place earlier than it would have happened said mr. - there 's a good chance that gold will retain its gains and rise further he said - he expects a drop in interest rates which would help gold by keeping the dollar from rising - finally according to mr. the impact of the strong dollar should be reflected in reduced exports in the august merchandise trade deficit when the figures are released today - this would be damaging to the dollar and supportive for gold he said - energy - worried that friday 's 190-point stock market plunge might be a of things to come for the economy petroleum futures traders called a halt to the recent string of increases in crude oil futures prices - the u.s. benchmark crude west texas intermediate closed at $ N a barrel for november delivery down N cents - some analysts said crude was due for a correction following several days of significant gains - but most market observers agreed that friday 's stock market drop is what spirits in the petroleum pits yesterday - until yesterday futures prices had been headed up on expectations that world oil demand will continue to be strong - the organization of petroleum exporting countries increased its production ceiling for the fourth quarter based on projections of robust demand - so any bearish indicator such as friday 's drop in the stock market sends through the oil markets as well - indeed after reacting early in the trading day to friday 's plummet futures prices firmed up again as traders took note of the stock market 's partial recovery yesterday - copper - futures prices fell and showed little rebound as one major labor problem that had been prices appeared to be solved - the december contract declined N cents a pound to $ N - prices were down from the outset of trading on concern that a drop in the stock market might create a weakened economy and a reduction in copper use - but the recovery in the stock market provided little help for copper as word spread that a three-month strike at the highland valley mine in british columbia was about over according to an analyst - highland valley is a large canadian producer and principal supplier to japan which recently began seeking copper elsewhere as its inventories shrank - last week it was reported that company and union negotiations had overcome the major hurdle the contracting out of work by the company - now the analyst said only minor points remain to be up - for all and purposes an agreement appears to have been achieved he said - copper inventories in new york 's commodity exchange warehouses rose yesterday by N tons to N tons - london metal exchange copper inventories last week declined N tons to N tons - the stocks decline was about as expected but the comex gain was n't - however this was brushed aside by concern over the stock market the analyst said - at one point in futures trading as the stock market firmed the december contract rose to as high as $ N but it was n't able to sustain the gain - it was simply he said and selling by funds that are computer helped depress prices - cotton - futures prices eased more in reaction to hurricane jerry than to any influence of the stock market - the december contract ended with a loss of N cent a pound at N cents - technical considerations following the hurricane which was a factor in the market friday caused prices to decline yesterday said ernest simon cotton specialist for prudential-bache securities new york - prices rose sharply friday as the storm approached texas and louisiana which is part of the mississippi delta area - however after the potential effect of the hurricane prices began to slip late friday mr. simon said - that selling continued yesterday and kept prices under pressure he said - weather is being predicted for the high plains of texas and the northern states of the delta during the coming weekend mr. simon said - that has n't yet captured traders ' attention he added - sugar - futures prices declined - the march contract was off N cent a pound at N cents - at one point in early trading the march price rose to as high as N cents when the stock market recovered but the price then fell back - a factor one analyst said was that india which had been expected to buy around N tons of sugar in the world market did n't make any purchases - india recently bought N tons and was expected to buy more the analyst said - another analyst thought that india may have pulled back because of the concern over the stock market - india may have felt that if there was a severe drop in the stock market and it affected sugar it could buy at lower prices said analyst for shearson lehman hutton new york - at any rate she added india needs the sugar so it will be in sooner or later to buy it - farm products - the prices of cattle and futures contracts dropped sharply because traders speculated that the stock market plunge friday will in the minds of u.s. consumers long enough to prompt them to rein in their spending at the supermarket which would hurt demand for beef and pork - the price of the contract for october delivery dropped its maximum permissible daily limit of N cents a pound - the prices of most grain futures contracts rose slightly yesterday out of relief that the stock market was showing signs of recovering - earlier in the session the prices of several soybean contracts set new lows - a broad rally began when several major processors began buying futures contracts apparently to take advantage of the price dip - knight-ridder inc. said it would report increased earnings per share for the third quarter contrary to reported analysts ' comments that the publishing company 's earnings would be down - a company spokesman said he believed the confusion was caused when james knight-ridder 's chairman and chief executive told new york analysts two weeks ago that knight-ridder 's earnings per share for the first nine months of N would be behind a little bit from like period of - the knight-ridder spokesman said the third-quarter earnings that the company plans to report oct. N are expected to be up - the spokesman said he was comfortable with revised analysts ' projections that the company would report earnings of between N cents and N cents a share compared with the N cents a share it reported for the N third quarter - knight-ridder said it agreed with estimates that net income for all of N would be around $ N a share compared with $ N a share a year earlier - in new york stock exchange composite trading yesterday knight-ridder closed at $ N down N cents - dd acquisition corp. said it extended its $ offer for dunkin donuts inc. to nov. N from yesterday - the offer has an indicated value of $ N million - dd acquisition is a partnership of unicorp canada corp. 's capital group unit and cara operations ltd - as previously reported under the terms of a agreement with dunkin donuts the partners agreed to keep their offer open until nov. N and not to acquire any additional shares except through a tender offer on that date - dd acquisition said that it already owns N N of the common shares of the shop chain and that as of the close of business friday an additional N N had been tendered to its offer - dunkin donuts is based in mass - cara operations a food services concern and unicorp a holding company with interests in oil and natural gas and financial services are based in toronto - golden west financial corp. riding above the turbulence that has troubled most of the thrift industry posted a N N increase of third-quarter earnings to $ N or N cents a share - the company earned $ N million or N cents a share in the year-ago quarter - herbert m. chairman and chief executive officer of the oakland calif. savings-and-loan holding company credited the high number of loans added to the company 's portfolio over the last N months for its earning asset base and improving profit performance - however the executive noted that demand for new mortgages depressed new loan to $ N billion N N below the same period last year - in savings activity mr. said consumer deposits have enjoyed a steady increase throughout N and topped $ N billion at quarter 's end for the first time in the company 's history - deposit growth amounted to $ N million more than double the year-ago figure - corp. benton harbor mich. said it has developed a process to recover environmentally harmful chlorofluorocarbons or cfcs that previously entered the atmosphere during repair of refrigerators and - the maker of home appliances said the process which involves the use of a plastic bag during repairs to capture the substance and transport it to a recycling center is already in use at a number of its service centers and will be available to all authorized repair centers by spring - earlier repairs the cfcs out of the home through a directly into the atmosphere - cfcs are widely used as and fire - but their use has been linked to a potentially dangerous depletion of the earth 's ozone layer and a number of companies are seeking to curtail use or at least of the substance - said we see this process as a small but important step toward eventual elimination of use in manufacture - energy corp. dallas said it discovered a new oil field northeast of its previously discovered field in the southeast area of indonesia - said it did n't run a production test on the three discovery wells it in the field which is about N miles from the field because the wells are similar to others at its and fields - however said it believes the reserves in the field are about N million barrels of oil - the field has estimated reserves of N million barrels and the field has estimated reserves of N million barrels - an independent oil and gas concern is the operator and owns a N N interest in the new field called northeast - other interests are owned by petroleum development ltd. c. energy co. ltd. g.m.b h. production ltd. oil indonesia ltd. co. ltd. ltd. shell a.g. and oil co - the contract area is held with the state oil company - environmental systems co. said it is its results to reduce its reported net income for the first nine months of its fiscal year after it took tax credits that already had been taken last year - the little rock services company said the will reduce its net for the nine months ended july N to $ N million or N cents a share from $ N million or N cents a share - net for the third quarter restated is $ N million or N cents a share - the company previously reported net of $ N million or N cents a share - the company said that for financial reporting purposes last year it took tax credits that will be recognized for tax purposes this year - but because of confusion it took those credits again in reporting its results through the first nine months - jack w. environmental systems president and chief executive officer said the change increases the company 's effective tax rate to about N N from N N - memotec data inc. said it signed a definitive merger agreement with isi systems inc. under which memotec will acquire isi for $ N u.s. a share or about $ N million in cash and securities - in american stock exchange composite trading isi closed up $ N at $ N - in montreal exchange trading memotec closed unchanged at N canadian dollars us$ N - memotec said under the agreement isi a mass. provider of computer software and services to the insurance industry will merge with a u.s. unit of memotec created for that purpose - memotec is a maker of telecommunications products and provider of telecommunications and computer services - memotec said the agreement calls for it to make a $ cash tender offer for all shares outstanding of isi - but it said charles johnston isi chairman and president agreed to sell his N N stake in isi to memotec upon completion of the tender offer for a combination of cash memotec stock and debentures - memotec said the tender offer is on among other things holders at least N N of the shares outstanding other than the shares held by mr. johnston - isi said its board has instructed management to accept inquiries from any others interested in making a bid - isi said it can withdraw from the merger agreement with memotec if a better bid - cms energy corp. jackson mich. said it has resumed the purchase of its common stock under a program approved by its directors in N - at the time of the original announcement cms said its board authorized the purchase of as many as five million of its shares - a spokesman said N million shares have been purchased since then - the company said it will buy additional shares from time to time in the open market or in private transactions at prevailing market prices - in composite trading on the new york stock exchange cms energy closed at $ N a share down N cents from the closing price of $ N a share on thursday before friday 's plunge - the utility company currently has about N million shares outstanding - morgan stanley & co. will act as the exclusive broker for the repurchase - hughes aircraft co. a unit of general motors corp. said it agreed to purchase the technology division of corp - terms of the agreement were n't disclosed - but for the fiscal year ended july N N the most recent period for which results were broken out the unit accounted for more than half the $ N million in sales recorded by the company 's government systems sector - which is based in conn. said the sale of the conn. unit is consistent with its restructuring strategy announced in april - in addition to making systems the unit also makes laser warning - these are used aboard military to warn pilots that a laser weapon has been focused on them - hughes of los angeles said the unit 's work efforts by its and data systems group which makes military and night vision equipment - hughes said it expects the sale to close by year end - the communications workers of america ratified a new regional contract and all but one of the local agreements with bell atlantic corp - 's new jersey commercial local which represents about N service representatives and marketing employees rejected the tentative agreement - both the union and the regional telephone company said they were working together to resolve differences - the new three-year contracts which replace ones that expired aug. N cover N bell atlantic employees - the follows a strike against the company - meanwhile and international of electrical workers members remain on strike against nynex corp. the new york-based regional phone company - the unions and the company last week agreed to - the represents N nynex workers and the represents N workers - for the moment at least euphoria has replaced anxiety on wall street - the dow jones industrial average jumped sharply yesterday to close at N panic did n't sweep the world 's markets and investors large and small seemed to accept friday 's dizzying 190-point plunge as a sharp correction not a - many went bargain-hunting - among those with relief was john h. gutfreund chairman of salomon brothers who took to the firm 's trading floor to monitor yesterday 's events - as the rally gained strength at N p.m. he broadly his and stanley his top stock trader on the back - at first it seemed as if history might repeat itself - as trading opened yesterday morning on the big board stocks of many of the nation 's biggest companies could n't open for trading because a wave of sell orders was overwhelming buyers - by N the dow industrials were off N points and the stock of ual corp. whose troubles had kicked off friday 's plunge still had n't opened - but then as quickly as the dow had fallen it began to turn around - it ended with a gain of N points - by the market 's close volume on the new york exchange totaled more than N million the fourth highest on record - the big board handled the huge volume without any obvious strain in sharp contrast to black monday of N - but the rally was largely confined to the blue-chip stocks which had been hard hit during friday 's selling frenzy - overall more big board stocks lost money than gained - and many arbitragers already reeling from friday 's collapse of the ual deal were further hurt yesterday when a proposed takeover of amr corp. the parent of american airlines collapsed - indeed the dow jones transportation average plunged N points its drop in history - world-wide trading was generally - the frankfurt stock exchange was hardest hit of the major markets with blue chips there falling N N - in london a midday rally left the market 's major index off N N and tokyo 's leading stock index fell only N N in surprisingly lackluster trading - other more traded asian markets were hit harder than tokyo 's but there were no declines - investors big and small say they learned valuable since the N crash in this age of computerized trading huge or in a few hours ' time must be expected - what 's more such short-term are and are no cause for panic selling - stephen boesel a major money manager for t. rowe price in baltimore says there was less panic than in N we had been through it once - in wis. who owns a supplier of equipment and is n't active in the stock market agrees - i look at it as a matter he says - many other factors played a part in yesterday 's comeback - the federal reserve signaled its willingness to provide liquidity the interest rate on its loans to major banks inched downward early in the day - foreign stock markets which kicked off black monday with a huge selling spree began the day off by relatively modest amounts - the dollar after falling sharply in overnight trading to N yen bounced back strongly to N thus easing fears that foreigners would unload u.s. stocks - and the widely opinion among most market experts that a crash was n't in store also helped calm investors - many major institutions for example came into work yesterday ready to buy some of the blue chips they felt had been sharply undervalued on friday - still amid all the and signs of relief over yesterday 's events some market professionals cautioned that there is nothing present in the current market system to prevent another dizzying drop such as friday 's - there is too much says money manager barry - computers have increasingly connected securities markets world-wide so that a buying or selling wave in one market is often passed around the globe - so investors everywhere nervously yesterday 's opening in tokyo where the nikkei average of N blue-chip stocks got off to a rocky start - the average plunged some N points or N N in the first N minutes of trading - but the selling wave had no conviction and the market first surged upward by N points then drifted lower closing down N - unlike two years ago most of japan 's major investors chose to sit this out - in merrill lynch & co. 's tokyo trading room some N traders and sat quietly with few orders to process - clients are all staying out of the market one merrill trader says - the relative calm in tokyo proved little comfort to markets opening up in europe - frankfurt 's opening was delayed a half hour because of a crush of sell orders - the beginning was chaotic says nigel a broker for commerzbank - in london the view from the trading floor of an american securities firm jefferies & co. also was troubling - a computer screen N blue-chip stocks colors each one red when its price is falling - the screen was a sea of red - i see concern but i do n't see panic says j. francis a new yorker who runs the office - london 's blue-chip stock index turned up just before N a.m new york time sending an encouraging message to wall street - when trading opened in new york at N a.m. edt stocks fell sharply as expected - futures markets in chicago had opened at a level suggesting the dow would fall by about N points - with sell orders up from friday about half the stocks in the dow could n't open on time - by N the industrial average had dropped N points - by N a.m. it was down N - ten minutes later the dow hit bottom down N points another N N - but shortly before then some of wall street 's sharpest traders say they a turn - the first thing that caught my eye that was encouraging was treasury bonds were off says austin george head of stock trading at t. rowe price - it meant that people were n't running to the safety of bonds - shortly after N a.m. the major market index a chicago board of trade futures contract of N stocks designed to mimic the exploded upward - stock traders were buoyed because an in the mmi had also started the recovery in stocks on the tuesday following black monday - the mmi has gone better shouted a trader in the london office of shearson lehman hutton - shearson 's london trading room went wild - traders shouted out as their reuters quotron and telerate screens posted an loss on wall street - then nine minutes later wall street suddenly rebounded to a gain on the day - rally rally rally shouted shearson 's andy rosen - this is panic buying - major blue-chip stocks like philip morris general motors and & gamble led the rally - japanese were said to be heavy buyers - german and dutch investors reportedly loaded up on kellogg co - then traders say corporations with share buy-back programs kicked into high gear triggering gains in among other issues and mcdonald 's - walt disney co. which had one of the biggest imbalances on friday and was one of seven stocks that halted trading and never reopened that day opened yesterday late at N down N - but then it suddenly burst upward N as goldman sachs & co. stepped in and bought almost every share offer traders said - by N the dow had turned up for the day prompting on trading desks and exchange floors - among big board specialists the cry was pull your offers meaning that specialists soon expected to get higher prices for their shares - it was on the upside said one big board specialist - what we had was a real old-fashioned rally - this technical strength spurred buying from wall street 's black boxes computer programs designed to trigger large stock purchases during bullish periods - typical perhaps was 's dean - mr. who manages $ N billion says we turned the trading system on and it did whatever it was to do - asked what stocks the computer bought the money manager says i do n't know - not everybody was making money - the on the chicago board options exchange the nation 's major options market was heavy after the trading in s&p N stock-index options was halted friday - many market makers in the s&p N index options contract had bullish positions friday and when the shutdown came they were frozen with huge losses - over the weekend clearing firms told the chicago market makers to get out of their positions at any cost monday morning - they were absolutely killed said one chicago-based options trader - meanwhile a test of the stock market 's rally came at about N p.m. with the dow at N up N points on the day - charles a strategist at merrill lynch says bargain hunting had explained the dow 's strength up to that point and that many market professionals were anticipating a drop in the dow - moreover the announcement that real estate and sometime raider donald trump was his offer for amr corp. might have been expected to traders - instead the rally only for about N minutes and then forward as institutions resumed buying - the market closed minutes after reaching its high for the day of - across the country many people took yesterday 's events in while remaining generally uneasy about the stock market in general - says james norman the mayor of mo. i do n't invest in stocks - i much prefer money i can put my hands on - while mayor norman found the market 's performance monday reassuring he says he remains uneasy - we have half the experts saying one thing and half the other about the course of the economy - ralph a farmer and store operator in neb. says of the last few days events if anything good comes out of this it might be that it puts some of these lbos on the - says gordon fines a money manager at financial services in minneapolis you 're on a roller and that may last - the public is still cautious - skipper 's inc. wash. said it signed a definitive merger agreement for a national pizza corp. unit to acquire the N N of skipper 's inc. it does n't own for $ N a share or about $ N million - acquisition co. a national pizza unit plans to begin a tender offer for skipper 's on friday on at least two-thirds of skipper 's shares being tendered - national pizza said the transaction will be financed under its revolving credit agreement - in national over-the-counter trading skipper 's shares rose N cents to $ N - skipper 's said the merger will help finance remodeling and future growth - skipper 's previously turned down a $ proposal from national pizza and pizza hut inc. questioned whether the purchase would violate national pizza 's franchise agreements - national pizza said it settled its dispute with pizza hut allowing it to make the purchase - also skipper 's results began to turn around permitting a higher offer national pizza said - for the N weeks ended sept. N skipper 's had net income of $ N or N cents a share compared with a net loss a year earlier - revenue was $ N million - east germans rallied as officials reportedly sought honecker 's - in what was considered the largest protest in the communist state 's history at least N demonstrators marched through the southern city of leipzig to press demands for democratic freedoms opposition activists said - police did n't intervene - meanwhile as the first of more than N east germans trying to to the west through poland their a west german newspaper reported that regional communist officials demanded the dismissal of hard-line leader honecker - secretary of state baker in a foreign policy speech called for the reunification of germany saying it was the legitimate right of the german people - gorbachev blamed the soviet union 's press for contributing to the nation 's mounting problems - at a meeting friday the kremlin leader complained about recent articles that raised the of civil unrest and accused the media of fueling panic buying of goods by publishing stories about impending shortages - house-senate conferees approved a permanent smoking ban on domestic airline routes within the continental u.s. and on flights of less than six hours to alaska and hawaii - the curbs would cover all but a small percentage of flights and represent an expansion of the current ban on flights of less than two hours - e. robert was sentenced by a u.s. judge in new york to six years in prison and fined $ N for his racketeering conviction in the wedtech scandal - an associate of general was found guilty in august of taking $ N in illegal from the defense contractor - nasa resumed the for today 's launch of the space shuttle atlantis and a federal appeals court in washington dismissed a lawsuit by anti-nuclear groups to delay the flight because the galileo space probe was aboard - the space agency said it did n't expect weather or protesters to block the - the bush administration is preparing to extend a ban on federal financing of research using tissue government sources said - a temporary prohibition was imposed in march N - while anti-abortion groups are opposed to such research scientists have said such tissue could be effective in treating - delegates from N nations endorsed a ban on world ivory trade in an attempt to rescue the endangered elephant from - five african nations however said they would continue selling the valuable - held reconciliation talks with at the egyptian resort of - it was the leader 's first trip to egypt in N years - they announced a reduction in for travel but did n't show any real signs of full diplomatic ties - the egyptian president said he would visit libya today to resume the talks - seoul and reached a tentative agreement to allow visits between families on the divided korean peninsula - such family would be the second since N - differences remained between the north and south korean governments however over conditions for the exchanges - freed black resumed political activity in south africa and vowed to fight against apartheid raising fears of a possible white backlash - the nation 's main white opposition party warned that the government 's release sunday of eight black political bringing chaos and eventual black marxist rule to the nation - the white house said bush is fully satisfied with cia director webster and the intelligence agency 's performance during the oct. N failed coup in panama - the washington post reported that unidentified senior administration officials were frustrated with webster 's activities during the and wanted him replaced - poland 's legislature approved limits on automatic wage increases without special provisions for food price rises - the vote was considered a test of the government 's resolve to proceed with a harsh program - norway 's king installed a government as 's labor regime power - the cabinet is led by prime minister jan who acknowledged a difficult situation since the coalition controls only N seats in 's legislature - el salvador 's government opened a new round of talks with the country 's leftist rebels in an effort to end a civil war - a spokesman said the guerrillas would present a cease-fire proposal during the negotiations in costa rica that includes constitutional and economic changes - the state department said there was a possibility that some nicaraguan rebels were selling their arms to guerrillas but insisted it was n't an organized effort - separately secretary of state baker complained about a u.n. aide who last week told the contras to as part of a regional peace accord - died N actor and director in los angeles of - N novelist and sunday in paris of cancer - british retail sales volume rose a provisional N N in september from august and was up N N from september N the department of trade and industry said - for the three months ended in september retail sales volume was down N N from the previous three months and up N N from a year earlier - chicago investor william agreed to sell three divisions of cluett peabody & co. for about $ N million to s.a. a closely held clothing maker based in paris - shortly after completing the $ N billion acquisition of west inc. in april mr. 's holding company inc. said it was considering the sale of cluett a leading maker and one of west 's biggest units - included in the sale are cluett units that make men 's shirts under the arrow name under the gold name and through the division - the companies said the agreement is subject to 's of financing and to regulatory and other approvals - they said the sale is expected to be concluded by the end of november - mr. said the sale of three of cluett 's four main divisions plus other anticipated west asset sales by december should bring in a total of about $ N million - he did n't elaborate on other asset sales being considered - mr. followed a similar pattern when he acquired northwest industries inc. and then sold much of its assets - but he kept fruit of the inc. the underwear maker that he still controls and serves as chairman and chief executive - cluett was an independent company until west acquired it for $ N million in cash and stock in N - in the fiscal year ended sept. N N cluett had operating profit of $ N million on sales of $ N million - sells clothes under various labels including and bill robinson for men and ralph for women - a spokesman said the company had sales of $ N million in N - in new york stock exchange composite trading west fell N cents to $ N - britain 's blue arrow plc intends to change its name to manpower plc and write off a chunk of the nearly $ N billion in good will realized in the takeover of manpower inc - blue arrow chairman mitchell fromstein said in an interview that the two steps may be a prelude to the world 's biggest group in the u.s. - mr. fromstein disclosed the planned steps expected within a few months as blue arrow posted a N N drop in its third-quarter pretax earnings - the name change and good will write-off could help blue arrow 's dominance of the u.s. market and give it a more american image as u.s. investors turn jittery about foreign stocks after friday 's market plunge - u.s. holders now own more than N N of blue arrow compared with N N last january - in the u.s. market the recognition of the manpower name is stronger than blue arrow mr. fromstein said - the moves also could shareholders ' perception of blue arrow as a company in turmoil - it further the concept that blue arrow is a thing of the past said doug arthur an analyst at kidder peabody & co. in new york - the proposed changes all make a lot of sense to me he added - in a widely publicized coup mr. fromstein ousted berry as blue arrow chief executive in january a month after mr. berry had forced mr. fromstein out as the $ N chief of manpower - mr. fromstein his control in april by taking over from mr. berry as chairman - but the blue arrow is n't over yet as the british government is investigating a disputed # N million $ N million loan which mr. fromstein has said was made under mr. berry 's direction - blue arrow was able to pull off the $ N billion takeover of manpower in N largely because different british and american accounting standards produce higher reported earnings for british companies - under british rules blue arrow was able to write off at once the $ N billion in good will arising from the purchase - as a company blue arrow would have to the good will over as many as N years creating a continuing drag on reported earnings - good will is the excess of cost of an acquired firm operating unit or assets over the current or fair market value of those assets - but with so many shares now held in the u.s. blue arrow reports its earnings two ways based on both u.k. and u.s. accounting standards - our balance sheets look like they came from alice 's mr. fromstein said - the british version shows a handful of pounds of net worth following the N write-off of good will while the american version reflects $ N billion of net worth because almost none of the good will has been written off - mr. fromstein said he hopes to some of the good will left on blue arrow 's u.s. books in one fell but would n't specify how much - people close to blue arrow suggested the write-down would represent a sizable chunk with executives claiming prior management the extent of manpower 's good will - that move along with the return to the manpower name could bolster the company 's prospects during possibly difficult times for temporary help - the number of u.s. temporary workers fell about N N in the N months ending aug. N after sliding nearly N N in july said kidder peabody 's mr. arthur - blue arrow blamed the pretax profit drop in the quarter ended july N partly on slower earnings growth of units in britain - overall pretax profit slid to # N million in the quarter from # N million a year earlier - richard g. sim the man credited with applied power inc. from an into a player in the global market for tools hopes to guide a similar turnaround at the company 's latest acquisition barry wright corp - the 45-year-old former general electric co. executive figures it will be easier this time - but analysts while the acquisition say applied 's chief executive faces a tough challenge in the two companies - barry wright acquired by applied for $ N million makes equipment and systems - the mass. company 's sales have been and its profits have dropped - last year 's earnings of $ N million including $ N from a restructuring gain were far below the year-earlier $ N million - besides spurring barry wright 's sales which were $ N million in N mr. sim must its costs and product line - the question is how long it 's going to take barry wright to make a contribution says f. john an analyst at blunt ellis in milwaukee - the answer will help determine whether applied continues to reach the ambitious goals set by mr. sim - the butler wis. manufacturer went public at $ N a share in august N and mr. sim 's goal then was a $ N per-share price by N - strong earnings growth helped achieve that price far ahead of schedule in august N - the stock has since trading around $ N a share last week and closing yesterday at $ N in national over-the-counter trading - but mr. sim has set a fresh target of $ N a share by the end of - reaching that goal says robert t. applied 's chief financial officer will require efficient reinvestment of cash by applied and of its healthy N N rate of return on operating capital - in barry wright mr. sim sees a situation very similar to the one he faced when he joined applied as president and chief operating officer in N - applied then a closely held company was under the management of its controlling family - while profitable it was n't growing and was n't providing a satisfactory return on invested capital he says - mr. sim is confident that the drive to dominate certain niche markets will work at barry wright as it has at applied - he also an to develop a corporate culture that rewards managers who produce and where is shared - mr. sim considers the new unit 's operations fundamentally sound and adds that barry wright has been fairly successful in moving into markets that have n't interested larger competitors - with a little patience these businesses will perform very mr. sim says - within about six months things will be moving in the right direction he predicts - mr. sim figures it will be easier to turn barry wright around since he 's now in the driver 's seat - when he came to applied i did n't have the power to execute as i do today he says - he was named chief executive officer of applied in N and became chairman last november - at applied mr. sim set growth as his first objective - he took the company public in an offering that applied about $ N million which helped launch the company 's acquisition program - sales climbed to an estimated $ N million in fiscal N ended aug. N from $ N million in fiscal N - the company expects that earnings which have marched steadily upward in recent years reached about $ N million or $ N a share in the fiscal year just ended up from $ N million in fiscal N and $ N million in N diff --git a/test/data_for_tests/people_infer.txt b/test/data_for_tests/people_infer.txt deleted file mode 100644 index 639ea413..00000000 --- a/test/data_for_tests/people_infer.txt +++ /dev/null @@ -1,2 +0,0 @@ -迈向充满希望的新世纪——一九九八年新年讲话 -（附图片1张） \ No newline at end of file diff --git a/test/data_for_tests/zh_sample.conllx b/test/data_for_tests/zh_sample.conllx new file mode 100644 index 00000000..dee802ef --- /dev/null +++ b/test/data_for_tests/zh_sample.conllx @@ -0,0 +1,100 @@ +1 上海 _ NR NR _ 3 nsubj _ _ +2 积极 _ AD AD _ 3 advmod _ _ +3 准备 _ VV VV _ 0 root _ _ +4 迎接 _ VV VV _ 3 ccomp _ _ +5 欧元 _ NN NN _ 6 nn _ _ +6 诞生 _ NN NN _ 4 dobj _ _ + +1 新华社 _ NR NR _ 7 dep _ _ +2 上海 _ NR NR _ 7 dep _ _ +3 十二月 _ NT NT _ 7 dep _ _ +4 三十日 _ NT NT _ 7 dep _ _ +5 电 _ NN NN _ 7 dep _ _ +6 （ _ PU PU _ 7 punct _ _ +7 记者 _ NN NN _ 0 root _ _ +8 潘清 _ NR NR _ 7 dep _ _ +9 ） _ PU PU _ 7 punct _ _ + +1 即将 _ AD AD _ 2 advmod _ _ +2 诞生 _ VV VV _ 4 rcmod _ _ +3 的 _ DEC DEC _ 2 cpm _ _ +4 欧元 _ NN NN _ 6 nsubj _ _ +5 ， _ PU PU _ 6 punct _ _ +6 引起 _ VV VV _ 0 root _ _ +7 了 _ AS AS _ 6 asp _ _ +8 上海 _ NR NR _ 14 nn _ _ +9 这 _ DT DT _ 14 det _ _ +10 个 _ M M _ 9 clf _ _ +11 中国 _ NR NR _ 13 nn _ _ +12 金融 _ NN NN _ 13 nn _ _ +13 中心 _ NN NN _ 14 nn _ _ +14 城市 _ NN NN _ 16 assmod _ _ +15 的 _ DEG DEG _ 14 assm _ _ +16 关注 _ NN NN _ 6 dobj _ _ +17 。 _ PU PU _ 6 punct _ _ + +1 上海 _ NR NR _ 2 nn _ _ +2 银行界 _ NN NN _ 4 nsubj _ _ +3 纷纷 _ AD AD _ 4 advmod _ _ +4 推出 _ VV VV _ 0 root _ _ +5 了 _ AS AS _ 4 asp _ _ +6 与 _ P P _ 8 prep _ _ +7 之 _ PN PN _ 6 pobj _ _ +8 相关 _ VA VA _ 15 rcmod _ _ +9 的 _ DEC DEC _ 8 cpm _ _ +10 外汇 _ NN NN _ 15 nn _ _ +11 业务 _ NN NN _ 15 nn _ _ +12 品种 _ NN NN _ 15 conj _ _ +13 和 _ CC CC _ 15 cc _ _ +14 服务 _ NN NN _ 15 nn _ _ +15 举措 _ NN NN _ 4 dobj _ _ +16 ， _ PU PU _ 4 punct _ _ +17 积极 _ AD AD _ 18 advmod _ _ +18 准备 _ VV VV _ 4 dep _ _ +19 启动 _ VV VV _ 18 ccomp _ _ +20 欧元 _ NN NN _ 21 nn _ _ +21 业务 _ NN NN _ 19 dobj _ _ +22 。 _ PU PU _ 4 punct _ _ + +1 一些 _ CD CD _ 8 nummod _ _ +2 热衷于 _ VV VV _ 8 rcmod _ _ +3 个人 _ NN NN _ 5 nn _ _ +4 外汇 _ NN NN _ 5 nn _ _ +5 交易 _ NN NN _ 2 dobj _ _ +6 的 _ DEC DEC _ 2 cpm _ _ +7 上海 _ NR NR _ 8 nn _ _ +8 市民 _ NN NN _ 13 nsubj _ _ +9 ， _ PU PU _ 13 punct _ _ +10 也 _ AD AD _ 13 advmod _ _ +11 对 _ P P _ 13 prep _ _ +12 欧元 _ NN NN _ 11 pobj _ _ +13 表示 _ VV VV _ 0 root _ _ +14 出 _ VV VV _ 13 rcomp _ _ +15 极 _ AD AD _ 16 advmod _ _ +16 大 _ VA VA _ 18 rcmod _ _ +17 的 _ DEC DEC _ 16 cpm _ _ +18 兴趣 _ NN NN _ 13 dobj _ _ +19 。 _ PU PU _ 13 punct _ _ + +1 继 _ P P _ 38 prep _ _ +2 上海 _ NR NR _ 6 nn _ _ +3 大众 _ NR NR _ 6 nn _ _ +4 汽车 _ NN NN _ 6 nn _ _ +5 有限 _ JJ JJ _ 6 amod _ _ +6 公司 _ NN NN _ 13 nsubj _ _ +7 十八日 _ NT NT _ 13 tmod _ _ +8 在 _ P P _ 13 prep _ _ +9 中国 _ NR NR _ 10 nn _ _ +10 银行 _ NN NN _ 12 nn _ _ +11 上海 _ NR NR _ 12 nn _ _ +12 分行 _ NN NN _ 8 pobj _ _ +13 开立 _ VV VV _ 19 lccomp _ _ +14 上海 _ NR NR _ 16 dep _ _ +15 第一 _ OD OD _ 16 ordmod _ _ +16 个 _ M M _ 18 clf _ _ +17 欧元 _ NN NN _ 18 nn _ _ +18 帐户 _ NN NN _ 13 dobj _ _ +19 后 _ LC LC _ 1 plmod _ _ +20 ， _ PU PU _ 38 punct _ _ +21 工商 _ NN NN _ 28 nn _ _ +22 银行 _ NN NN _ 28 conj _ _ From d1b5adabc4c6a10f372683b945f01130f40bf544 Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Mon, 4 Feb 2019 09:56:08 +0800 Subject: [PATCH 31/32] add codecov fix --- codecov.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..f91e0445 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,5 @@ +ignore: +- "reproduction" # ignore folders and all its contents +- "setup.py" +- "docs" +- "tutorials" \ No newline at end of file From b66d7b8f51f33afb6979f7f7578529db1e612e1e Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Mon, 4 Feb 2019 10:07:56 +0800 Subject: [PATCH 32/32] update API introduction --- fastNLP/api/README.md | 21 +++++++++++---------- fastNLP/api/examples.py | 20 +++++++++++--------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/fastNLP/api/README.md b/fastNLP/api/README.md index 3604bd07..73560f9f 100644 --- a/fastNLP/api/README.md +++ b/fastNLP/api/README.md @@ -18,26 +18,27 @@ print(cws.predict(text)) # ['编者 按 : 7月 12日 , 英国 航空 航天 系统 公司 公布 了 该 公司 研制 的 第一 款 高 科技 隐形 无人 机雷电 之 神 。', '这 款 飞行 从 外型 上 来 看 酷似 电影 中 的 太空 飞行器 , 据 英国 方面 介绍 , 可以 实现 洲际 远程 打击 。', '那么 这 款 无人 机 到底 有 多 厉害 ?'] ``` -### 中文分词+词性标注 +### 词性标注 ```python -text = ['编者按：7月12日，英国航空航天系统公司公布了该公司研制的第一款高科技隐形无人机雷电之神。', - '这款飞行从外型上来看酷似电影中的太空飞行器，据英国方面介绍，可以实现洲际远程打击。', - '那么这款无人机到底有多厉害？'] +# 输入已分词序列 +text = [['编者', '按：', '7月', '12日', '，', '英国', '航空', '航天', '系统', '公司', '公布', '了', '该', '公司', + '研制', '的', '第一款', '高科技', '隐形', '无人机', '雷电之神', '。'], + ['那么', '这', '款', '无人机', '到底', '有', '多', '厉害', '？']] from fastNLP.api import POS pos = POS(device='cpu') print(pos.predict(text)) -# [['编者/NN', '按/P', '：/PU', '7月/NT', '12日/NR', '，/PU', '英国/NR', '航空/NN', '航天/NN', '系统/NN', '公司/NN', '公布/VV', '了/AS', '该/DT', '公司/NN', '研制/VV', '的/DEC', '第一/OD', '款高/NN', '科技/NN', '隐形/NN', '无/VE', '人机/NN', '雷电/NN', '之/DEG', '神/NN', '。/PU'], ['这/DT', '款/NN', '飞行/VV', '从/P', '外型/NN', '上/LC', '来/MSP', '看/VV', '酷似/VV', '电影/NN', '中/LC', '的/DEG', '太空/NN', '飞行器/NN', '，/PU', '据/P', '英国/NR', '方面/NN', '介绍/VV', '，/PU', '可以/VV', '实现/VV', '洲际/NN', '远程/NN', '打击/NN', '。/PU'], ['那么/AD', '这/DT', '款/NN', '无/VE', '人机/NN', '到底/AD', '有/VE', '多/CD', '厉害/NN', '？/PU']] +# [['编者/NN', '按：/NN', '7月/NT', '12日/NT', '，/PU', '英国/NR', '航空/NN', '航天/NN', '系统/NN', '公司/NN', '公布/VV', '了/AS', '该/DT', '公司/NN', '研制/VV', '的/DEC', '第一款/NN', '高科技/NN', '隐形/AD', '无人机/VV', '雷电之神/NN', '。/PU'], ['那么/AD', '这/DT', '款/NN', '无人机/VV', '到底/AD', '有/VE', '多/AD', '厉害/VA', '？/PU']] ``` -### 中文分词+词性标注+句法分析 +### 句法分析 ```python -text = ['编者按：7月12日，英国航空航天系统公司公布了该公司研制的第一款高科技隐形无人机雷电之神。', - '这款飞行从外型上来看酷似电影中的太空飞行器，据英国方面介绍，可以实现洲际远程打击。', - '那么这款无人机到底有多厉害？'] +text = [['编者', '按：', '7月', '12日', '，', '英国', '航空', '航天', '系统', '公司', '公布', '了', '该', '公司', + '研制', '的', '第一款', '高科技', '隐形', '无人机', '雷电之神', '。'], + ['那么', '这', '款', '无人机', '到底', '有', '多', '厉害', '？']] from fastNLP.api import Parser parser = Parser(device='cpu') print(parser.predict(text)) -# [['12/nsubj', '12/prep', '2/punct', '5/nn', '2/pobj', '12/punct', '11/nn', '11/nn', '11/nn', '11/nn', '2/pobj', '0/root', '12/asp', '15/det', '16/nsubj', '21/rcmod', '16/cpm', '21/nummod', '21/nn', '21/nn', '22/top', '12/ccomp', '24/nn', '26/assmod', '24/assm', '22/dobj', '12/punct'], ['2/det', '8/xsubj', '8/mmod', '8/prep', '6/lobj', '4/plmod', '8/prtmod', '0/root', '8/ccomp', '11/lobj', '14/assmod', '11/assm', '14/nn', '9/dobj', '8/punct', '22/prep', '18/nn', '19/nsubj', '16/pccomp', '22/punct', '22/mmod', '8/dep', '25/nn', '25/nn', '22/dobj', '8/punct'], ['4/advmod', '3/det', '4/nsubj', '0/root', '4/dobj', '7/advmod', '4/conj', '9/nummod', '7/dobj', '4/punct']] +# [['2/nn', '4/nn', '4/nn', '20/tmod', '11/punct', '10/nn', '10/nn', '10/nn', '10/nn', '11/nsubj', '20/dep', '11/asp', '14/det', '15/nsubj', '18/rcmod', '15/cpm', '18/nn', '11/dobj', '20/advmod', '0/root', '20/dobj', '20/punct'], ['4/advmod', '3/det', '8/xsubj', '8/dep', '8/advmod', '8/dep', '8/advmod', '0/root', '8/punct']] ``` 完整样例见`examples.py` \ No newline at end of file diff --git a/fastNLP/api/examples.py b/fastNLP/api/examples.py index 9d9f190e..a85e7c30 100644 --- a/fastNLP/api/examples.py +++ b/fastNLP/api/examples.py @@ -22,8 +22,9 @@ def chinese_word_segmentation_test(): def pos_tagging(): # 输入已分词序列 - text = ['编者 按： 7月 12日 ， 英国 航空 航天 系统 公司 公布 了 该 公司 研制 的 第一款 高科技 隐形 无人机 雷电之神 。'] - text = [text[0].split()] + text = [['编者', '按：', '7月', '12日', '，', '英国', '航空', '航天', '系统', '公司', '公布', '了', '该', '公司', + '研制', '的', '第一款', '高科技', '隐形', '无人机', '雷电之神', '。'], + ['那么', '这', '款', '无人机', '到底', '有', '多', '厉害', '？']] pos = POS(device='cpu') print(pos.predict(text)) @@ -34,8 +35,9 @@ def pos_tagging_test(): def syntactic_parsing(): - text = ['编者 按： 7月 12日 ， 英国 航空 航天 系统 公司 公布 了 该 公司 研制 的 第一款 高科技 隐形 无人机 雷电之神 。'] - text = [text[0].split()] + text = [['编者', '按：', '7月', '12日', '，', '英国', '航空', '航天', '系统', '公司', '公布', '了', '该', '公司', + '研制', '的', '第一款', '高科技', '隐形', '无人机', '雷电之神', '。'], + ['那么', '这', '款', '无人机', '到底', '有', '多', '厉害', '？']] parser = Parser(device='cpu') print(parser.predict(text)) @@ -46,9 +48,9 @@ def syntactic_parsing_test(): if __name__ == "__main__": - chinese_word_segmentation() - chinese_word_segmentation_test() - pos_tagging() - pos_tagging_test() + # chinese_word_segmentation() + # chinese_word_segmentation_test() + # pos_tagging() + # pos_tagging_test() syntactic_parsing() - syntactic_parsing_test() + # syntactic_parsing_test()