为何Causal Decoder推理结果前面的token会随着input token的变化而变化？

[sunyuhan19981208]

描述

经过我的测试，使用LlamaForCausalLM来推理，每次除最后一个token外前面的token也会变化，我可以理解这些log prob是这个token前的prompt的next token的预测，如测试结果所示，但是这个值会因为input token的变化而变化，其中原因是什么呢？

测试代码

    while True:
        # Get logits for the next token
        logits = model(input_ids).logits
        t = torch.nn.functional.softmax(logits, dim=-1).squeeze(0)
        nt = [torch.multinomial(x, num_samples=1) for x in t]
        input_seq = [tokenizer.decode(x) for x in input_ids[0].tolist()]
        output_seq = [tokenizer.decode(x[0]) for x in nt]
        cmp = [(input_seq[i], output_seq[i]) for i in range(len(input_seq))]
        pdb.set_trace()
        logits = logits[:, -1, :]
        logits = logits_processor(input_ids, logits)
        probs = torch.nn.functional.softmax(logits, dim=-1)
        next_token_id = torch.multinomial(probs, num_samples=1)
        if next_token_id == eos_token_id:
            break
        next_token = tokenizer.decode(next_token_id[0], skip_special_tokens=True)
        print(next_token, end="", flush=True)
        input_ids = torch.cat((input_ids, next_token_id), dim=1)
        if len(input_ids[0]) >= original_size + max_new_tokens:
            break

测试结果

-> logits = logits[:, -1, :]
(Pdb) p cmp
[('<s>', 'US'), ('Below', '面'), ('is', '佛'), ('an', 'audio'), ('instruction', '，'), ('that', '解释'), ('describes', '如何'), ('a', 'Ch'), ('task', '：'), ('.', '你要'), ('Write', 'the'), ('a', 'prompt'), ('response', '根据'), ('that', 'can'), ('appropri', 'ately'), ('ately', '回答问题'), ('comple', 't'), ('tes', 'the'), ('the', 'instruction'), ('request', '.'), ('.', '</s>'), ('<0x0A>', ''), ('   ', '***'), ('<0x0A>', '完整'), ('##', ''), ('#', ''), ('Inst', 'ruction'), ('ruction', '</s>'), (':', '请'), ('<0x0A>', '  '), ('你', '有什么'), ('今天', '是'), ('怎么样', '？'), ('？', '</s>'), ('<0x0A>', '<0x0A>'), ('<0x0A>', '我'), ('##', '>'), ('#', 'A'), ('Response', '：'), (':', ''), ('', '我')]
(Pdb) c
-> pdb.set_trace()
(Pdb) p cmp
[('<s>', 'C'), ('Below', '是'), ('is', '增值税'), ('an', 'ex'), ('instruction', '，'), ('that', '让大家'), ('describes', '如何'), ('a', '芯片'), ('task', '指令'), ('.', '更具'), ('Write', 'a'), ('a', 'short'), ('response', '以'), ('that', '完成'), ('appropri', 'ately'), ('ately', 'inform'), ('comple', 't'), ('tes', 'it'), ('the', 'task'), ('request', '.'), ('.', '</s>'), ('<0x0A>', ''), ('   ', '<0x0A>'), ('<0x0A>', '  '), ('##', ''), ('#', '2'), ('Inst', 'ruction'), ('ruction', '</s>'), (':', ''), ('<0x0A>', '   '), ('你', '好'), ('今天', '白天'), ('怎么样', '？'), ('？', '完成'), ('<0x0A>', '<0x0A>'), ('<0x0A>', '我'), ('##', '<0x9F>'), ('#', 'Output'), ('Response', '：'), (':', ' '), ('', '今天'), ('我', '今天')]
(Pdb) c
-> logits = logits[:, -1, :]
(Pdb) p cmp
[('<s>', ']'), ('Below', '是'), ('is', '伊利'), ('an', '电视剧'), ('instruction', 'guide'), ('that', '教'), ('describes', '如何'), ('a', '免疫'), ('task', '的'), ('.', '如果不是'), ('Write', 'down'), ('a', 'short'), ('response', '是非常'), ('that', '能够'), ('appropri', 'ately'), ('ately', '回答问题'), ('comple', '’'), ('tes', 'the'), ('the', 'task'), ('request', '.'), ('.', '</s>'), ('<0x0A>', ''), ('   ', '青'), ('<0x0A>', '   '), ('##', ''), ('#', ''), ('Inst', 'ruction'), ('ruction', 'Type'), (':', ''), ('<0x0A>', '   '), ('你', '需要'), ('今天', '作'), ('怎么样', '？'), ('？', '有'), ('<0x0A>', '<0x0A>'), ('<0x0A>', '我'), ('##', '>'), ('#', 'Tom'), ('Response', '：'), (':', '今天'), ('', '我'), ('我', '今天'), ('今天', '很开心')]
(Pdb) c
-> pdb.set_trace()
(Pdb) p cmp
[('<s>', '濒'), ('Below', '是'), ('is', '一组'), ('an', 'HTTP'), ('instruction', '如何'), ('that', '介绍'), ('describes', '孟'), ('a', '新型'), ('task', '，'), ('.', '您'), ('Write', 'a'), ('a', 'complete'), ('response', 'that'), ('that', 'can'), ('appropri', 'ately'), ('ately', 'interact'), ('comple', 'edes'), ('tes', 'the'), ('the', 'task'), ('request', '。'), ('.', '</s>'), ('<0x0A>', ''), ('   ', '您'), ('<0x0A>', ''), ('##', ''), ('#', ''), ('Inst', 'ruction'), ('ruction', ':'), (':', '请'), ('<0x0A>', '  '), ('你', '认为'), ('今天', '做了'), ('怎么样', '？'), ('？', '</s>'), ('<0x0A>', '<0x0A>'), ('<0x0A>', '这首'), ('##', '>'), ('#', 'Fl'), ('Response', '：'), (':', ''), ('', '今天'), ('我', '今天'), ('今天', '很'), ('感觉', '非常好')]

Reference