seminars/sem12/main.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EcHffgVhRT5H"
      },
      "source": [
        "# Скрытые марковские модели"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "em_U31o1RT5L"
      },
      "source": [
        "## Библиотеки"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MLhNjz_PRT5M"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "\n",
        "import urllib\n",
        "import numpy as np\n",
        "import nltk\n",
        "from nltk.tag import hmm\n",
        "from nltk.corpus import brown\n",
        "import pandas as pd\n",
        "\n",
        "from collections import Counter\n",
        "\n",
        "import nltk.lm as lm\n",
        "from nltk.util import ngrams as nltk_ngrams\n",
        "import numpy as np\n",
        "import scipy.stats as st\n",
        "\n",
        "from nltk.tokenize import RegexpTokenizer\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L0AfTKAPRT5N"
      },
      "source": [
        "## Brown Dataset HMM"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V8ppPChJRT5O"
      },
      "source": [
        "### Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5ZtJIU1FRT5O"
      },
      "outputs": [],
      "source": [
        "nltk.download('brown')\n",
        "english = re.compile('^[a-z]+$')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5nJkEUPSRT5P"
      },
      "outputs": [],
      "source": [
        "tokens = []\n",
        "for sent in brown.sents():\n",
        "    for w in sent:\n",
        "        w = w.lower()\n",
        "        if english.match(w):\n",
        "                tokens.append(w)\n",
        "print(f'Number of tokens: {len(tokens)}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UAusC2rZRT5P"
      },
      "outputs": [],
      "source": [
        "text = ' '.join(tokens)\n",
        "len(text), text[:100]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dZrKjFpqRT5Q"
      },
      "source": [
        "### Unsupervised обучение скрытой марковской модели (Алгоритм Баума-Велша)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uUGAQQ2sRT5Q"
      },
      "outputs": [],
      "source": [
        "vocab = sorted(list(set(text)))\n",
        "len(vocab)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Y8s1El4yRT5Q"
      },
      "outputs": [],
      "source": [
        "trainer = hmm.HiddenMarkovModelTrainer(range(2), vocab)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7yJiBdXkRT5R"
      },
      "outputs": [],
      "source": [
        "tagger = trainer.train_unsupervised([text[:5000]], max_iterations=50)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fjgmp6QORT5R"
      },
      "source": [
        "#### Исследуем полученную модель\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V-mcMDFjRT5S"
      },
      "source": [
        "Матрица переходов $$\\{a_{ij} = p(s_j|s_i)\\}_{i,j = 1}^{|S|}$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ydoGaAEbRT5T"
      },
      "outputs": [],
      "source": [
        "trans_matr = pd.DataFrame(\n",
        "    data=np.array([\n",
        "        [2 ** log_p for log_p in tagger._transitions[0]._data],\n",
        "        [2 ** log_p for log_p in tagger._transitions[1]._data]\n",
        "    ]),\n",
        "    columns=[0, 1],\n",
        "    index=[0, 1])\n",
        "trans_matr"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3u_FSx-qRT5T"
      },
      "source": [
        "Матрица выходных вероятностей $$\\{ b_{ij} = p(x_j|s_i) \\}_{i, j = 1}^{|S|, |X|}$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WrKM_c_cRT5U"
      },
      "outputs": [],
      "source": [
        "out_matr = pd.DataFrame(\n",
        "    data=np.array([\n",
        "        [2 ** log_p for log_p in tagger._outputs[0]._data],\n",
        "        [2 ** log_p for log_p in tagger._outputs[1]._data]\n",
        "    ]),\n",
        "    index=[0, 1],\n",
        "    columns=vocab)\n",
        "out_matr"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FeLpdkslRT5V"
      },
      "source": [
        "### Supervised обучение скрытой марковской модели (максимум правдоподобия)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yN4lgLPnRT5V"
      },
      "outputs": [],
      "source": [
        "def make_tag(c):\n",
        "    if c in 'aeiouy':\n",
        "        return (c, '1')\n",
        "    else:\n",
        "        return (c, '0')\n",
        "supervised = [make_tag(c) for c in text]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8_zlBW0yRT5V"
      },
      "outputs": [],
      "source": [
        "tagger = trainer.train_supervised([supervised[:500]])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6YpAnxOZRT5W"
      },
      "source": [
        "#### Исследуем полученную модель"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IblN01rzRT5W"
      },
      "source": [
        "Частоты совстречаемостей тегов"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JI_oAPOxRT5X"
      },
      "outputs": [],
      "source": [
        "for t in tagger._transitions:\n",
        "    print(t, tagger._transitions[t].__dict__)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LyB8nhvxRT5X"
      },
      "source": [
        "Матрица переходов $$\\{a_{ij} = p(s_j|s_i)\\}_{i,j = 1}^{|S|}$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ujIdn6DNRT5X"
      },
      "outputs": [],
      "source": [
        "trans_matr = pd.DataFrame(\n",
        "    data=np.array([\n",
        "        [tagger._transitions['0'].prob('0'), tagger._transitions['0'].prob('1')],\n",
        "        [tagger._transitions['1'].prob('0'), tagger._transitions['1'].prob('1')]\n",
        "    ]),\n",
        "    columns=[0, 1],\n",
        "    index=[0, 1])\n",
        "trans_matr"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VHn2uq-kRT5Y"
      },
      "source": [
        "Матрица выходных вероятностей $$\\{ b_{ij} = p(x_j|s_i) \\}_{i, j = 1}^{|S|, |X|}$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oyWPOd1PRT5Y"
      },
      "outputs": [],
      "source": [
        "out_matr = pd.DataFrame(\n",
        "    data=np.array([\n",
        "        [tagger._outputs['0'].prob(c) for c in vocab],\n",
        "        [tagger._outputs['1'].prob(c) for c in vocab]\n",
        "    ]),\n",
        "    index=[0, 1],\n",
        "    columns=vocab)\n",
        "out_matr"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OTQ6evofRT5Z"
      },
      "source": [
        "## Языковая модель"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-V-mSyQQRT5Z"
      },
      "source": [
        "Построим языковую модель сначала вручную на простом синтетическом корпусе, затем обучим модель из пакета `nltk` на стихотворении \"Дом, который построил Джек\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "COkikTDORT5Z"
      },
      "source": [
        "### Первый пример"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cN6C-TmeRT5a"
      },
      "outputs": [],
      "source": [
        "text = 'SOS SOS ' + 'А Б ' * 100 + 'EOS'\n",
        "tokens = text.split()\n",
        "n = len(tokens)\n",
        "tokens[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0ubomQg2RT5a"
      },
      "outputs": [],
      "source": [
        "def ngrams_and_prefix_counts(tokens, n_max):\n",
        "    # словарь n-грамм и их частот\n",
        "    ngrams_counts = {}\n",
        "    # словарь n-граммных префиксов и их частот\n",
        "    prefix_counts = {}\n",
        "    \n",
        "    n = len(tokens)\n",
        "    for i in range(n_max):\n",
        "        ngrams_counts[i + 1] = Counter([tuple(tokens[j : j + i + 1]) for j in range(n - i)])\n",
        "        prefix_counts[i + 1] = Counter([tuple(tokens[j : j + i] + ['*']) for j in range(n - i)])\n",
        "\n",
        "    return ngrams_counts, prefix_counts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7xJSXH9ORT5b"
      },
      "outputs": [],
      "source": [
        "ngram_counts, prefix_counts = ngrams_and_prefix_counts(tokens, 3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zH9oRVQdRT5c"
      },
      "outputs": [],
      "source": [
        "ngram_counts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qWyYcudmRT5d"
      },
      "outputs": [],
      "source": [
        "prefix_counts"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pBqO7zhCRT5e"
      },
      "source": [
        "#### N-граммы и их частотные вероятности\n",
        "\n",
        "$$\\hat p_i = \\hat p(w_i)$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "G7vuESQVRT5f"
      },
      "outputs": [],
      "source": [
        "def unigram_probas(ngram_counts):\n",
        "    p1 = {}\n",
        "    n = sum(ngram_counts[1].values())\n",
        "    for w in ngram_counts[1]:\n",
        "        p1[w] = ngram_counts[1][w] / n\n",
        "    return p1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4S95yvHQRT5f"
      },
      "outputs": [],
      "source": [
        "p1 = unigram_probas(ngram_counts)\n",
        "p1"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lqHAcEnfRT5g"
      },
      "source": [
        "$$\\hat p_{i, i - 1} = \\hat p(w_i|w_{i - 1})$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9XhGF7uLRT5g"
      },
      "outputs": [],
      "source": [
        "def bigram_probas(ngram_counts, prefix_counts):\n",
        "    p2 = {}\n",
        "    for w in ngram_counts[2]:\n",
        "        pre_w = tuple([w[0]] + ['*'])\n",
        "        p2[u'{1}|{0}'.format(*w)] = ngram_counts[2][w] / prefix_counts[2][pre_w]\n",
        "    return p2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZSqBROVKRT5h"
      },
      "outputs": [],
      "source": [
        "p2 = bigram_probas(ngram_counts, prefix_counts)\n",
        "p2"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mmF8FrFIRT5h"
      },
      "source": [
        "$$\\hat p_{i, i - 1, i - 2} = \\hat p(w_i|w_{i - 1}, w_{i - 2})$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cB6t-v6wRT5i"
      },
      "outputs": [],
      "source": [
        "def trigram_probas(ngram_counts, prefix_counts):\n",
        "    p3 = {}\n",
        "    for w in ngram_counts[3]:\n",
        "        pre_w = w[:2] + tuple(['*'])\n",
        "        p3[u'{2}|{1},{0}'.format(*w)] = ngram_counts[3][w] / prefix_counts[3][pre_w]\n",
        "    return p3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1uDSP5VsRT5i"
      },
      "outputs": [],
      "source": [
        "p3 =  trigram_probas(ngram_counts, prefix_counts)\n",
        "p3"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qfNm2vlQRT5j"
      },
      "source": [
        "#### Проверка гипотезы, что триграммную модель можно свести к биграммной против правосторонней альтернативы"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UlZ1_UkqRT5j"
      },
      "source": [
        "Статистика:\n",
        "$$-2 \\log (\\prod_{i, j, k = 1}^m (\\hat p_{ij} / \\hat p_{ijk})^{n_{ijk}}) = \\sum_{i, j, k}^m -2 n_{ijk} \\log \\hat p_{ij} + 2 n_{ijk} \\log \\hat p_{ijk} = \\sum_{i = 3}^N -2 \\log \\hat p_{i,i - 1} + 2 \\log \\hat p_{i, i - 1, i - 2},$$\n",
        "$$n_{ijk} = |\\{X_t: X_t = O_i, X_{t + 1} = O_j, X_{t + 2} = O_k\\}|$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Y0sYiOnqRT5k"
      },
      "outputs": [],
      "source": [
        "def chi2_statistic(p2, p3, tokens):\n",
        "    stat2 = []\n",
        "    stat3 = []\n",
        "    n = len(tokens)\n",
        "    for i in range(n - 2):\n",
        "        w = tokens[i : i + 3]\n",
        "        ngram3 = '{2}|{1},{0}'.format(*w)\n",
        "        ngram2 = '{1}|{0}'.format(*w)\n",
        "\n",
        "        stat2.append(np.log(p2[ngram2]))\n",
        "        stat3.append(np.log(p3[ngram3]))\n",
        "    return - 2 * np.sum(stat2) + 2 * np.sum(stat3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PbCfL68IRT5k"
      },
      "outputs": [],
      "source": [
        "m = len(p3)\n",
        "stat = chi2_statistic(p2, p3, tokens)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4u01LLL6RT5k"
      },
      "outputs": [],
      "source": [
        "print(f'p-value = {1 - st.distributions.chi2(m * ((m - 1) ** 2) - 1).cdf(stat)}')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l1GXwSUrRT5l"
      },
      "source": [
        "### Второй пример"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9NzCWmTXRT5m"
      },
      "outputs": [],
      "source": [
        "text = 'SOS SOS ' + 'А Б Б А Б А Б А Б Б А А ' * 100\n",
        "tokens = text.split()\n",
        "tokens[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3QAlOwO_RT5m"
      },
      "outputs": [],
      "source": [
        "ngram_counts, prefix_counts = ngrams_and_prefix_counts(tokens, 3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Vojg3xB_RT5m"
      },
      "outputs": [],
      "source": [
        "ngram_counts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "819yFAzURT5n"
      },
      "outputs": [],
      "source": [
        "prefix_counts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZebTMmh5RT5n"
      },
      "outputs": [],
      "source": [
        "p1 = unigram_probas(ngram_counts)\n",
        "p1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OR8nuVGFRT5n"
      },
      "outputs": [],
      "source": [
        "p2 = bigram_probas(ngram_counts, prefix_counts)\n",
        "p2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UywYB2WjRT5o"
      },
      "outputs": [],
      "source": [
        "p3 =  trigram_probas(ngram_counts, prefix_counts)\n",
        "p3"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SoW0jTHCRT5o"
      },
      "source": [
        "#### Проверка той же гипотезы"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9Lrqq3sdRT5o"
      },
      "outputs": [],
      "source": [
        "stat = chi2_statistic(p2, p3, tokens)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PFcMhacVRT5o"
      },
      "outputs": [],
      "source": [
        "print(f'p-value = {1 - st.distributions.chi2(m * ((m - 1) ** 2) - 1).cdf(stat)}')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1l6QqZPnRT5p"
      },
      "source": [
        "#### Сглаживание Лапласа"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "m6sq_KJ9RT5p"
      },
      "outputs": [],
      "source": [
        "n1 = list(nltk_ngrams(tokens, 1))\n",
        "n2 = list(nltk_ngrams(tokens, 2))\n",
        "n3 = list(nltk_ngrams(tokens, 3))\n",
        "n3[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Fz9IEpP5RT5p"
      },
      "outputs": [],
      "source": [
        "laplace = lm.Laplace(order=3)\n",
        "laplace.fit([n1] + [n2] + [n3], vocabulary_text=list(set(tokens)))\n",
        "regular_lm = lm.MLE(order=3)\n",
        "regular_lm.fit([n1] + [n2] + [n3], vocabulary_text=list(set(tokens)))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0KNYDsx4RT5p"
      },
      "source": [
        "#### Перплексия (Меньше => лучше)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Bd3yoahSRT5q"
      },
      "outputs": [],
      "source": [
        "laplace.perplexity(n1), regular_lm.perplexity(n1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZaJCuWqsRT5q"
      },
      "outputs": [],
      "source": [
        "foo = [('b'), ('a'), ('r')]\n",
        "laplace.perplexity(foo), regular_lm.perplexity(foo)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xvbmdhGBRT5q"
      },
      "source": [
        "#### Сглаженная по Лапласу оценка вероятности\n",
        "\n",
        "$$p_L(w_i) = \\frac{c_i + 1}{\\sum_{i = 1}^v c_i + v}$$\n",
        "$$p_L(w_i|w_j) = \\frac{c_{ij} + 1}{\\sum_{j=1}^v (c_{ij} + 1)} = \\frac{c_{ij} + 1}{c_i + v}$$\n",
        "\n",
        "$$p_L('А'|'SOS')$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fzxk-Jm_RT5q"
      },
      "outputs": [],
      "source": [
        "laplace.score('А', context=['SOS']), regular_lm.score('А', context=['SOS'])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "miJh49bzRT5r"
      },
      "source": [
        "$$p_L('SOS')$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z6oRUT8BRT5r"
      },
      "outputs": [],
      "source": [
        "laplace.score('SOS'), regular_lm.score('SOS')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WS-QCNtdRT5r"
      },
      "source": [
        "#### n-граммы не встречаючиеся в тексте:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gVd9AWgqRT5s"
      },
      "outputs": [],
      "source": [
        "laplace.score('C', context=['SOS']), laplace.score('ыаываа', context=['B']), laplace.score('B')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "m9UJKdpmRT5s"
      },
      "outputs": [],
      "source": [
        "regular_lm.score('C', context=['SOS']), regular_lm.score('ыаываа', context=['B']), regular_lm.score('B')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fi6kpBi3RT5s"
      },
      "source": [
        "### Генерация текста"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v5DH4iZpRT5t"
      },
      "outputs": [],
      "source": [
        "rt = RegexpTokenizer(u'\\w+')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FHNLm4hPRT5t"
      },
      "outputs": [],
      "source": [
        "with urllib.request.urlopen('https://raw.githubusercontent.com/Intelligent-Systems-Phystech/psad/master/seminars/sem12/data/jack.txt') as f:\n",
        "    text = f.read().decode().lower()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "text[:100]"
      ],
      "metadata": {
        "id": "1Q_YPheljVtu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yfT_pvyMRT5t"
      },
      "outputs": [],
      "source": [
        "tokens = rt.tokenize(text)\n",
        "len(tokens), len(set(tokens))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tvrjUbL-RT5u"
      },
      "outputs": [],
      "source": [
        "n1 = list(nltk_ngrams(tokens, 1) )\n",
        "n2 = list(nltk_ngrams(tokens, 2))\n",
        "n3 = list(nltk_ngrams(tokens, 3))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lUlM2hjTRT5u"
      },
      "outputs": [],
      "source": [
        "laplace = lm.Laplace(order=3)\n",
        "laplace.fit([n1] + [n2] + [n3], vocabulary_text=list(set(tokens)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mwlCvgq1RT5u"
      },
      "outputs": [],
      "source": [
        "' '.join(laplace.generate(50, random_seed=42))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dg7bp7RSRT5u"
      },
      "outputs": [],
      "source": [
        "' '.join(laplace.generate(50, text_seed='вот дом который построил джек'.split(), random_seed=42))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sS0SuFQ1RT5v"
      },
      "outputs": [],
      "source": [
        "' '.join(laplace.generate(50, text_seed='привет как дела'.split(), random_seed=42))"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "name": "main.ipynb",
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}