Wav2Lip_TenDeepfake_eng.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mibrahimy/playbooks/blob/main/Wav2Lip_TenDeepfake_eng.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sE22UQtQ9YYi"
      },
      "source": [
        "# **Please use @justinjohn colab notebook which is better and actually works  https://colab.research.google.com/github/justinjohn0306/Wav2Lip/blob/master/Wav2Lip_simplified_v5.ipynb**\n",
        "\n",
        "# **There is no need for colab pro to use it.**\n",
        "[Discord](https://discord.gg/F5WzXC8vXJ)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dT9AQwdf8sJK"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yJ5taGmPcWV-"
      },
      "source": [
        "# **Get the code and models**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Qgo-oaI3JU2u",
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "94e8f443-9ef2-4460-a642-3f8229dc08cf"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Done\n"
          ]
        }
      ],
      "source": [
        "#@title <h1>Step1: Setup Wav2Lip</h1>\n",
        "#@markdown * Install dependency\n",
        "#@markdown * Download pretrained model\n",
        "!rm -rf /content/sample_data\n",
        "!mkdir /content/sample_data\n",
        "\n",
        "!git clone https://github.com/zabique/Wav2Lip\n",
        "\n",
        "#download the pretrained model\n",
        "!wget 'https://iiitaphyd-my.sharepoint.com/personal/radrabha_m_research_iiit_ac_in/_layouts/15/download.aspx?share=EdjI7bZlgApMqsVoEUUXpLsBxqXbn5z8VTmoxp55YNDcIA' -O '/content/Wav2Lip/checkpoints/wav2lip_gan.pth'\n",
        "a = !pip install https://raw.githubusercontent.com/AwaleSajil/ghc/master/ghc-1.0-py3-none-any.whl\n",
        "\n",
        "# !pip uninstall tensorflow tensorflow-gpu\n",
        "!cd Wav2Lip && pip install -r requirements.txt\n",
        "\n",
        "#download pretrained model for face detection\n",
        "!wget \"https://www.adrianbulat.com/downloads/python-fan/s3fd-619a316812.pth\" -O \"/content/Wav2Lip/face_detection/detection/sfd/s3fd.pth\"\n",
        "\n",
        "!pip install -q youtube-dl\n",
        "!pip install ffmpeg-python\n",
        "!pip install librosa==0.9.1\n",
        "\n",
        "#this code for recording audio\n",
        "\"\"\"\n",
        "To write this piece of code I took inspiration/code from a lot of places.\n",
        "It was late night, so I'm not sure how much I created or just copied o.O\n",
        "Here are some of the possible references:\n",
        "https://blog.addpipe.com/recording-audio-in-the-browser-using-pure-html5-and-minimal-javascript/\n",
        "https://stackoverflow.com/a/18650249\n",
        "https://hacks.mozilla.org/2014/06/easy-audio-capture-with-the-mediarecorder-api/\n",
        "https://air.ghost.io/recording-to-an-audio-file-using-html5-and-js/\n",
        "https://stackoverflow.com/a/49019356\n",
        "\"\"\"\n",
        "from IPython.display import HTML, Audio\n",
        "from google.colab.output import eval_js\n",
        "from base64 import b64decode\n",
        "import numpy as np\n",
        "from scipy.io.wavfile import read as wav_read\n",
        "import io\n",
        "import ffmpeg\n",
        "\n",
        "from IPython.display import clear_output\n",
        "clear_output()\n",
        "print(\"\\nDone\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vzokJMO19IyY"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RgjaWJFs8B38"
      },
      "source": [
        "# **Quick guide**\n",
        "1. Create video file named input_video.mp4 - audio track removed\n",
        "2. Create audio file named input_audio.wav\n",
        "3. Both files have to be exact same length\n",
        "4. Target face in the input_video.mp4, must be \"detectable\" in ALL videoframes (So no black or blurry frames etc)\n",
        "5. Make sure u use correct file extensions\n",
        "6. wav2lip does not like very long and high res clips (1080p/30seconds max)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qdIQfY2Kswcb"
      },
      "source": [
        "# **Now lets try!**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9bDRYsfXTzXD"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vsphzJawLF-f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 160
        },
        "outputId": "6700a71e-e87e-41a0-b78a-4abae7b7a843"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Errno 2] No such file or directory: 'sample_data/'\n",
            "/content/sample_data\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-9b6d9717-9d90-4db8-92d3-f6a17be1944a\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-9b6d9717-9d90-4db8-92d3-f6a17be1944a\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving input_audio.wav to input_audio.wav\n",
            "Saving input_video.mp4 to input_video.mp4\n",
            "/content\n"
          ]
        }
      ],
      "source": [
        "#@title 1.Upload input_video.mp4 & input_audio.wav files\n",
        "%cd sample_data/\n",
        "from google.colab import files\n",
        "uploaded = files.upload()\n",
        "%cd .."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "jR5utmDMcSZY"
      },
      "outputs": [],
      "source": [
        "#@title 2.Create Wav2Lip video (using wav2lip_gan.pth) GAN\n",
        "!cd Wav2Lip && python inference.py --checkpoint_path checkpoints/wav2lip_gan.pth --face \"/content/sample_data/input_video.mp4\" --audio \"/content/sample_data/input_audio.wav\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "colab": {
          "background_save": true
        },
        "id": "WxbzXZvLliiA"
      },
      "outputs": [],
      "source": [
        "#@title 3.Play result video -  50% scaling\n",
        "from IPython.display import HTML\n",
        "from base64 import b64encode\n",
        "mp4 = open('/content/Wav2Lip/results/result_voice.mp4','rb').read()\n",
        "data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n",
        "HTML(f\"\"\"\n",
        "<video width=\"50%\" height=\"50%\" controls>\n",
        "      <source src=\"{data_url}\" type=\"video/mp4\">\n",
        "</video>\"\"\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "1kt-krsEbz5j",
        "outputId": "4b675100-7c3b-434f-d9a2-2aeeea1a3f43"
      },
      "outputs": [
        {
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": [
              "download(\"download_d8aad7a1-02ca-4045-ab68-bf606791fa82\", \"result_voice.mp4\", 3170343)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "#@title 4.Download Result.mp4 to your computer\n",
        "from google.colab import files\n",
        "files.download('/content/Wav2Lip/results/result_voice.mp4')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "fT8njpBCJ7gD"
      },
      "outputs": [],
      "source": [
        "#@title 5. Delete old uploaded samples & result files, so you can start over again.\n",
        "%rm /content/sample_data/*\n",
        "%rm /content/Wav2Lip/results/*\n",
        "from IPython.display import clear_output\n",
        "clear_output()\n",
        "print(\"\\nDone! now press X\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BgMkHOFqT2fK"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "colab": {
          "background_save": true
        },
        "id": "2OhT0w2uT4rQ"
      },
      "outputs": [],
      "source": [
        "#@title 1-4. Batch processing - Upload -> process -> download -> play result\n",
        "%cd sample_data/\n",
        "%rm input_audio.wav\n",
        "%rm input_video.mp4\n",
        "from google.colab import files\n",
        "uploaded = files.upload()\n",
        "%cd ..\n",
        "!cd Wav2Lip && python inference.py --checkpoint_path checkpoints/wav2lip_gan.pth --face \"/content/sample_data/input_video.mp4\" --audio \"/content/sample_data/input_audio.wav\"\n",
        "from google.colab import files\n",
        "files.download('/content/Wav2Lip/results/result_voice.mp4')\n",
        "from IPython.display import HTML\n",
        "from base64 import b64encode\n",
        "mp4 = open('/content/Wav2Lip/results/result_voice.mp4','rb').read()\n",
        "data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n",
        "HTML(f\"\"\"\n",
        "<video width=\"50%\" height=\"50%\" controls>\n",
        "      <source src=\"{data_url}\" type=\"video/mp4\">\n",
        "</video>\"\"\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q9NvwrJ3vRcs"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d7zgfrQqbKom"
      },
      "source": [
        "# **Variations to try**\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "xw0xFtZ2bsx8"
      },
      "outputs": [],
      "source": [
        "#@title 2.Use resize_factor to reduce the video resolution, as there is a change you might get better results for lower resolution videos. Why? Because the model was trained on low resolution faces.\n",
        "!cd Wav2Lip && python inference.py --checkpoint_path checkpoints/wav2lip_gan.pth --face \"/content/sample_data/input_video.mp4\" --audio \"/content/sample_data/input_audio.wav\" --resize_factor 2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "45XW4SZAzIz5"
      },
      "outputs": [],
      "source": [
        "#@title 3.Use more padding to include the chin region (u can manually edit pads dimensions viewing and changing the code)\n",
        "!cd Wav2Lip && python inference.py --checkpoint_path checkpoints/wav2lip_gan.pth --face \"/content/sample_data/input_video.mp4\" --audio \"/content/sample_data/input_audio.wav\" --pads 0 20 0 0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "colab": {
          "background_save": true
        },
        "id": "X1Z0zRdZR5BZ"
      },
      "outputs": [],
      "source": [
        "#@title 4.Play result video -  50% scaling\n",
        "from IPython.display import HTML\n",
        "from base64 import b64encode\n",
        "mp4 = open('/content/Wav2Lip/results/result_voice.mp4','rb').read()\n",
        "data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n",
        "HTML(f\"\"\"\n",
        "<video width=\"50%\" height=\"50%\" controls>\n",
        "      <source src=\"{data_url}\" type=\"video/mp4\">\n",
        "</video>\"\"\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "gfXFOpAmR_dh"
      },
      "outputs": [],
      "source": [
        "#@title 5.Download Result.mp4 to your computer\n",
        "from google.colab import files\n",
        "files.download('/content/Wav2Lip/results/result_voice.mp4')\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}