VectorMemoryManagement.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyO663BiFrygy06LNkKg7bd7",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/IkuStudies/abraxalexicon/blob/main/VectorMemoryManagement.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#prepare the files for vector database\n",
        "\n",
        "import pandas as pd\n",
        "\n",
        "# Read the input files as plain text files\n",
        "with open('input_file1.csv', 'r') as f1, open('input_file2.csv', 'r') as f2:\n",
        "    lines_words = f1.readlines()\n",
        "    lines_phonemes = f2.readlines()\n",
        "\n",
        "# Remove any leading or trailing whitespace from each line\n",
        "lines_words = [line.strip() for line in lines_words]\n",
        "lines_phonemes = [line.strip() for line in lines_phonemes]\n",
        "\n",
        "# Combine the words and phonemes into a single dataframe\n",
        "df_combined = pd.DataFrame({'word': lines_words, 'phoneme': lines_phonemes})\n",
        "\n",
        "# Prepare the data for Milvus\n",
        "data = df_combined.apply(lambda row: [row['word'], row['phoneme']], axis=1).tolist()\n",
        "\n",
        "# Save the formatted data to a file\n",
        "with open('formatted_data.csv', 'w') as f:\n",
        "    for row in data:\n",
        "        f.write(','.join(row) + '\\n')\n"
      ],
      "metadata": {
        "id": "cPVoysZPvdf2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NvUEwTLBvNah"
      },
      "outputs": [],
      "source": [
        "#install, configure, and initiate milvus\n",
        "\n",
        "!pip install pymilvus==2.0.0\n",
        "!pip install ujson==4.0.2\n",
        "!pip install python-rapidjson==1.4\n",
        "\n",
        "import os\n",
        "\n",
        "# Set the username and password for Milvus\n",
        "username = \"josh\"\n",
        "password = \"password\"\n",
        "\n",
        "# Install and start Milvus\n",
        "!wget https://milvus.io/download/milvus-2.0.0-ubuntu20.04.tar.gz\n",
        "!tar -xvf milvus-2.0.0-ubuntu20.04.tar.gz\n",
        "%cd milvus\n",
        "!bash install_milvus.sh\n",
        "!bash start_milvus.sh\n",
        "\n",
        "# Configure Milvus\n",
        "!milvus config set server_addr=localhost:19530\n",
        "!milvus config set username={username}\n",
        "!milvus config set password={password}\n",
        "!milvus config show\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import csv\n",
        "from milvus import Milvus, DataType, FieldSchema, CollectionSchema\n",
        "\n",
        "# Connect to Milvus\n",
        "milvus = Milvus()\n",
        "milvus.connect(host='localhost', port='19530')\n",
        "\n",
        "# Define collection and field names\n",
        "collection_name = 'my_collection'\n",
        "word_field = 'word'\n",
        "phoneme_field = 'phoneme'\n",
        "\n",
        "# Create collection schema\n",
        "collection_schema = CollectionSchema(collection_name, fields=[\n",
        "    FieldSchema(name=word_field, dtype=DataType.STRING),\n",
        "    FieldSchema(name=phoneme_field, dtype=DataType.STRING),\n",
        "])\n",
        "\n",
        "# Create the collection\n",
        "milvus.create_collection(collection_schema)\n",
        "\n",
        "# Load data from formatted_data.csv\n",
        "with open('formatted_data.csv', 'r') as f:\n",
        "    reader = csv.reader(f)\n",
        "    entities = []\n",
        "    for row in reader:\n",
        "        word = row[0]\n",
        "        phoneme = row[1]\n",
        "        entity = {\n",
        "            word_field: word,\n",
        "            phoneme_field: phoneme\n",
        "        }\n",
        "        entities.append(entity)\n",
        "\n",
        "# Insert the entities into the collection\n",
        "milvus.insert(collection_name, entities)\n",
        "\n",
        "# Flush the inserted data to disk\n",
        "milvus.flush([collection_name])\n",
        "\n",
        "# Disconnect from Milvus\n",
        "milvus.disconnect()\n"
      ],
      "metadata": {
        "id": "nPb-nddQzQ3R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#export finished vector memory files\n",
        "\n",
        "from milvus import Milvus\n",
        "\n",
        "# Connect to Milvus\n",
        "milvus = Milvus()\n",
        "milvus.connect(host='localhost', port='19530')\n",
        "\n",
        "# Get a list of all collections\n",
        "collections = milvus.list_collections()\n",
        "\n",
        "# Export each collection as a separate file\n",
        "for collection_name in collections:\n",
        "    output_file = f\"{collection_name}.csv\"\n",
        "    milvus.export(collection_name, output_file)\n",
        "\n",
        "# Disconnect from Milvus\n",
        "milvus.disconnect()\n"
      ],
      "metadata": {
        "id": "5l6leban0ei0"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}