SemanticSearch1k.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe5cea5e-9bfc-469a-ae51-5bbf21c5df33",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time\n",
    "from redis import Redis\n",
    "from redis.commands.search.field import VectorField\n",
    "from redis.commands.search.field import TextField\n",
    "from redis.commands.search.field import TagField\n",
    "from redis.commands.search.query import Query\n",
    "from redis.commands.search.result import Result\n",
    "\n",
    "class color:\n",
    "   PURPLE = '\\033[95m'\n",
    "   CYAN = '\\033[96m'\n",
    "   DARKCYAN = '\\033[36m'\n",
    "   BLUE = '\\033[94m'\n",
    "   GREEN = '\\033[92m'\n",
    "   YELLOW = '\\033[93m'\n",
    "   RED = '\\033[91m'\n",
    "   BOLD = '\\033[1m'\n",
    "   UNDERLINE = '\\033[4m'\n",
    "   END = '\\033[0m'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba969e44-d988-43bd-9953-73bdc3c19a1c",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Load Amazon Product Data\n",
    "\n",
    "Truncate text selected fields on load."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc187a4a-ef97-41fc-92c7-d0c34eb5ada0",
   "metadata": {},
   "outputs": [],
   "source": [
    "NUMBER_PRODUCTS=1000\n",
    "\n",
    "#Load Product data and truncate long text fields\n",
    "all_prods_df = pd.read_csv(\"data/product_data.csv\")\n",
    "all_prods_df['primary_key'] = all_prods_df['item_id'] + '-' + all_prods_df['domain_name']\n",
    "all_prods_df['item_keywords'].replace('', np.nan, inplace=True)\n",
    "all_prods_df.dropna(subset=['item_keywords'], inplace=True)\n",
    "all_prods_df.reset_index(drop=True,inplace=True)\n",
    "\n",
    "#get the first 1000 products with non-empty item keywords\n",
    "product_metadata = all_prods_df.head(NUMBER_PRODUCTS).to_dict(orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50c78cdd-49f6-40da-a391-eeb1d2af8139",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_prods_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2261b86-598b-4ab4-be93-e58f34aff3dd",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Connect to Redis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef37492f-3360-4cd5-8dd2-23ccdd2b9042",
   "metadata": {},
   "outputs": [],
   "source": [
    "host = 'vecsim'\n",
    "port = 6379\n",
    "redis_conn = Redis(host = host, port = port)\n",
    "print ('Connected to redis')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fde0ad07-363e-4414-8684-bbf6c1d13409",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Generate Embeddings\n",
    "\n",
    "We will use a pre-trained sentence embedding generator from\n",
    "\n",
    "https://huggingface.co/sentence-transformers/all-distilroberta-v1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dbf1ccf-1d3a-4f8b-9457-a9fb648859dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "model = SentenceTransformer('sentence-transformers/all-distilroberta-v1')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5298c8d-3b69-4927-93b5-1981ee729f60",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Generate Embeddings\n",
    "\n",
    "We will use a Sentence Transformer model (distilroberta-v1 ) 'Img2Vec' to generate embeddings (vectors) for item keywords \n",
    "\n",
    "The distilroberta-v1 generates a 768-float vector for a given sentence of up to 512 characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78834b6f-c383-453d-bd35-e0613c7274ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "item_keywords =  [product_metadata[i]['item_keywords']  for i in product_metadata.keys()]\n",
    "item_keywords_vectors = [ model.encode(sentence) for sentence in item_keywords]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39427b16-dc63-45f9-8a2f-e032d8cb010f",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Check the dimensions of one of the vectors generated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6593534b-9a81-4da3-ae71-7492ef343789",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(item_keywords_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a408f6da-1202-4f23-95b7-a775f7f8f562",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(product_metadata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70259445-b83d-4e93-8afd-2443dc15a1c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check one of the products\n",
    "product_metadata[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbc7f4d2-1c1a-4847-bf6d-b08610b30c01",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Utility Functions to Load Product Data\n",
    "Each product will be stored in a redis hash\n",
    "* **Hash Key** = **key='product:index+:primary_key'**\n",
    "\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e86b393b-2273-4f26-8779-73896b43c1f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_vectors(client:Redis, product_metadata, vector_dict, vector_field_name):\n",
    "    p = client.pipeline(transaction=False)\n",
    "    for index in product_metadata.keys():    \n",
    "        #hash key\n",
    "        key='product:'+ str(index)+ ':' + product_metadata[index]['primary_key']\n",
    "        \n",
    "        #hash values\n",
    "        item_metadata = product_metadata[index]\n",
    "        item_keywords_vector = vector_dict[index].astype(np.float32).tobytes()\n",
    "        item_metadata[vector_field_name]=item_keywords_vector\n",
    "        \n",
    "        # HSET\n",
    "        p.hset(key,mapping=item_metadata)\n",
    "            \n",
    "    p.execute()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5607848c-ce11-44d2-892f-1ffb4778f405",
   "metadata": {},
   "source": [
    "# Utility Functions to Create Indexes on Vector field"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54a6d4e3-b15d-499e-864d-c420fe140810",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_flat_index (redis_conn,vector_field_name,number_of_vectors, vector_dimensions=512, distance_metric='L2'):\n",
    "    redis_conn.ft().create_index([\n",
    "        VectorField(vector_field_name, \"FLAT\", {\"TYPE\": \"FLOAT32\", \"DIM\": vector_dimensions, \"DISTANCE_METRIC\": distance_metric, \"INITIAL_CAP\": number_of_vectors, \"BLOCK_SIZE\":number_of_vectors }),\n",
    "        TagField(\"product_type\"),\n",
    "        TextField(\"item_name\"),\n",
    "        TextField(\"item_keywords\"),\n",
    "        TagField(\"country\")        \n",
    "    ])\n",
    "\n",
    "def create_hnsw_index (redis_conn,vector_field_name,number_of_vectors, vector_dimensions=512, distance_metric='L2',M=40,EF=200):\n",
    "    redis_conn.ft().create_index([\n",
    "        VectorField(vector_field_name, \"HNSW\", {\"TYPE\": \"FLOAT32\", \"DIM\": vector_dimensions, \"DISTANCE_METRIC\": distance_metric, \"INITIAL_CAP\": number_of_vectors, \"M\": M, \"EF_CONSTRUCTION\": EF}),\n",
    "        TagField(\"product_type\"),\n",
    "        TextField(\"item_keywords\"),        \n",
    "        TextField(\"item_name\"),\n",
    "        TagField(\"country\")     \n",
    "    ])    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ab8085f-b70b-4e8f-831a-21a78d7273a1",
   "metadata": {
    "tags": []
   },
   "source": [
    "# FLAT - Load and Index Product Data\n",
    "Let's create an index for the image vectors and load information for 1000 products\n",
    "\n",
    "**This might take 1-2 minutes**\n",
    "\n",
    "A FLAT index is used to perform an exact nearest neighbors search. \n",
    "\n",
    "A query vector will be compared against all other image vectors in the database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71a27ca3-5ce0-4174-aac1-86ffa6224070",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "ITEM_KEYWORD_EMBEDDING_FIELD='item_keyword_vector'\n",
    "TEXT_EMBEDDING_DIMENSION=768\n",
    "NUMBER_PRODUCTS=1000\n",
    "\n",
    "print ('Loading and Indexing + ' +  str(NUMBER_PRODUCTS) + ' products')\n",
    "\n",
    "#flush all data\n",
    "redis_conn.flushall()\n",
    "\n",
    "#create flat index & load vectors\n",
    "create_flat_index(redis_conn, ITEM_KEYWORD_EMBEDDING_FIELD,NUMBER_PRODUCTS,TEXT_EMBEDDING_DIMENSION,'COSINE')\n",
    "load_vectors(redis_conn,product_metadata,item_keywords_vectors,ITEM_KEYWORD_EMBEDDING_FIELD)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bcdd5a01-5736-4081-b3e4-1551bbb9bd0f",
   "metadata": {},
   "source": [
    "# FLAT index - FIND The Top K MOST SEMANTICALLY Similar Products\n",
    "Let's use the brute-force index to find the exact top k nearest neighbors of a given text query\n",
    "\n",
    "Check the output for 2 very different queries:\n",
    "* Query 1 = 'beautifully crafted present for her. a special occasion'\n",
    "* Query 2 = 'Ultra modern cool way to pimp up my phone'\n",
    "\n",
    "Feel free to experiment with other text queries to match against the item keyword data.\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "315e6873-5e0f-4924-a64d-8df14f00cdc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "topK=5\n",
    "product_query='beautifully crafted present for her. a special occasion'\n",
    "#product_query='cool way to pimp up my cell'\n",
    "\n",
    "#vectorize the query\n",
    "query_vector = model.encode(product_query).astype(np.float32).tobytes()\n",
    "\n",
    "#prepare the query\n",
    "q = Query(f'*=>[KNN {topK} @{ITEM_KEYWORD_EMBEDDING_FIELD} $vec_param AS vector_score]').sort_by('vector_score').paging(0,topK).return_fields('vector_score','item_name','item_id','item_keywords').dialect(2)\n",
    "params_dict = {\"vec_param\": query_vector}\n",
    "\n",
    "\n",
    "#Execute the query\n",
    "results = redis_conn.ft().search(q, query_params = params_dict)\n",
    "\n",
    "#Print similar products found\n",
    "for product in results.docs:\n",
    "    print ('***************Product  found ************')\n",
    "    print (color.BOLD + 'hash key = ' +  color.END + product.id)\n",
    "    print (color.YELLOW + 'Item Name = ' +  color.END  + product.item_name)\n",
    "    print (color.YELLOW + 'Item Id = ' +  color.END  + product.item_id)\n",
    "    print (color.YELLOW + 'Item keywords = ' +  color.END  + product.item_keywords)\n",
    "    print (color.YELLOW + 'Score = ' +  color.END  + product.vector_score)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e748638-50be-4e4f-b953-a87b5fdea311",
   "metadata": {},
   "source": [
    "# HNSW - Load and Index Product Data\n",
    "Let's try indexing with an HNSW index.\n",
    "This index is used to calculate Top K Approximate Nearest Neighbors of a given vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "551253af-d833-44b3-97dc-4c0e8dd4ba14",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "print ('Loading and Indexing + ' +  str(NUMBER_PRODUCTS) + ' products')\n",
    "\n",
    "ITEM_KEYWORD_EMBEDDING_FIELD='item_keyword_vector'\n",
    "NUMBER_PRODUCTS=1000\n",
    "TEXT_EMBEDDING_DIMENSION=768\n",
    "\n",
    "\n",
    "\n",
    "#flush all data\n",
    "redis_conn.flushall()\n",
    "\n",
    "#create flat index & load vectors\n",
    "create_hnsw_index(redis_conn, ITEM_KEYWORD_EMBEDDING_FIELD,NUMBER_PRODUCTS,TEXT_EMBEDDING_DIMENSION,'COSINE',M=40,EF=200)\n",
    "load_vectors(redis_conn,product_metadata,item_keywords_vectors,ITEM_KEYWORD_EMBEDDING_FIELD)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff708fb3-2926-4201-ab18-d1a6193f569b",
   "metadata": {},
   "source": [
    "# HNSW - Query The Top 5 semantically Similar Products\n",
    "Let's repeat the similarity search but this time using the HNSW index\n",
    "\n",
    "Check the output for 2 very different queries:\n",
    "* Query 1 = 'beautifully crafted present for her. a special occasion'\n",
    "* Query 2 = 'Ultra modern cool way to pimp up my cell'\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3eeb687a-2f75-458f-b75c-0a10057a163d",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "topK=5\n",
    "product_query='beautifully crafted present for her. a special occasion'\n",
    "#product_query='cool way to pimp up my cell'\n",
    "\n",
    "#vectorize the query\n",
    "query_vector = model.encode(product_query).astype(np.float32).tobytes()\n",
    "\n",
    "#prepare the query\n",
    "q = Query(f'*=>[KNN {topK} @{ITEM_KEYWORD_EMBEDDING_FIELD} $vec_param AS vector_score]').sort_by('vector_score').paging(0,topK).return_fields('vector_score','item_name','item_id','item_keywords','country').dialect(2)\n",
    "params_dict = {\"vec_param\": query_vector}\n",
    "\n",
    "\n",
    "#Execute the query\n",
    "results = redis_conn.ft().search(q, query_params = params_dict)\n",
    "\n",
    "#Print similar products found\n",
    "for product in results.docs:\n",
    "    print ('***************Product  found ************')\n",
    "    print (color.BOLD + 'hash key = ' +  color.END + product.id)\n",
    "    print (color.YELLOW + 'Item Name = ' +  color.END  + product.item_name)\n",
    "    print (color.YELLOW + 'Item Id = ' +  color.END  + product.item_id)\n",
    "    print (color.YELLOW + 'Item keywords = ' +  color.END  + product.item_keywords)\n",
    "    print (color.YELLOW + 'Country = ' +  color.END  + product.country)\n",
    "    print (color.YELLOW + 'Score = ' +  color.END  + product.vector_score)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca26e65b-cd0f-4fc7-bfab-099c6edfa9f4",
   "metadata": {},
   "source": [
    "# HNSW - Hybrid Query the top 5 most visually semantically products available ONLY in selected markets\n",
    "\n",
    "Let's repeat our Top 5 search but this time limit to products that meet the following criteria:\n",
    "* **Listed on** Amazon Germany (DE), India (IN) or Italy (IT)\n",
    "\n",
    "\n",
    "This RediSearch query has this form:\n",
    "\n",
    "**(@country:{{DE|IN|IT}})=> [KNN 5 vector_field_name $query_vector EF_RUNTIME 10 AS vector_score])**\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c330ab4e-0d8a-4872-842d-2f0718039f83",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "topK=5\n",
    "product_query='beautifully crafted carpets for a special occasion'\n",
    "product_query='cool way to pimp up my cell'\n",
    "\n",
    "#vectorize the query\n",
    "query_vector = model.encode(product_query).astype(np.float32).tobytes()\n",
    "\n",
    "#prepare the query\n",
    "q = Query(f'(@country:{{DE|IN|IT}})=>[KNN {topK} @{ITEM_KEYWORD_EMBEDDING_FIELD} $vec_param AS vector_score]').sort_by('vector_score').paging(0,topK).return_fields('vector_score','item_name','item_id','item_keywords','country').dialect(2)\n",
    "params_dict = {\"vec_param\": query_vector}\n",
    "\n",
    "\n",
    "#Execute the query\n",
    "results = redis_conn.ft().search(q, query_params = params_dict)\n",
    "\n",
    "#Print similar products found\n",
    "for product in results.docs:\n",
    "    print ('***************Product  found ************')\n",
    "    print (color.BOLD + 'hash key = ' +  color.END + product.id)\n",
    "    print (color.YELLOW + 'Item Name = ' +  color.END  + product.item_name)\n",
    "    print (color.YELLOW + 'Item Id = ' +  color.END  + product.item_id)\n",
    "    print (color.YELLOW + 'Item keywords = ' +  color.END  + product.item_keywords)\n",
    "    print (color.YELLOW + 'Score = ' +  color.END  + product.vector_score)\n",
    "    print (color.YELLOW + 'Country = ' +  color.END  + product.country)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "306c89af-e093-4d35-888b-19ae63a058d9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}