example_api.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First start the server:\n",
    "# python run_api.py --model unsloth/Llama-3.2-1B-Instruct\n",
    "# \n",
    "# Or with default slop phrase list (will be used with all queries unless slop_phrases is specified in the query):\n",
    "# python run_api.py --model unsloth/Llama-3.2-1B-Instruct --slop_adjustments_file slop_phrase_prob_adjustments.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! Important:\n",
    "# \n",
    "# No slop adjustments will be applied if:\n",
    "#  - no slop_adjustments_file specified at API launch \n",
    "#  and\n",
    "#  - no slop_phrases param specified in the query\n",
    "#\n",
    "# If you specified a slop_adjustments_file file at API launch, it will be used by default with queries\n",
    "# that do not specify a slop_phrases param. The query's slop_phrases param overrides the defaults."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Once upon a time, in a small village nestled in the Andes mountains of South America, there lived a young boy named Kusi. Kusi was a curious and adventurous soul, with a heart full of wonder and a spirit that longed to explore the world beyond his village. He spent most of his days helping his mother with the family's small farm, tending to the crops and animals, and listening to the tales of the elderly villagers about the mystical creatures that roamed the mountains.\n",
      "\n",
      "One day, while out collecting firewood, Kusi stumbled upon a strange and beautiful creature. It was a young ladyma (a type of Andean camelid) unlike any he had ever seen before. Her fur was a soft, creamy white, and her eyes shone the brightest stars in the night sky. She had a delicate, almost ethereal quality to her, and Kusi felt an instant connection to this enchanting creature.\n",
      "\n",
      "The ladyma, whose name was Akira,"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import json\n",
    "\n",
    "prompt = \"tell me a story about a magical llama\"\n",
    "api_url = 'http://localhost:8000/v1/chat/completions'\n",
    "\n",
    "messages = [{\"role\": \"user\", \"content\": prompt}]\n",
    "data = {\n",
    "    \"messages\": messages,    \n",
    "    \"max_tokens\": 200,\n",
    "    \"temperature\": 1,\n",
    "    \"min_p\": 0.1,\n",
    "    \"stream\": True,\n",
    "    \"adjustment_strength\": 100,\n",
    "    \"antislop_enabled\": True, # Defaults to true\n",
    "    \"slop_phrases\": [[\"a testament to\", 0.3], [\"llama\", 0.1]] # this overrides the default list\n",
    "}\n",
    "\n",
    "try:\n",
    "    # Using `stream=True` to handle the response as a stream of data\n",
    "    response = requests.post(api_url, json=data, stream=True, timeout=30)\n",
    "    #print(response)\n",
    "    \n",
    "    # Read and print the stream of data in chunks\n",
    "    for chunk in response.iter_lines():\n",
    "        if chunk:\n",
    "            decoded_chunk = chunk.decode('utf-8')\n",
    "            # OpenAI streams responses in `data: {json}` format, so we need to parse that\n",
    "            if decoded_chunk.startswith('data:'):\n",
    "                try:\n",
    "                    json_data = json.loads(decoded_chunk[len('data: '):])  # Remove 'data: ' prefix\n",
    "                    if 'choices' in json_data and len(json_data['choices']) > 0:\n",
    "                        print(json_data['choices'][0]['delta'].get('content', ''), end='', flush=True)\n",
    "                except json.JSONDecodeError as e:\n",
    "                    print(f\"Error decoding JSON: {e}\")\n",
    "except Exception as e:\n",
    "    print(f\"An error occurred: {e}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "In the small village of Brindlemark, nestled in the rolling hills of the countryside, there lived a young woman named Eira. She was known throughout the village as the loomweaver, Eira, for her extraordinary talent in weaving intricate patterns with the threads of the finest wool.\n",
      "\n",
      "Eira's fascination with weaving began when she was just a child, watching her mother and grandmother work the loom in the evenings. The soft rustle of the wool against the fabric, the gentle hum of the shuttle, and the way the colors seemed to come alive as the loom wove its magic – all of these things captivated Eira, and she spent countless hours practicing and experimenting with different techniques.\n",
      "\n",
      "As she grew older, Eira's skills improved dramatically, and she became one of the most sought-after weavers in the region. Her creations were renowned for their beauty and complexity, with intricate patterns and delicate colors that seemed to dance across the fabric.\n",
      "\n",
      "But Eira's life\n"
     ]
    }
   ],
   "source": [
    "# Example using regex bans\n",
    "prompt = \"write a story about the loomweaver elara\"\n",
    "messages = [{\"role\": \"user\", \"content\": prompt}]\n",
    "try:\n",
    "    data = {    \n",
    "        \"messages\": messages,    \n",
    "        \"max_tokens\": 200,\n",
    "        \"temperature\": 1,\n",
    "        \"min_p\": 0.1,\n",
    "        \"stream\": False,\n",
    "        \"adjustment_strength\": 100,\n",
    "        \"slop_phrases\": [[\"a testament to\", 0.3], [\"kusi\", 0.1]],\n",
    "        \"antislop_enabled\": True,\n",
    "        \"regex_bans\": ['(?i)not [^.!?]{3,60} but', '(?i)elara'] # Not compatible with streaming\n",
    "    }\n",
    "    \n",
    "    response = requests.post(api_url, json=data, stream=False, timeout=30)\n",
    "    data = response.json()\n",
    "    print(data['choices'][0]['message']['content'])\n",
    "\n",
    "except Exception as e:\n",
    "    print(f\"An error occurred: {e}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "In the heart of a mystical Andean village, there lived a legend about a magical llama named Luna. Her coat shone like the moon, and her eyes glowed with an ethereal light that seemed to hold the secrets of the universe. Luna was no ordinary llama – she possessed the power to heal the sick, grant wisdom to those who sought it, and even bend the fabric of time to bring peace to those who were troubled.\n",
      "\n",
      "One day, a young girl named Sophia wandered into the village, searching for her ailing mother. Sophia's mother had fallen gravely ill, and the village healer had exhausted all his remedies. Desperate for a solution, Sophia set out to find the legendary magical llama, Luna.\n",
      "\n",
      "As she trekked through the high-altitude grasslands, Sophia encountered many creatures – the majestic condors soaring above, the chattering vicuñas below, and the gentle guanacos grazing in the distance. But none of them led her to Luna, until finally"
     ]
    }
   ],
   "source": [
    "# Default slop list example\n",
    "#\n",
    "# If the user hasn't specified a list of slop phrases & adjustments in their \n",
    "# query, then it will default to use whatever you specified in the\n",
    "# --slop_adjustments_file argument when launching the api.\n",
    "#\n",
    "# If you didn't specify anything, it won't make any adjustments unless the\n",
    "# user specifies their adjustments in the query.\n",
    "\n",
    "prompt = \"tell me a story about a magical llama\"\n",
    "api_url = 'http://localhost:8000/v1/chat/completions'\n",
    "\n",
    "messages = [{\"role\": \"user\", \"content\": prompt}]\n",
    "data = {    \n",
    "    \"messages\": messages,    \n",
    "    \"max_tokens\": 200,\n",
    "    \"temperature\": 1,\n",
    "    \"min_p\": 0.1,\n",
    "    \"stream\": True,\n",
    "    \"antislop_enabled\": True,\n",
    "    \"adjustment_strength\": 100,\n",
    "}\n",
    "\n",
    "try:\n",
    "    # Using `stream=True` to handle the response as a stream of data\n",
    "    response = requests.post(api_url, json=data, stream=True, timeout=30)\n",
    "    #print(response)\n",
    "    \n",
    "    # Read and print the stream of data in chunks\n",
    "    for chunk in response.iter_lines():\n",
    "        if chunk:\n",
    "            decoded_chunk = chunk.decode('utf-8')\n",
    "            # OpenAI streams responses in `data: {json}` format, so we need to parse that\n",
    "            if decoded_chunk.startswith('data:'):\n",
    "                try:\n",
    "                    json_data = json.loads(decoded_chunk[len('data: '):])  # Remove 'data: ' prefix\n",
    "                    if 'choices' in json_data and len(json_data['choices']) > 0:\n",
    "                        print(json_data['choices'][0]['delta'].get('content', ''), end='', flush=True)\n",
    "                except json.JSONDecodeError as e:\n",
    "                    print(f\"Error decoding JSON: {e}\")\n",
    "except Exception as e:\n",
    "    print(f\"An error occurred: {e}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}