refactor(runner): update LLM 'tokens_used' to usage (#402)

This commit updates the `tokens_used` field in the LLM API spec with `usage` to make it more compatible with the OpenAI spec.
livepeer · Jan 27, 2025 · a11302b · a11302b
1 parent cd56616
commit a11302b
Show file tree

Hide file tree

Showing 8 changed files with 2,732 additions and 96 deletions.
diff --git a/gateway.openapi.yaml b/gateway.openapi.yaml
diff --git a/openapi.yaml b/openapi.yaml
diff --git a/runner/app/pipelines/llm.py b/runner/app/pipelines/llm.py
@@ -231,7 +231,7 @@ async def generate(
                                 index=0
                             )
                         ],
-                        tokens_used=LLMTokenUsage(
+                        usage=LLMTokenUsage(
                             prompt_tokens=input_tokens,
                             completion_tokens=total_tokens,
                             total_tokens=input_tokens + total_tokens
@@ -266,7 +266,7 @@ async def generate(
                         finish_reason="stop"
                     )
                 ],
-                tokens_used=LLMTokenUsage(
+                usage=LLMTokenUsage(
                     prompt_tokens=input_tokens,
                     completion_tokens=total_tokens,
                     total_tokens=input_tokens + total_tokens

diff --git a/runner/app/routes/llm.py b/runner/app/routes/llm.py
@@ -89,7 +89,7 @@ async def llm(
                             finish_reason="stop"
                         )
                     ],
-                    tokens_used=last_chunk.tokens_used,
+                    usage=last_chunk.usage,
                     id=last_chunk.id,
                     model=last_chunk.model,
                     created=last_chunk.created

diff --git a/runner/app/routes/utils.py b/runner/app/routes/utils.py
@@ -95,7 +95,7 @@ class LLMResponse(BaseModel):
     id: str
     model: str
     created: int
-    tokens_used: LLMTokenUsage
+    usage: LLMTokenUsage
     choices: List[LLMChoice]
 
 

diff --git a/runner/gateway.openapi.yaml b/runner/gateway.openapi.yaml
@@ -946,7 +946,7 @@ components:
         created:
           type: integer
           title: Created
-        tokens_used:
+        usage:
           $ref: '#/components/schemas/LLMTokenUsage'
         choices:
           items:
@@ -958,7 +958,7 @@ components:
       - id
       - model
       - created
-      - tokens_used
+      - usage
       - choices
       title: LLMResponse
     LLMTokenUsage:

diff --git a/runner/openapi.yaml b/runner/openapi.yaml
@@ -1092,7 +1092,7 @@ components:
         created:
           type: integer
           title: Created
-        tokens_used:
+        usage:
           $ref: '#/components/schemas/LLMTokenUsage'
         choices:
           items:
@@ -1104,7 +1104,7 @@ components:
       - id
       - model
       - created
-      - tokens_used
+      - usage
       - choices
       title: LLMResponse
     LLMTokenUsage:

diff --git a/worker/runner.gen.go b/worker/runner.gen.go