Add C API and C# interop files

This change adds a basic C API that allows access to Gemma functionality from other programming languages. The functionality is exposed via a shared library (DLL on Windows), with C++ interfaces and a basic C# interop wrapper included.

To build the DLL, use the `windows-dll` preset, which includes the C and C++ sources as follows:
```
cmake --preset windows-dll
cmake --build --config Release --preset windows-dll -j 4
```
This should generate a `gemma.dll` in `<build-dir>/Release`.

To build for non-Windows, the appropriate C++ DLL linking will need to be done to generate a shared library for the target OS.

PiperOrigin-RevId: 747940577

Author: The gemma.cpp Authors

BUILD.bazel

@@ -428,6 +428,40 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "gemma_shared_lib",
+    srcs = [
+        "gemma/bindings/c_api.cc",
+        "gemma/bindings/context.cc",
+    ],
+    hdrs = [
+        "gemma/bindings/c_api.h",
+        "gemma/bindings/context.h",
+    ],
+    exec_properties = {
+        # Avoid linker OOMs when building with sanitizer instrumentation.
+        "mem": "28g",
+    },
+    deps = [
+        ":allocator",
+        ":basics",
+        ":benchmark_helper",
+        ":common",
+        ":gemma_args",
+        ":gemma_lib",
+        ":kv_cache",
+        ":mat",
+        ":ops",
+        ":threading",
+        ":threading_context",
+        ":tokenizer",
+        ":weights",
+        "//compression:shared",
+        "//paligemma:image",
+        "@highway//:hwy",
+    ],
+)
+
 cc_library(
     name = "cross_entropy",
     srcs = ["evals/cross_entropy.cc"],
@@ -465,6 +499,7 @@ cc_library(
         ":gemma_lib",
         ":ops",
         ":threading_context",
+        ":tokenizer",
         "@google_benchmark//:benchmark",
         "//compression:compress",
         "@highway//:hwy",
@@ -522,6 +557,7 @@ cc_binary(
         ":gemma_lib",
         ":ops",
         ":threading_context",
+        ":tokenizer",
         "//compression:shared",
         "//paligemma:image",
         "@highway//:hwy",

CMakeLists.txt

@@ -39,6 +39,7 @@ set(BENCHMARK_ENABLE_GTEST_TESTS OFF)
 FetchContent_Declare(benchmark GIT_REPOSITORY https://github.com/google/benchmark.git GIT_TAG v1.8.2 EXCLUDE_FROM_ALL)
 FetchContent_MakeAvailable(benchmark)
 
+# Base source files
 set(SOURCES
   compression/blob_store.cc
   compression/blob_store.h
@@ -115,6 +116,17 @@ set(SOURCES
   util/topology.h
 )
 
+# Add C API sources only when building DLL
+if(BUILD_GEMMA_DLL)
+  list(APPEND SOURCES
+    gemma/bindings/context.h
+    gemma/bindings/context.cc
+    gemma/bindings/c_api.h
+    gemma/bindings/c_api.cc
+  )
+  message(STATUS "Including C API files for DLL build")
+endif()
+
 if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE "Release")
 endif()
@@ -134,6 +146,33 @@ target_compile_definitions(libgemma PRIVATE $<$<PLATFORM_ID:Windows>:_CRT_SECURE
 target_compile_options(libgemma PRIVATE $<$<PLATFORM_ID:Windows>:-Wno-deprecated-declarations>)
 install(TARGETS libgemma DESTINATION lib)
 
+# Shared library target for C# interop
+if(BUILD_GEMMA_DLL)
+    add_library(gemma_shared SHARED ${SOURCES})
+set_property(TARGET gemma_shared PROPERTY CXX_STANDARD 17)
+set_target_properties(gemma_shared PROPERTIES 
+    PREFIX ""
+    OUTPUT_NAME "gemma"
+)
+set_property(TARGET gemma_shared PROPERTY POSITION_INDEPENDENT_CODE ON)
+target_include_directories(gemma_shared PUBLIC ./)
+target_link_libraries(gemma_shared PRIVATE 
+    $<LINK_LIBRARY:WHOLE_ARCHIVE,hwy>
+    $<LINK_LIBRARY:WHOLE_ARCHIVE,hwy_contrib>
+    $<LINK_LIBRARY:WHOLE_ARCHIVE,sentencepiece-static>
+)
+target_include_directories(gemma_shared PUBLIC ${sentencepiece_SOURCE_DIR})
+target_compile_definitions(gemma_shared 
+    PRIVATE 
+    GEMMA_EXPORTS
+    $<$<PLATFORM_ID:Windows>:_CRT_SECURE_NO_WARNINGS NOMINMAX>
+)
+target_compile_options(gemma_shared PRIVATE $<$<PLATFORM_ID:Windows>:-Wno-deprecated-declarations>)
+install(TARGETS gemma_shared DESTINATION lib)
+install(FILES gemma/c_api.h DESTINATION include/gemma)
+install(FILES gemma/GemmaInterop.cs DESTINATION include/gemma)
+endif()
+
 # Executable Target
 
 add_executable(gemma gemma/run.cc)

CMakePresets.json

@@ -31,6 +31,24 @@
           "lhs": "${hostSystemName}",
           "rhs": "Windows"
         }
+      },
+      {
+        "name": "windows-dll",
+        "inherits": "__defaults__",
+        "displayName": "Windows DLL",
+        "description": "Visual Studio 2022 with Clang/LLVM frontend (DLL build)",
+        "generator": "Visual Studio 17 2022",
+        "toolset": "ClangCL",
+        "condition": {
+          "type": "equals",
+          "lhs": "${hostSystemName}",
+          "rhs": "Windows"
+        },
+        "cacheVariables": {
+          "BUILD_SHARED_LIBS": "OFF",
+          "CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS": "ON",
+          "BUILD_GEMMA_DLL": "ON"
+        }
       }
     ],
     "buildPresets": [
@@ -54,6 +72,15 @@
         "displayName": "Windows",
         "configuration": "Release",
         "configurePreset": "windows"
+      },
+      {
+        "name": "windows-dll",
+        "displayName": "Windows DLL",
+        "configuration": "Release",
+        "configurePreset": "windows-dll",
+        "targets": [
+            "gemma_shared"
+        ]
       }
     ]
   }

evals/benchmark_helper.h

@@ -25,6 +25,7 @@
 
 #include "gemma/gemma.h"
 #include "gemma/gemma_args.h"
+#include "gemma/tokenizer.h"  // WrapAndTokenize
 #include "ops/matmul.h"
 #include "util/threading_context.h"
 #include "hwy/base.h"
@@ -54,8 +55,9 @@ class GemmaEnv {
   size_t MaxGeneratedTokens() const {
     return runtime_config_.max_generated_tokens;
   }
-  void SetMaxGeneratedTokens(size_t max_generated_tokens) {
-    runtime_config_.max_generated_tokens = max_generated_tokens;
+  void SetMaxGeneratedTokens(int max_generated_tokens) {
+    runtime_config_.max_generated_tokens =
+        static_cast<size_t>(max_generated_tokens);
   }
 
   std::vector<int> Tokenize(const std::string& input) const {