From 4c155bd3df70e45837a50a3d7496733ba47e000f Mon Sep 17 00:00:00 2001 From: Dan Zheng Date: Sun, 25 Feb 2024 19:31:27 -0800 Subject: [PATCH] Restore reverted changes. Sync to https://github.com/google/gemma.cpp/commit/84444c93a44f484442fda2523dde7e77dbd3a53c. PiperOrigin-RevId: 610263918 --- .github/workflows/build.yml | 4 +- CMakeLists.txt | 8 ++- CMakePresets.json | 59 ++++++++++++++++++++++ README.md | 42 +++++++++++++--- compression/blob_store.cc | 97 +++++++++++++++++++++++++++++++++---- run.cc | 5 ++ util/app.h | 2 + 7 files changed, 196 insertions(+), 21 deletions(-) create mode 100644 CMakePresets.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b0d4b6e7..da63c1ca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,7 +1,7 @@ name: build -# Trigger on push or via manual dispatch. -on: [push, workflow_dispatch] +# Trigger on push, pull request, or via manual dispatch. +on: [push, pull_request, workflow_dispatch] jobs: build: diff --git a/CMakeLists.txt b/CMakeLists.txt index 722e4084..c7828cc7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) FetchContent_Declare(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG da250571a45826b21eebbddc1e50d0c1137dee5f) FetchContent_MakeAvailable(highway) -## Note: absl meeds tp be installed by sentencepiece. This will only happen if +## Note: absl needs to be installed by sentencepiece. This will only happen if ## cmake is invoked with -DSPM_ENABLE_SHARED=OFF and -DSPM_ABSL_PROVIDER=module FetchContent_Declare(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 53de76561cfc149d3c01037f0595669ad32a5e7c) FetchContent_MakeAvailable(sentencepiece) @@ -49,7 +49,7 @@ endif() # Allowable types for WEIGHT_TYPE: # float - slow, not recommended -# hwy::bfloat16_t - bfloat16 as impemented by https://github.com/google/highway +# hwy::bfloat16_t - bfloat16 as implemented by https://github.com/google/highway # SfpStream - 8-bit switched floating point (recommended) # NuqStream - experimental, work-in-progress option(WEIGHT_TYPE "Set weight type" "") @@ -67,6 +67,8 @@ target_link_libraries(gemma hwy hwy_contrib sentencepiece) target_include_directories(gemma PRIVATE ./) FetchContent_GetProperties(sentencepiece) target_include_directories(gemma PRIVATE ${sentencepiece_SOURCE_DIR}) +target_compile_definitions(gemma PRIVATE $<$:_CRT_SECURE_NO_WARNINGS NOMINMAX>) +target_compile_options(gemma PRIVATE $<$:-Wno-deprecated-declarations>) ## Library Target @@ -76,3 +78,5 @@ set_target_properties(libgemma PROPERTIES PREFIX "") target_include_directories(libgemma PUBLIC ./) target_link_libraries(libgemma hwy hwy_contrib sentencepiece) target_include_directories(libgemma PRIVATE ${sentencepiece_SOURCE_DIR}) +target_compile_definitions(libgemma PRIVATE $<$:_CRT_SECURE_NO_WARNINGS NOMINMAX>) +target_compile_options(libgemma PRIVATE $<$:-Wno-deprecated-declarations>) diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 00000000..5fe13c82 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,59 @@ +{ + "version": 3, + "cmakeMinimumRequired": { + "major": 3, + "minor": 11, + "patch": 0 + }, + "configurePresets": [ + { + "name": "__defaults__", + "hidden": true, + "binaryDir": "${sourceDir}/build" + }, + { + "name": "make", + "inherits": "__defaults__", + "displayName": "Make", + "description": "Unix Makefiles", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build" + }, + { + "name": "windows", + "inherits": "__defaults__", + "displayName": "Windows", + "description": "Visual Studio 2022 with Clang/LLVM frontend", + "generator": "Visual Studio 17 2022", + "toolset": "ClangCL", + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Windows" + } + } + ], + "buildPresets": [ + { + "name": "__defaults__", + "hidden": true, + "targets": [ + "gemma", + "libgemma" + ] + }, + { + "name": "make", + "inherits": "__defaults__", + "displayName": "Unix Makefiles", + "configurePreset": "make" + }, + { + "name": "windows", + "inherits": "__defaults__", + "displayName": "Windows", + "configuration": "Release", + "configurePreset": "windows" + } + ] + } diff --git a/README.md b/README.md index e278833d..d31bbaf3 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,16 @@ Before starting, you should have installed: least C++17. - `tar` for extracting archives from Kaggle. +Building natively on Windows requires the Visual Studio 2012 Build Tools with the +optional Clang/LLVM C++ frontend (`clang-cl`). This can be installed from the +command line with +[`winget`](https://learn.microsoft.com/en-us/windows/package-manager/winget/): + +```sh +winget install --id Kitware.CMake +winget install --id Microsoft.VisualStudio.2022.BuildTools --force --override "--passive --wait --add Microsoft.VisualStudio.Workload.VCTools;installRecommended --add Microsoft.VisualStudio.Component.VC.Llvm.Clang --add Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset" +``` + ### Step 1: Obtain model weights and tokenizer from Kaggle Visit [the Gemma model page on @@ -107,6 +117,7 @@ runtime, create a build directory and generate the build files using `cmake` from the top-level project directory. For the 8-bit switched floating point weights (sfp), run cmake with no options: +#### Unix-like Platforms ```sh cmake -B build ``` @@ -126,17 +137,18 @@ your weights, you can enter the `build/` directory and run `make` to build the `./gemma` executable: ```sh -cd build -make -j [number of parallel threads to use] gemma +# Configure `build` directory +cmake --preset make + +# Build project using make +cmake --build --preset make -j [number of parallel threads to use] ``` Replace `[number of parallel threads to use]` with a number - the number of -cores available on your system is a reasonable heuristic. - -For example, `make -j4 gemma` will build using 4 threads. If this is successful, -you should now have a `gemma` executable in the `build/` directory. If the -`nproc` command is available, you can use `make -j$(nproc) gemma` as a -reasonable default for the number of threads. +cores available on your system is a reasonable heuristic. For example, +`make -j4 gemma` will build using 4 threads. If the `nproc` command is +available, you can use `make -j$(nproc) gemma` as a reasonable default +for the number of threads. If you aren't sure of the right value for the `-j` flag, you can simply run `make gemma` instead and it should still build the `./gemma` executable. @@ -145,6 +157,20 @@ If you aren't sure of the right value for the `-j` flag, you can simply run > On Windows Subsystem for Linux (WSL) users should set the number of > parallel threads to 1. Using a larger number may result in errors. +If the build is successful, you should now have a `gemma` executable in the `build/` directory. + +#### Windows + +```sh +# Configure `build` directory +cmake --preset windows + +# Build project using Visual Studio Build Tools +cmake --build --preset windows -j [number of parallel threads to use] +``` + +If the build is successful, you should now have a `gemma.exe` executable in the `build/` directory. + ### Step 4: Run You can now run `gemma` from inside the `build/` directory. diff --git a/compression/blob_store.cc b/compression/blob_store.cc index 8d6c1d0f..550c727d 100644 --- a/compression/blob_store.cc +++ b/compression/blob_store.cc @@ -16,11 +16,16 @@ // copybara:import_next_line:gemma_cpp #include "compression/blob_store.h" -#include // open #include #include // SEEK_END - unistd isn't enough for IDE. #include // O_RDONLY -#include // read, close +#include // open +#if HWY_OS_WIN +#include // read, write, close +#include +#else +#include // read, write, close +#endif #include #include @@ -30,6 +35,54 @@ #include "hwy/contrib/thread_pool/thread_pool.h" #include "hwy/detect_compiler_arch.h" +namespace { +#if HWY_OS_WIN + +// pread is not supported on Windows +static int64_t pread(int fd, void* buf, uint64_t size, uint64_t offset) { + HANDLE file = reinterpret_cast(_get_osfhandle(fd)); + if (file == INVALID_HANDLE_VALUE) { + return -1; + } + + OVERLAPPED overlapped = {0}; + overlapped.Offset = offset & 0xFFFFFFFF; + overlapped.OffsetHigh = (offset >> 32) & 0xFFFFFFFF; + + DWORD bytes_read; + if (!ReadFile(file, buf, size, &bytes_read, &overlapped)) { + if (GetLastError() != ERROR_HANDLE_EOF) { + return -1; + } + } + + return bytes_read; +} + +// pwrite is not supported on Windows +static int64_t pwrite(int fd, const void* buf, uint64_t size, uint64_t offset) { + HANDLE file = reinterpret_cast(_get_osfhandle(fd)); + if (file == INVALID_HANDLE_VALUE) { + return -1; + } + + OVERLAPPED overlapped = {0}; + overlapped.Offset = offset & 0xFFFFFFFF; + overlapped.OffsetHigh = (offset >> 32) & 0xFFFFFFFF; + + DWORD bytes_written; + if (!WriteFile(file, buf, size, &bytes_written, &overlapped)) { + if (GetLastError() != ERROR_HANDLE_EOF) { + return -1; + } + } + + return bytes_written; +} + +#endif +} + namespace gcpp { hwy::uint128_t MakeKey(const char* string) { @@ -64,19 +117,30 @@ static void EnqueueChunkRequests(uint64_t offset, uint64_t size, uint8_t* data, } } + struct IO { // Returns size in bytes or 0. static uint64_t FileSize(const char* filename) { int fd = open(filename, O_RDONLY); - if (fd >= 0) { - const off_t size = lseek(fd, 0, SEEK_END); - HWY_ASSERT(close(fd) != -1); - if (size != static_cast(-1)) { - return static_cast(size); - } + if (fd < 0) { + return 0; } - return 0; +#if HWY_OS_WIN + const int64_t size = _lseeki64(fd, 0, SEEK_END); + HWY_ASSERT(close(fd) != -1); + if (size < 0) { + return 0; + } +#else + const off_t size = lseek(fd, 0, SEEK_END); + HWY_ASSERT(close(fd) != -1); + if (size == static_cast(-1)) { + return 0; + } +#endif + + return static_cast(size); } static bool Read(int fd, uint64_t offset, uint64_t size, void* to) { @@ -252,7 +316,14 @@ class BlobStore { #pragma pack(pop) BlobError BlobReader::Open(const char* filename) { +#if HWY_OS_WIN + DWORD flags = FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN; + HANDLE file = CreateFileA(filename, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, flags, nullptr); + if (file == INVALID_HANDLE_VALUE) return __LINE__; + fd_ = _open_osfhandle(reinterpret_cast(file), _O_RDONLY); +#else fd_ = open(filename, O_RDONLY); +#endif if (fd_ < 0) return __LINE__; #if _POSIX_C_SOURCE >= 200112L @@ -330,7 +401,14 @@ BlobError BlobWriter::WriteAll(hwy::ThreadPool& pool, keys_.data(), blobs_.data(), keys_.size()); // Create/replace existing file. +#if HWY_OS_WIN + DWORD flags = FILE_ATTRIBUTE_NORMAL; + HANDLE file = CreateFileA(filename, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, flags, nullptr); + if (file == INVALID_HANDLE_VALUE) return __LINE__; + const int fd = _open_osfhandle(reinterpret_cast(file), _O_WRONLY); +#else const int fd = open(filename, O_CREAT | O_RDWR | O_TRUNC, 0644); +#endif if (fd < 0) return __LINE__; std::atomic_flag err = ATOMIC_FLAG_INIT; @@ -341,6 +419,7 @@ BlobError BlobWriter::WriteAll(hwy::ThreadPool& pool, err.test_and_set(); } }); + HWY_ASSERT(close(fd) != -1); if (err.test_and_set()) return __LINE__; return 0; } diff --git a/run.cc b/run.cc index 87d84453..96ba3169 100644 --- a/run.cc +++ b/run.cc @@ -144,6 +144,11 @@ void ReplGemma(gcpp::Gemma& model, hwy::ThreadPool& pool, return; } + if (prompt_string == "%c" || prompt_string == "%C") { + abs_pos = 0; + continue; + } + if (model.model_training == ModelTraining::GEMMA_IT) { // For instruction-tuned models: add control tokens. prompt_string = "user\n" + prompt_string + diff --git a/util/app.h b/util/app.h index 966fa417..bd665a46 100644 --- a/util/app.h +++ b/util/app.h @@ -18,7 +18,9 @@ #ifndef THIRD_PARTY_GEMMA_CPP_UTIL_APP_H_ #define THIRD_PARTY_GEMMA_CPP_UTIL_APP_H_ +#if HWY_OS_LINUX #include +#endif #include #include // std::clamp