diff --git a/runtime/src/iree/base/internal/file_io.h b/runtime/src/iree/base/internal/file_io.h index 7f64b583061c9..85b55c699e591 100644 --- a/runtime/src/iree/base/internal/file_io.h +++ b/runtime/src/iree/base/internal/file_io.h @@ -48,6 +48,8 @@ void iree_file_contents_free(iree_file_contents_t* contents); typedef enum iree_file_read_flag_bits_t { IREE_FILE_READ_FLAG_PRELOAD = (1u << 0), + // TODO(benvanik): drop this (and possibly all file utilities) in favor of + // iree_io_file_handle_t + iree_io_file_map_view. IREE_FILE_READ_FLAG_MMAP = (1u << 1), IREE_FILE_READ_FLAG_DEFAULT = IREE_FILE_READ_FLAG_PRELOAD, } iree_file_read_flags_t; diff --git a/runtime/src/iree/io/file_handle.c b/runtime/src/iree/io/file_handle.c index 40d5f8562b2d9..ad2d3ba9bda56 100644 --- a/runtime/src/iree/io/file_handle.c +++ b/runtime/src/iree/io/file_handle.c @@ -12,11 +12,13 @@ #if IREE_FILE_IO_ENABLE #if defined(IREE_PLATFORM_WINDOWS) +#include // _open_osfhandle constants #include // _commit #include // WerRegisterExcludedMemoryBlock #else +#include // open #include // mmap #include // fstat #include // fsync @@ -160,6 +162,276 @@ iree_io_file_handle_flush(iree_io_file_handle_t* handle) { return status; } +//===----------------------------------------------------------------------===// +// iree_io_file_handle_t utilities +//===----------------------------------------------------------------------===// + +#if IREE_FILE_IO_ENABLE + +// Creates a new platform file at |path| for usage as defined by |mode|. +// The file will be extended to |initial_size| upon creation. +// Returns IREE_STATUS_ALREADY_EXISTS if the file already exists. +// Returns IREE_STATUS_PERMISSION_DENIED if the file cannot be created. +// Opens an existing platform file at |path| for usage as defined by |mode|. +// Returns IREE_STATUS_NOT_FOUND if the file does not exist. +// Returns IREE_STATUS_PERMISSION_DENIED if the specified |mode| is disallowed. + +#if defined(IREE_PLATFORM_WINDOWS) + +static iree_status_t iree_io_file_handle_platform_open( + iree_io_file_mode_t mode, iree_string_view_t path, bool open_existing, + uint64_t initial_size, + iree_io_file_handle_primitive_t* out_handle_primitive) { + IREE_ASSERT_ARGUMENT(out_handle_primitive); + memset(out_handle_primitive, 0, sizeof(*out_handle_primitive)); + + // MAX_PATH is 260 but most systems nowadays have long paths enabled. + char path_str[2048] = {0}; + iree_string_view_to_cstring(path, path_str, sizeof(path_str)); + + DWORD desired_access = 0; + if (iree_all_bits_set(mode, IREE_IO_FILE_MODE_READ)) { + desired_access |= GENERIC_READ; + } + if (iree_all_bits_set(mode, IREE_IO_FILE_MODE_WRITE)) { + desired_access |= GENERIC_WRITE; + } + + DWORD share_mode = 0; + if (iree_all_bits_set(mode, IREE_IO_FILE_MODE_SHARE_READ)) { + share_mode |= FILE_SHARE_READ; + } + if (iree_all_bits_set(mode, IREE_IO_FILE_MODE_SHARE_WRITE)) { + share_mode |= FILE_SHARE_WRITE; + } + + DWORD creation_disposition = open_existing ? OPEN_EXISTING : CREATE_ALWAYS; + + DWORD flags = FILE_ATTRIBUTE_NORMAL; + if (iree_all_bits_set(mode, IREE_IO_FILE_MODE_RANDOM_ACCESS)) { + flags |= FILE_FLAG_RANDOM_ACCESS; + } else if (iree_all_bits_set(mode, IREE_IO_FILE_MODE_SEQUENTIAL_SCAN)) { + flags |= FILE_FLAG_SEQUENTIAL_SCAN; + } + if (iree_all_bits_set(mode, IREE_IO_FILE_MODE_TEMPORARY)) { + flags |= FILE_FLAG_DELETE_ON_CLOSE; + } + + // Create or open the file. + HANDLE handle = CreateFileA(path_str, desired_access, share_mode, NULL, + creation_disposition, flags, NULL); + if (handle == INVALID_HANDLE_VALUE) { + return iree_make_status(iree_status_code_from_win32_error(GetLastError()), + "failed to open file '%.*s'", (int)path.size, + path.data); + } + + // If we were provided an initialize size and are creating the file then + // adjust the file length. + if (!open_existing) { + // Zeroish-extend the file up to the total file size specified by the + // caller. This may be larger than the virtual address space can handle but + // so long as the length requested for mapping is under the size_t limit + // this will succeed. + LARGE_INTEGER file_size = {0}; + file_size.QuadPart = initial_size; + if (!SetFilePointerEx(handle, file_size, NULL, FILE_BEGIN) || + !SetEndOfFile(handle)) { + CloseHandle(handle); + return iree_make_status(iree_status_code_from_win32_error(GetLastError()), + "failed to extend file '%.*s' to %" PRIu64 + " bytes (out of disk space or permission denied)", + (int)path.size, path.data, initial_size); + } + } + + // Transfer ownership of the handle to a CRT file descriptor. + // After this succeeds we cannot call CloseHandle as the CRT owns it. + int open_flags = 0; + if (!iree_all_bits_set(mode, IREE_IO_FILE_MODE_WRITE)) { + open_flags |= _O_RDONLY; + } + int fd = _open_osfhandle((intptr_t)handle, open_flags); + if (fd == -1) { + CloseHandle(handle); // must close since we didn't transfer + return iree_make_status( + IREE_STATUS_INTERNAL, + "unable to transfer Win32 HANDLE to a CRT file descriptor"); + } + + out_handle_primitive->type = IREE_IO_FILE_HANDLE_TYPE_FD; + out_handle_primitive->value.fd = fd; + return iree_ok_status(); +} + +static void iree_io_file_handle_platform_close( + void* user_data, iree_io_file_handle_primitive_t handle_primitive) { + // NOTE: we opened the file using Win32 APIs but it's safe to _close since we + // transferred ownership to the CRT with _open_osfhandle. If we used + // IREE_IO_FILE_HANDLE_TYPE_WIN32_HANDLE we'd want to switch on that instead. + IREE_ASSERT_EQ(handle_primitive.type, IREE_IO_FILE_HANDLE_TYPE_FD); + _close(handle_primitive.value.fd); +} + +#else + +static iree_status_t iree_io_file_handle_platform_open( + iree_io_file_mode_t mode, iree_string_view_t path, bool open_existing, + uint64_t initial_size, + iree_io_file_handle_primitive_t* out_handle_primitive) { + IREE_ASSERT_ARGUMENT(out_handle_primitive); + memset(out_handle_primitive, 0, sizeof(*out_handle_primitive)); + + char path_str[2048] = {0}; + iree_string_view_to_cstring(path, path_str, sizeof(path_str)); + + int flags = 0; + // TODO(benvanik): add a flag for forking behavior. + flags |= O_CLOEXEC; + if (!open_existing) { + // If the file exists open anyway and truncate as if it had been recreated. + // This matches Win32 CREATE_ALWAYS behavior. + flags |= O_CREAT | O_TRUNC; + } + if (iree_all_bits_set(mode, + IREE_IO_FILE_MODE_READ | IREE_IO_FILE_MODE_WRITE)) { + // NOTE: O_RDWR != O_RDONLY | O_WRONLY! + flags |= O_RDWR; + } else if (iree_all_bits_set(mode, IREE_IO_FILE_MODE_READ)) { + flags |= O_RDONLY; + } else if (iree_all_bits_set(mode, IREE_IO_FILE_MODE_WRITE)) { + flags |= O_WRONLY; + } +#if defined(O_DIRECT) + if (iree_all_bits_set(mode, IREE_IO_FILE_MODE_DIRECT)) { + flags |= O_DIRECT; + } +#endif // O_DIRECT +#if defined(O_TMPFILE) + if (iree_all_bits_set(mode, IREE_IO_FILE_MODE_TEMPORARY)) { + flags |= O_TMPFILE; + } +#endif // O_TMPFILE + + // I don't know, unix file permissions are dumb. User and group seems fine? + const mode_t open_mode = (S_IRUSR | S_IWUSR) | (S_IRGRP | S_IWGRP); + + int fd = open(path_str, flags, open_mode); + if (fd == -1) { + return iree_make_status(iree_status_code_from_errno(errno), + "failed to open file '%.*s'", (int)path.size, + path.data); + } + + // If we were provided an initialize size and are creating the file then + // adjust the file length. + if (!open_existing) { + // Zero-extend the file up to the total file size specified by the + // caller. Note that `ftruncate` extends too. + if (ftruncate(fd, (off_t)initial_size) == -1) { + return iree_make_status(iree_status_code_from_errno(errno), + "failed to extend file '%.*s' to %" PRIu64 + " bytes (out of disk space or permission denied)", + (int)path.size, path.data, initial_size); + } + } + + out_handle_primitive->type = IREE_IO_FILE_HANDLE_TYPE_FD; + out_handle_primitive->value.fd = fd; + return iree_ok_status(); +} + +static void iree_io_file_handle_platform_close( + void* user_data, iree_io_file_handle_primitive_t handle_primitive) { + IREE_ASSERT_EQ(handle_primitive.type, IREE_IO_FILE_HANDLE_TYPE_FD); + close(handle_primitive.value.fd); +} + +#endif // IREE_PLATFORM_WINDOWS + +static iree_status_t iree_io_file_handle_create_or_open( + iree_io_file_mode_t mode, iree_string_view_t path, bool open_existing, + uint64_t initial_size, iree_allocator_t host_allocator, + iree_io_file_handle_t** out_handle) { + iree_io_file_handle_primitive_t handle_primitive; + IREE_RETURN_IF_ERROR(iree_io_file_handle_platform_open( + mode, path, open_existing, initial_size, &handle_primitive)); + + iree_io_file_access_t allowed_access = 0; + if (iree_all_bits_set(mode, IREE_IO_FILE_MODE_READ)) { + allowed_access |= IREE_IO_FILE_ACCESS_READ; + } + if (iree_all_bits_set(mode, IREE_IO_FILE_MODE_WRITE)) { + allowed_access |= IREE_IO_FILE_ACCESS_WRITE; + } + iree_io_file_handle_release_callback_t release_callback = { + .fn = iree_io_file_handle_platform_close, + .user_data = NULL, + }; + iree_io_file_handle_t* handle = NULL; + iree_status_t status = + iree_io_file_handle_wrap(allowed_access, handle_primitive, + release_callback, host_allocator, &handle); + + if (iree_status_is_ok(status)) { + *out_handle = handle; + } else { + release_callback.fn(release_callback.user_data, handle_primitive); + } + return status; +} + +IREE_API_EXPORT iree_status_t iree_io_file_handle_create( + iree_io_file_mode_t mode, iree_string_view_t path, uint64_t initial_size, + iree_allocator_t host_allocator, iree_io_file_handle_t** out_handle) { + IREE_ASSERT_ARGUMENT(out_handle); + *out_handle = NULL; + IREE_TRACE_ZONE_BEGIN(z0); + IREE_TRACE_ZONE_APPEND_TEXT(z0, path.data, path.size); + iree_status_t status = iree_io_file_handle_create_or_open( + mode, path, /*open_existing=*/false, initial_size, host_allocator, + out_handle); + IREE_TRACE_ZONE_END(z0); + return status; +} + +IREE_API_EXPORT iree_status_t iree_io_file_handle_open( + iree_io_file_mode_t mode, iree_string_view_t path, + iree_allocator_t host_allocator, iree_io_file_handle_t** out_handle) { + IREE_ASSERT_ARGUMENT(out_handle); + *out_handle = NULL; + IREE_TRACE_ZONE_BEGIN(z0); + IREE_TRACE_ZONE_APPEND_TEXT(z0, path.data, path.size); + iree_status_t status = iree_io_file_handle_create_or_open( + mode, path, /*open_existing=*/true, 0ull, host_allocator, out_handle); + IREE_TRACE_ZONE_END(z0); + return status; +} + +#else + +IREE_API_EXPORT iree_status_t iree_io_file_handle_create( + iree_io_file_mode_t mode, iree_string_view_t path, uint64_t initial_size, + iree_allocator_t host_allocator, iree_io_file_handle_t** out_handle) { + IREE_ASSERT_ARGUMENT(out_handle); + *out_handle = NULL; + return iree_make_status(IREE_STATUS_UNAVAILABLE, + "file support has been compiled out of this binary; " + "set IREE_FILE_IO_ENABLE=1 to include it"); +} + +IREE_API_EXPORT iree_status_t iree_io_file_handle_open( + iree_io_file_mode_t mode, iree_string_view_t path, + iree_allocator_t host_allocator, iree_io_file_handle_t** out_handle) { + IREE_ASSERT_ARGUMENT(out_handle); + *out_handle = NULL; + return iree_make_status(IREE_STATUS_UNAVAILABLE, + "file support has been compiled out of this binary; " + "set IREE_FILE_IO_ENABLE=1 to include it"); +} + +#endif // IREE_FILE_IO_ENABLE + //===----------------------------------------------------------------------===// // iree_io_file_mapping_t support //===----------------------------------------------------------------------===// diff --git a/runtime/src/iree/io/file_handle.h b/runtime/src/iree/io/file_handle.h index 53e5a37594cc7..c759244d7d8dd 100644 --- a/runtime/src/iree/io/file_handle.h +++ b/runtime/src/iree/io/file_handle.h @@ -156,6 +156,53 @@ static inline iree_io_file_handle_primitive_value_t iree_io_file_handle_value( IREE_API_EXPORT iree_status_t iree_io_file_handle_flush(iree_io_file_handle_t* handle); +//===----------------------------------------------------------------------===// +// iree_io_file_handle_t platform files +//===----------------------------------------------------------------------===// + +// Bits indicating how a file is opened. +typedef uint64_t iree_io_file_mode_t; +enum iree_io_file_mode_bits_t { + // Allow reads of both existing and new content. + IREE_IO_FILE_MODE_READ = 1ull << 0, + // Allow writes. + IREE_IO_FILE_MODE_WRITE = 1ull << 1, + // Hints that the file will be accessed at random (more-so than not). + IREE_IO_FILE_MODE_RANDOM_ACCESS = 1ull << 2, + // Hints that the file will be accessed sequentially (contiguous reads/writes + // or small skips forward only). + IREE_IO_FILE_MODE_SEQUENTIAL_SCAN = 1ull << 3, + // Hints that the library and system caching are not required. May hurt + // performance more than it helps unless the file is very large and + // exclusively accessed as part of bulk transfer operations that are + // page-aligned. + IREE_IO_FILE_MODE_DIRECT = 1ull << 4, + // Ensures the file is deleted when it is closed. Platforms may use this as a + // hint to avoid writing the file contents when cache is available. + IREE_IO_FILE_MODE_TEMPORARY = 1ull << 5, + // Allows subsequent operations to open the file for read access while the + // file is open by the creator. + IREE_IO_FILE_MODE_SHARE_READ = 1ull << 6, + // Allows subsequent operations to open the file for write access while the + // file is open by the creator. + IREE_IO_FILE_MODE_SHARE_WRITE = 1ull << 7, +}; + +// Creates a new platform file at |path| for usage as defined by |mode|. +// The file will be extended to |initial_size| upon creation. +// Returns IREE_STATUS_ALREADY_EXISTS if the file already exists. +// Returns IREE_STATUS_PERMISSION_DENIED if the file cannot be created. +IREE_API_EXPORT iree_status_t iree_io_file_handle_create( + iree_io_file_mode_t mode, iree_string_view_t path, uint64_t initial_size, + iree_allocator_t host_allocator, iree_io_file_handle_t** out_handle); + +// Opens an existing platform file at |path| for usage as defined by |mode|. +// Returns IREE_STATUS_NOT_FOUND if the file does not exist. +// Returns IREE_STATUS_PERMISSION_DENIED if the specified |mode| is disallowed. +IREE_API_EXPORT iree_status_t iree_io_file_handle_open( + iree_io_file_mode_t mode, iree_string_view_t path, + iree_allocator_t host_allocator, iree_io_file_handle_t** out_handle); + //===----------------------------------------------------------------------===// // iree_io_file_mapping_t //===----------------------------------------------------------------------===// diff --git a/runtime/src/iree/tooling/parameter_util.c b/runtime/src/iree/tooling/parameter_util.c index 294166daa3304..bd748ad02f175 100644 --- a/runtime/src/iree/tooling/parameter_util.c +++ b/runtime/src/iree/tooling/parameter_util.c @@ -18,8 +18,14 @@ // Parameter file I/O //===----------------------------------------------------------------------===// +#if IREE_FILE_IO_ENABLE +#define FLAG_PARAMETER_MODE_DEFAULT "file" +#else +#define FLAG_PARAMETER_MODE_DEFAULT "mmap" +#endif // IREE_FILE_IO_ENABLE + IREE_FLAG( - string, parameter_mode, "mmap", + string, parameter_mode, FLAG_PARAMETER_MODE_DEFAULT, "A parameter I/O mode of ['preload', 'mmap', 'file'].\n" " preload: read entire parameter files into wired memory on startup.\n" " mmap: maps the parameter files into discardable memory - can increase\n" @@ -33,6 +39,36 @@ static void iree_file_contents_release_callback( iree_file_contents_free(file_contents); } +// Legacy parameter file open path. We should be able to replace this usage with +// iree_io_file_handle_t-based logic. +static iree_status_t iree_io_open_parameter_file_legacy( + iree_string_view_t path, iree_file_read_flags_t read_flags, + iree_allocator_t host_allocator, iree_io_file_handle_t** out_file_handle) { + IREE_ASSERT_ARGUMENT(out_file_handle); + *out_file_handle = NULL; + + char path_str[2048] = {0}; + iree_string_view_to_cstring(path, path_str, sizeof(path_str)); + + // Read (or map) the entire file into host memory. + iree_file_contents_t* file_contents = NULL; + IREE_RETURN_IF_ERROR(iree_file_read_contents(path_str, read_flags, + host_allocator, &file_contents)); + + // Wrap the loaded memory file in a file handle. + const iree_io_file_handle_release_callback_t release_callback = { + .fn = iree_file_contents_release_callback, + .user_data = file_contents, + }; + iree_status_t status = iree_io_file_handle_wrap_host_allocation( + IREE_IO_FILE_ACCESS_READ, file_contents->buffer, release_callback, + host_allocator, out_file_handle); + if (!iree_status_is_ok(status)) { + iree_file_contents_free(file_contents); + } + return status; +} + // Opens the parameter file at |path| with the mode specified by the // --parameter_mode flag and returns its handle. static iree_status_t iree_io_open_parameter_file( @@ -43,37 +79,25 @@ static iree_status_t iree_io_open_parameter_file( IREE_TRACE_ZONE_BEGIN(z0); IREE_TRACE_ZONE_APPEND_TEXT(z0, path.data, path.size); - char path_str[2048] = {0}; - iree_string_view_to_cstring(path, path_str, sizeof(path_str)); - iree_file_read_flags_t read_flags = 0; + iree_status_t status = iree_ok_status(); + iree_io_file_handle_t* file_handle = NULL; if (strcmp(FLAG_parameter_mode, "mmap") == 0) { - read_flags |= IREE_FILE_READ_FLAG_MMAP; + status = iree_io_open_parameter_file_legacy(path, IREE_FILE_READ_FLAG_MMAP, + host_allocator, &file_handle); } else if (strcmp(FLAG_parameter_mode, "preload") == 0) { - read_flags |= IREE_FILE_READ_FLAG_PRELOAD; + status = iree_io_open_parameter_file_legacy( + path, IREE_FILE_READ_FLAG_PRELOAD, host_allocator, &file_handle); + } else if (strcmp(FLAG_parameter_mode, "file") == 0) { + status = iree_io_file_handle_open(IREE_IO_FILE_MODE_READ, path, + host_allocator, &file_handle); } else { - IREE_TRACE_ZONE_END(z0); - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "unrecognized --parameter_mode= value '%s'", - FLAG_parameter_mode); + status = iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "unrecognized --parameter_mode= value '%s'", + FLAG_parameter_mode); } - iree_file_contents_t* file_contents = NULL; - IREE_RETURN_AND_END_ZONE_IF_ERROR( - z0, iree_file_read_contents(path_str, read_flags, host_allocator, - &file_contents)); - - iree_io_file_handle_release_callback_t release_callback = { - .fn = iree_file_contents_release_callback, - .user_data = file_contents, - }; - iree_io_file_handle_t* file_handle = NULL; - iree_status_t status = iree_io_file_handle_wrap_host_allocation( - IREE_IO_FILE_ACCESS_READ, file_contents->buffer, release_callback, - host_allocator, &file_handle); if (iree_status_is_ok(status)) { *out_file_handle = file_handle; - } else { - iree_file_contents_free(file_contents); } IREE_TRACE_ZONE_END(z0); return status;