use host byte order consistently in precompiled module files (#26493)

Author: JeffBezanson

base/loading.jl

@@ -1200,7 +1200,7 @@ function compilecache(pkg::PkgId)
     if success(create_expr_cache(path, cachefile, concrete_deps, pkg.uuid))
         # append checksum to the end of the .ji file:
         open(cachefile, "a+") do f
-            write(f, hton(_crc32c(seekstart(f))))
+            write(f, _crc32c(seekstart(f)))
         end
     else
         error("Failed to precompile $name to $cachefile.")
@@ -1211,35 +1211,35 @@ end
 module_build_id(m::Module) = ccall(:jl_module_build_id, UInt64, (Any,), m)
 
 isvalid_cache_header(f::IOStream) = (0 != ccall(:jl_read_verify_header, Cint, (Ptr{Cvoid},), f.ios))
-isvalid_file_crc(f::IOStream) = (_crc32c(seekstart(f), filesize(f) - 4) == ntoh(read(f, UInt32)))
+isvalid_file_crc(f::IOStream) = (_crc32c(seekstart(f), filesize(f) - 4) == read(f, UInt32))
 
 function parse_cache_header(f::IO)
     modules = Vector{Pair{PkgId, UInt64}}()
     while true
-        n = ntoh(read(f, Int32))
+        n = read(f, Int32)
         n == 0 && break
         sym = String(read(f, n)) # module name
-        uuid = UUID((ntoh(read(f, UInt64)), ntoh(read(f, UInt64)))) # pkg UUID
-        build_id = ntoh(read(f, UInt64)) # build UUID (mostly just a timestamp)
+        uuid = UUID((read(f, UInt64), read(f, UInt64))) # pkg UUID
+        build_id = read(f, UInt64) # build UUID (mostly just a timestamp)
         push!(modules, PkgId(uuid, sym) => build_id)
     end
-    totbytes = ntoh(read(f, Int64)) # total bytes for file dependencies
+    totbytes = read(f, Int64) # total bytes for file dependencies
     # read the list of requirements
     # and split the list into include and requires statements
     includes = Tuple{PkgId, String, Float64}[]
     requires = Pair{PkgId, PkgId}[]
     while true
-        n2 = ntoh(read(f, Int32))
+        n2 = read(f, Int32)
         n2 == 0 && break
         depname = String(read(f, n2))
-        mtime = ntoh(read(f, Float64))
-        n1 = ntoh(read(f, Int32))
+        mtime = read(f, Float64)
+        n1 = read(f, Int32)
         # map ids to keys
         modkey = (n1 == 0) ? PkgId("") : modules[n1].first
         if n1 != 0
             # consume (and ignore) the module path too
             while true
-                n1 = ntoh(read(f, Int32))
+                n1 = read(f, Int32)
                 totbytes -= 4
                 n1 == 0 && break
                 skip(f, n1) # String(read(f, n1))
@@ -1254,15 +1254,15 @@ function parse_cache_header(f::IO)
         totbytes -= 4 + 4 + n2 + 8
     end
     @assert totbytes == 12 "header of cache file appears to be corrupt"
-    srctextpos = ntoh(read(f, Int64))
+    srctextpos = read(f, Int64)
     # read the list of modules that are required to be present during loading
     required_modules = Vector{Pair{PkgId, UInt64}}()
     while true
-        n = ntoh(read(f, Int32))
+        n = read(f, Int32)
         n == 0 && break
         sym = String(read(f, n)) # module name
-        uuid = UUID((ntoh(read(f, UInt64)), ntoh(read(f, UInt64)))) # pkg UUID
-        build_id = ntoh(read(f, UInt64)) # build id
+        uuid = UUID((read(f, UInt64), read(f, UInt64))) # pkg UUID
+        build_id = read(f, UInt64) # build id
         push!(required_modules, PkgId(uuid, sym) => build_id)
     end
     return modules, (includes, requires), required_modules, srctextpos
@@ -1302,10 +1302,10 @@ end
 
 function _read_dependency_src(io::IO, filename::AbstractString)
     while !eof(io)
-        filenamelen = ntoh(read(io, Int32))
+        filenamelen = read(io, Int32)
         filenamelen == 0 && break
         fn = String(read(io, filenamelen))
-        len = ntoh(read(io, UInt64))
+        len = read(io, UInt64)
         if fn == filename
             return String(read(io, len))
         end

src/dump.c

@@ -128,55 +128,47 @@ static arraylist_t builtin_typenames;
 #define write_int8(s, n) write_uint8(s, n)
 #define read_int8(s) read_uint8(s)
 
-/* read and write in network (bigendian) order: */
+/* read and write in host byte order */
 
 static void write_int32(ios_t *s, int32_t i)
 {
-    write_uint8(s, (i>>24) & 0xff);
-    write_uint8(s, (i>>16) & 0xff);
-    write_uint8(s, (i>> 8) & 0xff);
-    write_uint8(s, i       & 0xff);
+    ios_write(s, (char*)&i, 4);
 }
 
 static int32_t read_int32(ios_t *s)
 {
-    int b3 = read_uint8(s);
-    int b2 = read_uint8(s);
-    int b1 = read_uint8(s);
-    int b0 = read_uint8(s);
-    return b0 | (b1<<8) | (b2<<16) | (b3<<24);
+    int32_t x = 0;
+    ios_read(s, (char*)&x, 4);
+    return x;
 }
 
 static void write_uint64(ios_t *s, uint64_t i)
 {
-    write_int32(s, (i>>32) & 0xffffffff);
-    write_int32(s, i       & 0xffffffff);
+    ios_write(s, (char*)&i, 8);
 }
 
 static uint64_t read_uint64(ios_t *s)
 {
-    uint64_t b1 = (uint32_t)read_int32(s);
-    uint64_t b0 = (uint32_t)read_int32(s);
-    return b0 | (b1<<32);
+    uint64_t x = 0;
+    ios_read(s, (char*)&x, 8);
+    return x;
 }
 
 static void write_int64(ios_t *s, int64_t i)
 {
-    write_int32(s, (i>>32) & 0xffffffff);
-    write_int32(s, i       & 0xffffffff);
+    ios_write(s, (char*)&i, 8);
 }
 
 static void write_uint16(ios_t *s, uint16_t i)
 {
-    write_uint8(s, (i>> 8) & 0xff);
-    write_uint8(s, i       & 0xff);
+    ios_write(s, (char*)&i, 2);
 }
 
 static uint16_t read_uint16(ios_t *s)
 {
-    int b1 = read_uint8(s);
-    int b0 = read_uint8(s);
-    return b0 | (b1<<8);
+    int16_t x = 0;
+    ios_read(s, (char*)&x, 2);
+    return x;
 }
 
 static void writetag(ios_t *s, void *v)
@@ -1067,7 +1059,7 @@ static void write_mod_list(ios_t *s, jl_array_t *a)
 }
 
 // "magic" string and version header of .ji file
-static const int JI_FORMAT_VERSION = 5;
+static const int JI_FORMAT_VERSION = 6;
 static const char JI_MAGIC[] = "\373jli\r\n\032\n"; // based on PNG signature
 static const uint16_t BOM = 0xFEFF; // byte-order marker
 static void write_header(ios_t *s)

src/staticdata.c

@@ -107,26 +107,21 @@ enum RefTags {
 #define RELOC_TAG_OFFSET 28
 
 
-/* read and write in network (bigendian) order: */
+/* read and write in host byte order */
 
 #define write_uint8(s, n) ios_putc((n), (s))
 #define read_uint8(s) ((uint8_t)ios_getc((s)))
 
 static void write_uint32(ios_t *s, uint32_t i)
 {
-    write_uint8(s, (i >> 24) & 0xff);
-    write_uint8(s, (i >> 16) & 0xff);
-    write_uint8(s, (i >> 8) & 0xff);
-    write_uint8(s, i        & 0xff);
+    ios_write(s, (char*)&i, 4);
 }
 
 static uint32_t read_uint32(ios_t *s)
 {
-    uint32_t b3 = read_uint8(s);
-    uint32_t b2 = read_uint8(s);
-    uint32_t b1 = read_uint8(s);
-    uint32_t b0 = read_uint8(s);
-    return b0 | (b1 << 8) | (b2 << 16) | (b3 << 24);
+    uint32_t x = 0;
+    ios_read(s, (char*)&x, 4);
+    return x;
 }
 
 
@@ -767,11 +762,11 @@ static void jl_write_gv_syms(jl_serializer_state *s, jl_sym_t *v)
 }
 
 
-static inline uint32_t load_uint32_be(uintptr_t *base)
+static inline uint32_t load_uint32(uintptr_t *base)
 {
     uint32_t v = **(uint32_t**)base;
     *base += 4;
-    return ntohl(v);
+    return v;
 }
 
 
@@ -781,7 +776,7 @@ static void jl_read_symbols(jl_serializer_state *s)
     uintptr_t base = (uintptr_t)&s->symbols->buf[0];
     uintptr_t end = base + s->symbols->size;
     while (base < end) {
-        uint32_t len = load_uint32_be(&base);
+        uint32_t len = load_uint32(&base);
         const char *str = (const char*)base;
         base += len + 1;
         //printf("symbol %3d: %s\n", len, str);
@@ -884,7 +879,7 @@ static void jl_read_relocations(jl_serializer_state *s, uint8_t bits)
     size_t size = s->s->size;
     while (1) {
         uintptr_t val = (uintptr_t)&s->relocs->buf[s->relocs->bpos];
-        uint32_t offset = load_uint32_be(&val);
+        uint32_t offset = load_uint32(&val);
         s->relocs->bpos += sizeof(uint32_t);
         if (offset == 0)
             break;
@@ -904,7 +899,7 @@ void gc_sweep_sysimg(void)
     if (relocs == 0)
         return;
     while (1) {
-        uint32_t offset = load_uint32_be(&relocs);
+        uint32_t offset = load_uint32(&relocs);
         if (offset == 0)
             break;
         jl_taggedvalue_t *o = (jl_taggedvalue_t*)(base + offset);
@@ -931,7 +926,7 @@ static jl_value_t *jl_read_value(jl_serializer_state *s)
     uintptr_t base = (uintptr_t)&s->s->buf[0];
     size_t size = s->s->size;
     uintptr_t val = base + s->s->bpos;
-    uint32_t offset = load_uint32_be(&val);
+    uint32_t offset = load_uint32(&val);
     s->s->bpos += sizeof(uint32_t);
     if (offset == 0)
         return NULL;
@@ -952,7 +947,7 @@ static void jl_update_all_fptrs(jl_serializer_state *s)
     uint32_t clone_idx = 0;
     for (i = 0; i < sysimg_fvars_max; i++) {
         uintptr_t val = (uintptr_t)&linfos[i];
-        uint32_t offset = load_uint32_be(&val);
+        uint32_t offset = load_uint32(&val);
         if (offset != 0) {
             int cfunc = 0;
             if (offset & ((uintptr_t)1 << (8 * sizeof(uint32_t) - 1))) {
@@ -997,7 +992,7 @@ static void jl_update_all_gvars(jl_serializer_state *s)
     uintptr_t gvars = (uintptr_t)&s->gvar_record->buf[0];
     uintptr_t end = gvars + s->gvar_record->size;
     while (gvars < end) {
-        uint32_t offset = load_uint32_be(&gvars);
+        uint32_t offset = load_uint32(&gvars);
         if (offset) {
             uintptr_t v = get_item_for_reloc(s, base, size, offset);
             *sysimg_gvars(sysimg_gvars_base, gvname_index) = v;