From a144741f0bfb48dbf555d51cce75fc60ab40c3fa Mon Sep 17 00:00:00 2001 From: Morgan Thomas Date: Tue, 2 Apr 2024 03:27:55 -0400 Subject: [PATCH 1/7] wip: ELF file support --- Cargo.lock | 14 ++++++++++++++ Cargo.toml | 1 + elf/Cargo.toml | 9 +++++++++ elf/src/lib.rs | 16 ++++++++++++++++ 4 files changed, 40 insertions(+) create mode 100644 elf/Cargo.toml create mode 100644 elf/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index c57a567..16db62d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -279,6 +279,12 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +[[package]] +name = "elf" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4445909572dbd556c457c849c4ca58623d84b27c8fff1e74b0b4227d8b90d17b" + [[package]] name = "generic-array" version = "0.14.7" @@ -1121,6 +1127,14 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "valida-elf" +version = "0.1.0" +dependencies = [ + "elf", + "valida-machine", +] + [[package]] name = "valida-machine" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index c91bac6..553d52f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "bus", "cpu", "derive", + "elf", "native_field", "machine", "memory", diff --git a/elf/Cargo.toml b/elf/Cargo.toml new file mode 100644 index 0000000..c31aba3 --- /dev/null +++ b/elf/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "valida-elf" +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" + +[dependencies] +elf = "0.7.4" +valida-machine = { path = "../machine" } diff --git a/elf/src/lib.rs b/elf/src/lib.rs new file mode 100644 index 0000000..ddf58c6 --- /dev/null +++ b/elf/src/lib.rs @@ -0,0 +1,16 @@ +#![no_std] + +extern crate alloc; + +use alloc::collections::BTreeMap; +use alloc::vec::Vec; +use valida_machine::{ProgramROM, Word}; + +pub struct Program { + code: ProgramROM, + data: BTreeMap>, +} + +pub fn load_elf_object_file(file: Vec) -> Program { + todo!() +} From b5765b29d8d5a1a293bbaacfe024be653893faba Mon Sep 17 00:00:00 2001 From: Morgan Thomas Date: Wed, 3 Apr 2024 17:12:15 -0400 Subject: [PATCH 2/7] wip: ELF file support --- elf/src/lib.rs | 62 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/elf/src/lib.rs b/elf/src/lib.rs index ddf58c6..3e6db3a 100644 --- a/elf/src/lib.rs +++ b/elf/src/lib.rs @@ -3,14 +3,72 @@ extern crate alloc; use alloc::collections::BTreeMap; +use alloc::vec; use alloc::vec::Vec; +use elf::ElfBytes; +use elf::abi; +use elf::endian::AnyEndian; +use elf::section::SectionHeader; use valida_machine::{ProgramROM, Word}; pub struct Program { - code: ProgramROM, + code: ProgramROM, data: BTreeMap>, } pub fn load_elf_object_file(file: Vec) -> Program { - todo!() + let file = ElfBytes::::minimal_parse(file.as_slice()).unwrap(); + let mut data_sections: Vec::<(SectionHeader, &[u8])> = vec![]; + let mut bss_sections: Vec:: = vec![]; + let mut text_sections: Vec::<(SectionHeader, &[u8])> = vec![]; + for section_header in file.section_headers().unwrap().iter() { + let is_data: bool = section_header.sh_type == abi::SHT_PROGBITS + && section_header.sh_flags == (abi::SHF_ALLOC | abi::SHF_WRITE).into(); + let is_bss: bool = section_header.sh_type == abi::SHT_NOBITS + && section_header.sh_flags == (abi::SHF_ALLOC | abi::SHF_WRITE).into(); + let is_text: bool = section_header.sh_type == abi::SHT_PROGBITS + && section_header.sh_flags == (abi::SHF_ALLOC | abi::SHF_EXECINSTR).into(); + let is_useful: bool = is_data || is_bss || is_text; + if is_useful { + if is_data || is_text { + let section_data = file.section_data(§ion_header).unwrap(); + match section_data { + (section_data, None) => { + if is_data { + data_sections.push((section_header, section_data)); + } else if is_text { + text_sections.push((section_header, section_data)); + } + }, + _ => panic!("unsupported: compressed ELF section data"), + } + } else if is_bss { + bss_sections.push(section_header); + } + } + } + let code_size = + text_sections + .iter() + .map(|(section_header, _)| { + section_header.sh_addr + section_header.sh_size + }) + .fold(0, |a, b| a.max(b)); + let mut code: Vec = vec![0; code_size as usize]; + for (section_header, section_data) in text_sections { + for i in 0 .. section_header.sh_size as usize { + code[i + section_header.sh_addr as usize] = section_data[i]; + } + } + let mut data: BTreeMap> = BTreeMap::new(); + for (section_header, section_data) in data_sections { + for i in 0 .. (section_header.sh_size / 4) as usize { + data.insert(section_header.sh_addr.try_into().unwrap(), + Word([section_data[i*4], section_data[i*4+1], section_data[i*4+2], section_data[i*4+3]])); + } + } + Program { + code: ProgramROM::from_machine_code(code.as_slice()), + data: data, + } } From cf05780b17808b9b77fd071034e3d69b6d14816a Mon Sep 17 00:00:00 2001 From: Morgan Thomas Date: Wed, 3 Apr 2024 20:40:00 -0400 Subject: [PATCH 3/7] wip: ELF file support --- Cargo.lock | 1 + basic/Cargo.toml | 1 + basic/src/bin/valida.rs | 13 +++++++------ elf/src/lib.rs | 16 ++++++++++++++-- static_data/src/lib.rs | 4 ++++ 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 16db62d..a765341 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1079,6 +1079,7 @@ dependencies = [ "valida-bus", "valida-cpu", "valida-derive", + "valida-elf", "valida-machine", "valida-memory", "valida-opcodes", diff --git a/basic/Cargo.toml b/basic/Cargo.toml index 9662682..9c340ed 100644 --- a/basic/Cargo.toml +++ b/basic/Cargo.toml @@ -25,6 +25,7 @@ valida-assembler = { path = "../assembler" } valida-bus = { path = "../bus" } valida-cpu = { path = "../cpu" } valida-derive = { path = "../derive" } +valida-elf = { path = "../elf" } valida-machine = { path = "../machine" } valida-memory = { path = "../memory" } valida-opcodes = { path = "../opcodes" } diff --git a/basic/src/bin/valida.rs b/basic/src/bin/valida.rs index ebe1217..c462f9a 100644 --- a/basic/src/bin/valida.rs +++ b/basic/src/bin/valida.rs @@ -1,5 +1,6 @@ use clap::Parser; use std::fs::File; +use std::fs; use std::io::{stdout, Write}; use valida_basic::BasicMachine; @@ -10,7 +11,9 @@ use p3_fri::{FriConfig, TwoAdicFriPcs, TwoAdicFriPcsConfig}; use valida_cpu::MachineWithCpuChip; use valida_machine::{Machine, MachineProof, ProgramROM, StdinAdviceProvider}; +use valida_elf::{load_executable_file, Program}; use valida_program::MachineWithProgramChip; +use valida_static_data::MachineWithStaticDataChip; use p3_challenger::DuplexChallenger; use p3_dft::Radix2DitParallel; @@ -50,16 +53,14 @@ fn main() { let args = Args::parse(); let mut machine = BasicMachine::::default(); - let rom = match ProgramROM::from_file(&args.program) { - Ok(contents) => contents, - Err(e) => panic!("Failure to load file: {}. {}", &args.program, e), - }; - machine.program_mut().set_program_rom(&rom); + let Program { code, data } = load_executable_file(fs::read(&args.program).expect("Failed to read executable file")); + machine.program_mut().set_program_rom(&code); machine.cpu_mut().fp = args.stack_height; machine.cpu_mut().save_register_state(); + machine.static_data_mut().load(data); // Run the program - machine.run(&rom, &mut StdinAdviceProvider); + machine.run(&code, &mut StdinAdviceProvider); type Val = BabyBear; type Challenge = BinomialExtensionField; diff --git a/elf/src/lib.rs b/elf/src/lib.rs index 3e6db3a..bf188e6 100644 --- a/elf/src/lib.rs +++ b/elf/src/lib.rs @@ -12,8 +12,20 @@ use elf::section::SectionHeader; use valida_machine::{ProgramROM, Word}; pub struct Program { - code: ProgramROM, - data: BTreeMap>, + pub code: ProgramROM, + pub data: BTreeMap>, +} + +pub fn load_executable_file(file: Vec) -> Program { + if file[0] == 0x7F && file[1] == 0x45 + && file[2] == 0x4C && file[3] == 0x46 { + load_elf_object_file(file) + } else { + Program { + code: ProgramROM::from_machine_code(file.as_slice()), + data: BTreeMap::new(), + } + } } pub fn load_elf_object_file(file: Vec) -> Program { diff --git a/static_data/src/lib.rs b/static_data/src/lib.rs index 12889fd..df421d1 100644 --- a/static_data/src/lib.rs +++ b/static_data/src/lib.rs @@ -39,6 +39,10 @@ impl StaticDataChip { } } + pub fn load(&mut self, cells: BTreeMap>) { + self.cells = cells; + } + pub fn write(&mut self, address: u32, value: Word) { self.cells.insert(address, value); } From 57d16fa90a4cc983bfbf68a3d6b9b342f0fd8dcd Mon Sep 17 00:00:00 2001 From: Morgan Thomas Date: Wed, 3 Apr 2024 23:18:13 -0400 Subject: [PATCH 4/7] wip: ELF file support --- elf/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elf/src/lib.rs b/elf/src/lib.rs index bf188e6..8bcb019 100644 --- a/elf/src/lib.rs +++ b/elf/src/lib.rs @@ -75,7 +75,8 @@ pub fn load_elf_object_file(file: Vec) -> Program { let mut data: BTreeMap> = BTreeMap::new(); for (section_header, section_data) in data_sections { for i in 0 .. (section_header.sh_size / 4) as usize { - data.insert(section_header.sh_addr.try_into().unwrap(), + data.insert(>::try_into(section_header.sh_addr).unwrap() + + >::try_into(i*4).unwrap(), Word([section_data[i*4], section_data[i*4+1], section_data[i*4+2], section_data[i*4+3]])); } } From 6874437b5c37bd1c3b832ad9d1e59731dbdec139 Mon Sep 17 00:00:00 2001 From: Morgan Thomas Date: Fri, 5 Apr 2024 13:30:47 -0400 Subject: [PATCH 5/7] cargo fmt --- basic/src/bin/valida.rs | 5 +++-- elf/src/lib.rs | 47 ++++++++++++++++++++++------------------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/basic/src/bin/valida.rs b/basic/src/bin/valida.rs index c462f9a..98b3a9f 100644 --- a/basic/src/bin/valida.rs +++ b/basic/src/bin/valida.rs @@ -1,6 +1,6 @@ use clap::Parser; -use std::fs::File; use std::fs; +use std::fs::File; use std::io::{stdout, Write}; use valida_basic::BasicMachine; @@ -53,7 +53,8 @@ fn main() { let args = Args::parse(); let mut machine = BasicMachine::::default(); - let Program { code, data } = load_executable_file(fs::read(&args.program).expect("Failed to read executable file")); + let Program { code, data } = + load_executable_file(fs::read(&args.program).expect("Failed to read executable file")); machine.program_mut().set_program_rom(&code); machine.cpu_mut().fp = args.stack_height; machine.cpu_mut().save_register_state(); diff --git a/elf/src/lib.rs b/elf/src/lib.rs index 8bcb019..3cd5b4b 100644 --- a/elf/src/lib.rs +++ b/elf/src/lib.rs @@ -5,10 +5,10 @@ extern crate alloc; use alloc::collections::BTreeMap; use alloc::vec; use alloc::vec::Vec; -use elf::ElfBytes; use elf::abi; use elf::endian::AnyEndian; use elf::section::SectionHeader; +use elf::ElfBytes; use valida_machine::{ProgramROM, Word}; pub struct Program { @@ -17,8 +17,7 @@ pub struct Program { } pub fn load_executable_file(file: Vec) -> Program { - if file[0] == 0x7F && file[1] == 0x45 - && file[2] == 0x4C && file[3] == 0x46 { + if file[0] == 0x7F && file[1] == 0x45 && file[2] == 0x4C && file[3] == 0x46 { load_elf_object_file(file) } else { Program { @@ -30,16 +29,16 @@ pub fn load_executable_file(file: Vec) -> Program { pub fn load_elf_object_file(file: Vec) -> Program { let file = ElfBytes::::minimal_parse(file.as_slice()).unwrap(); - let mut data_sections: Vec::<(SectionHeader, &[u8])> = vec![]; - let mut bss_sections: Vec:: = vec![]; - let mut text_sections: Vec::<(SectionHeader, &[u8])> = vec![]; + let mut data_sections: Vec<(SectionHeader, &[u8])> = vec![]; + let mut bss_sections: Vec = vec![]; + let mut text_sections: Vec<(SectionHeader, &[u8])> = vec![]; for section_header in file.section_headers().unwrap().iter() { let is_data: bool = section_header.sh_type == abi::SHT_PROGBITS - && section_header.sh_flags == (abi::SHF_ALLOC | abi::SHF_WRITE).into(); + && section_header.sh_flags == (abi::SHF_ALLOC | abi::SHF_WRITE).into(); let is_bss: bool = section_header.sh_type == abi::SHT_NOBITS - && section_header.sh_flags == (abi::SHF_ALLOC | abi::SHF_WRITE).into(); + && section_header.sh_flags == (abi::SHF_ALLOC | abi::SHF_WRITE).into(); let is_text: bool = section_header.sh_type == abi::SHT_PROGBITS - && section_header.sh_flags == (abi::SHF_ALLOC | abi::SHF_EXECINSTR).into(); + && section_header.sh_flags == (abi::SHF_ALLOC | abi::SHF_EXECINSTR).into(); let is_useful: bool = is_data || is_bss || is_text; if is_useful { if is_data || is_text { @@ -51,7 +50,7 @@ pub fn load_elf_object_file(file: Vec) -> Program { } else if is_text { text_sections.push((section_header, section_data)); } - }, + } _ => panic!("unsupported: compressed ELF section data"), } } else if is_bss { @@ -59,25 +58,29 @@ pub fn load_elf_object_file(file: Vec) -> Program { } } } - let code_size = - text_sections - .iter() - .map(|(section_header, _)| { - section_header.sh_addr + section_header.sh_size - }) - .fold(0, |a, b| a.max(b)); + let code_size = text_sections + .iter() + .map(|(section_header, _)| section_header.sh_addr + section_header.sh_size) + .fold(0, |a, b| a.max(b)); let mut code: Vec = vec![0; code_size as usize]; for (section_header, section_data) in text_sections { - for i in 0 .. section_header.sh_size as usize { + for i in 0..section_header.sh_size as usize { code[i + section_header.sh_addr as usize] = section_data[i]; } } let mut data: BTreeMap> = BTreeMap::new(); for (section_header, section_data) in data_sections { - for i in 0 .. (section_header.sh_size / 4) as usize { - data.insert(>::try_into(section_header.sh_addr).unwrap() - + >::try_into(i*4).unwrap(), - Word([section_data[i*4], section_data[i*4+1], section_data[i*4+2], section_data[i*4+3]])); + for i in 0..(section_header.sh_size / 4) as usize { + data.insert( + >::try_into(section_header.sh_addr).unwrap() + + >::try_into(i * 4).unwrap(), + Word([ + section_data[i * 4], + section_data[i * 4 + 1], + section_data[i * 4 + 2], + section_data[i * 4 + 3], + ]), + ); } } Program { From ad8faea72d0078338b1a7a499186f9439cce73e8 Mon Sep 17 00:00:00 2001 From: Morgan Thomas Date: Mon, 8 Apr 2024 11:41:18 -0400 Subject: [PATCH 6/7] issue/135: better error message --- basic/src/bin/valida.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/basic/src/bin/valida.rs b/basic/src/bin/valida.rs index 98b3a9f..590f933 100644 --- a/basic/src/bin/valida.rs +++ b/basic/src/bin/valida.rs @@ -53,8 +53,10 @@ fn main() { let args = Args::parse(); let mut machine = BasicMachine::::default(); - let Program { code, data } = - load_executable_file(fs::read(&args.program).expect("Failed to read executable file")); + let Program { code, data } = load_executable_file( + fs::read(&args.program) + .expect(format!("Failed to read executable file: {}", &args.program).as_str()), + ); machine.program_mut().set_program_rom(&code); machine.cpu_mut().fp = args.stack_height; machine.cpu_mut().save_register_state(); From 28b8c5f7a7a5bd2d832375c9b04d7ee10867198a Mon Sep 17 00:00:00 2001 From: thealmarty <“thealmartyblog@gmail.com”> Date: Mon, 8 Apr 2024 08:30:59 -0700 Subject: [PATCH 7/7] Fix warnings. --- basic/src/bin/test_prover.rs | 4 +--- basic/src/bin/valida.rs | 2 +- derive/src/lib.rs | 4 ++-- memory/src/stark.rs | 5 +---- program/src/lib.rs | 5 ++--- static_data/src/lib.rs | 6 +++--- 6 files changed, 10 insertions(+), 16 deletions(-) diff --git a/basic/src/bin/test_prover.rs b/basic/src/bin/test_prover.rs index f754bce..6cc91f6 100644 --- a/basic/src/bin/test_prover.rs +++ b/basic/src/bin/test_prover.rs @@ -2,7 +2,7 @@ extern crate core; use p3_baby_bear::BabyBear; use p3_fri::{TwoAdicFriPcs, TwoAdicFriPcsConfig}; -use valida_alu_u32::add::{Add32Instruction, MachineWithAdd32Chip}; +use valida_alu_u32::add::Add32Instruction; use valida_basic::BasicMachine; use valida_cpu::{ BeqInstruction, BneInstruction, Imm32Instruction, JalInstruction, JalvInstruction, @@ -10,10 +10,8 @@ use valida_cpu::{ }; use valida_machine::{ FixedAdviceProvider, Instruction, InstructionWord, Machine, MachineProof, Operands, ProgramROM, - Word, }; -use valida_memory::MachineWithMemoryChip; use valida_opcodes::BYTES_PER_INSTR; use valida_program::MachineWithProgramChip; diff --git a/basic/src/bin/valida.rs b/basic/src/bin/valida.rs index 590f933..1232ed8 100644 --- a/basic/src/bin/valida.rs +++ b/basic/src/bin/valida.rs @@ -9,7 +9,7 @@ use p3_baby_bear::BabyBear; use p3_fri::{FriConfig, TwoAdicFriPcs, TwoAdicFriPcsConfig}; use valida_cpu::MachineWithCpuChip; -use valida_machine::{Machine, MachineProof, ProgramROM, StdinAdviceProvider}; +use valida_machine::{Machine, MachineProof, StdinAdviceProvider}; use valida_elf::{load_executable_file, Program}; use valida_program::MachineWithProgramChip; diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 27a8c22..ba8dc38 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -10,7 +10,7 @@ use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use quote::quote; use syn::parse::{Parse, ParseStream}; -use syn::{spanned::Spanned, Data, Field, Fields, Ident, Type, TypePath}; +use syn::{spanned::Spanned, Data, Field, Fields, Ident}; // TODO: now trivial with a single field struct MachineFields { @@ -162,7 +162,7 @@ fn run_method( .collect::(); let init_static_data: TokenStream2 = match static_data_chip { - Some(static_data_chip) => quote! { + Some(_static_data_chip) => quote! { self.initialize_memory(); }, None => quote! {}, diff --git a/memory/src/stark.rs b/memory/src/stark.rs index b238da2..cac818b 100644 --- a/memory/src/stark.rs +++ b/memory/src/stark.rs @@ -1,10 +1,7 @@ -use crate::columns::{MemoryCols, NUM_MEM_COLS}; +use crate::columns::NUM_MEM_COLS; use crate::MemoryChip; -use core::borrow::Borrow; use p3_air::{Air, AirBuilder, BaseAir}; -use p3_field::AbstractField; -use p3_matrix::MatrixRowSlices; impl BaseAir for MemoryChip { fn width(&self) -> usize { diff --git a/program/src/lib.rs b/program/src/lib.rs index 1adea7e..145171b 100644 --- a/program/src/lib.rs +++ b/program/src/lib.rs @@ -2,14 +2,13 @@ extern crate alloc; -use crate::columns::{COL_MAP, NUM_PROGRAM_COLS, PREPROCESSED_COL_MAP}; +use crate::columns::NUM_PROGRAM_COLS; use alloc::vec; use alloc::vec::Vec; use valida_bus::MachineWithProgramBus; use valida_machine::{Chip, Interaction, Machine, ProgramROM}; use valida_util::pad_to_power_of_two; -use p3_air::VirtualPairCol; use p3_field::{AbstractField, Field}; use p3_matrix::dense::RowMajorMatrix; use valida_machine::StarkConfig; @@ -48,7 +47,7 @@ where RowMajorMatrix::new(values, NUM_PROGRAM_COLS) } - fn global_receives(&self, machine: &M) -> Vec> { + fn global_receives(&self, _machine: &M) -> Vec> { // let pc = VirtualPairCol::single_preprocessed(PREPROCESSED_COL_MAP.pc); // let opcode = VirtualPairCol::single_preprocessed(PREPROCESSED_COL_MAP.opcode); // let mut fields = vec![pc, opcode]; diff --git a/static_data/src/lib.rs b/static_data/src/lib.rs index df421d1..cbe3615 100644 --- a/static_data/src/lib.rs +++ b/static_data/src/lib.rs @@ -11,8 +11,8 @@ use p3_air::VirtualPairCol; use p3_field::{AbstractField, Field}; use p3_matrix::dense::RowMajorMatrix; use valida_bus::MachineWithMemBus; -use valida_machine::{BusArgument, Chip, Interaction, Machine, StarkConfig, Word}; -use valida_memory::{MachineWithMemoryChip, MemoryChip}; +use valida_machine::{Chip, Interaction, StarkConfig, Word}; +use valida_memory::MachineWithMemoryChip; pub mod columns; pub mod stark; @@ -57,7 +57,7 @@ where M: MachineWithMemBus, SC: StarkConfig, { - fn generate_trace(&self, machine: &M) -> RowMajorMatrix { + fn generate_trace(&self, _machine: &M) -> RowMajorMatrix { let mut rows = self .cells .iter()