Skip to content

Commit

Permalink
API breaking: introduce a rust iterator over the HDU. Fix UB
Browse files Browse the repository at this point in the history
  • Loading branch information
@bmatthieu3
bmatthieu3 committed Jan 10, 2025
1 parent 0f8d363 commit 21e2059
Show file tree
Hide file tree
Showing 21 changed files with 1,368 additions and 1,502 deletions.
11 changes: 11 additions & 0 deletions CHANGELOG
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# CHANGELOG.md

## 0.3.0 (unreleased)

Features:

- BREAKING API change. Provide an iterator over the HDU list

Bugfixes:

- Be more safe about UB [https://github.com/cds-astro/fitsrs/issues/5]. BITPIX > 8 needs to be read from BigEndian scheme. It is thus necessary to create a new owned vector from the reading bytes. For BITPIX = 8, data can be directly read from the memory without allocating anything.
4 changes: 2 additions & 2 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "fitsrs"
version = "0.2.12"
version = "0.3.0"
authors = ["Matthieu Baumann <[email protected]>"]
edition = "2018"
description = "Implementation of the FITS image parser"
Expand All @@ -21,7 +21,7 @@ exclude = [
byteorder = "1.4.2"
serde = { version = "1.0.152", features = ["derive"] }
serde_json = "1.0"
futures = "0.3.24"
futures = "0.3.31"
async-trait = "0.1.66"
quick-error = "2.0.1"

Expand Down
264 changes: 129 additions & 135 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,228 +47,222 @@ Example
----------

For files that can fit in memory
```rust
use fitsrs::{
fits::Fits,
hdu::{
data::InMemData,
extension::XtensionHDU
}
};

```rust
use std::fs::File;
use std::io::Cursor;
use std::io::Read;
use fitsrs::{Fits, HDU};
use fitsrs::{hdu::header::Xtension, hdu::data::Data};

let mut f = File::open("samples/fits.gsfc.nasa.gov/EUVE.fits").unwrap();

let mut buf = Vec::new();
f.read_to_end(&mut buf).unwrap();
let mut reader = Cursor::new(&buf[..]);
let reader = Cursor::new(&buf[..]);

let Fits { hdu } = Fits::from_reader(&mut reader).unwrap();
let mut hdu_list = Fits::from_reader(reader);

// Access the HDU extensions
let mut hdu_ext = hdu.next();

while let Ok(Some(hdu)) = hdu_ext {
match &hdu {
XtensionHDU::Image(xhdu) => {
let xtension = xhdu.get_header().get_xtension();
while let Some(Ok(hdu)) = hdu_list.next() {
match hdu {
HDU::Primary(_) => (),
HDU::XImage(hdu) => {
let xtension = hdu.get_header().get_xtension();

let naxis1 = *xtension.get_naxisn(1).unwrap() as usize;
let naxis2 = *xtension.get_naxisn(2).unwrap() as usize;

let num_pixels = naxis2 * naxis1;

match xhdu.get_data() {
InMemData::U8(mem) => assert_eq!(num_pixels, mem.len()),
InMemData::I16(mem) => assert_eq!(num_pixels, mem.len()),
InMemData::I32(mem) => assert_eq!(num_pixels, mem.len()),
InMemData::I64(mem) => assert_eq!(num_pixels, mem.len()),
InMemData::F32(mem) => assert_eq!(num_pixels, mem.len()),
InMemData::F64(mem) => assert_eq!(num_pixels, mem.len()),
match hdu.get_data(&mut hdu_list) {
Data::U8(mem) => assert_eq!(num_pixels, mem.len()),
Data::I16(mem) => assert_eq!(num_pixels, mem.len()),
Data::I32(mem) => assert_eq!(num_pixels, mem.len()),
Data::I64(mem) => assert_eq!(num_pixels, mem.len()),
Data::F32(mem) => assert_eq!(num_pixels, mem.len()),
Data::F64(mem) => assert_eq!(num_pixels, mem.len()),
}
},
XtensionHDU::BinTable(xhdu) => {
let num_bytes = xhdu.get_header()
HDU::XBinaryTable(hdu) => {
let num_bytes = hdu.get_header()
.get_xtension()
.get_num_bytes_data_block();

match xhdu.get_data() {
InMemData::U8(mem) => assert_eq!(num_bytes as usize, mem.len()),
match hdu.get_data(&mut hdu_list) {
Data::U8(mem) => assert_eq!(num_bytes as usize, mem.len()),
_ => unreachable!()
}
},
XtensionHDU::AsciiTable(xhdu) => {
let num_bytes = xhdu.get_header()
HDU::XASCIITable(hdu) => {
let num_bytes = hdu.get_header()
.get_xtension()
.get_num_bytes_data_block();

match xhdu.get_data() {
InMemData::U8(mem) => assert_eq!(num_bytes as usize, mem.len()),
match hdu.get_data(&mut hdu_list) {
Data::U8(mem) => assert_eq!(num_bytes as usize, mem.len()),
_ => unreachable!()
}
},
}

hdu_ext = hdu.next();
}
```

For files that may not be contained into the memory
```rust
use fitsrs::{
fits::Fits,
hdu::{
data::iter,
extension::XtensionHDU
}
};
For BufReader

```rust
use std::fs::File;
use std::io::Cursor;
use fitsrs::hdu::HDU;
use fitsrs::hdu::data::DataIter;
use fitsrs::fits::Fits;
use fitsrs::hdu::header::extension::Xtension;

use std::io::{BufReader, Read};

let f = File::open("samples/fits.gsfc.nasa.gov/EUVE.fits").unwrap();
let mut reader = BufReader::new(f);

let Fits { hdu } = Fits::from_reader(&mut reader).unwrap();
let reader = BufReader::new(f);

let mut hdu_ext = hdu.next();
let mut hdu_list = Fits::from_reader(reader);

while let Ok(Some(mut xhdu)) = hdu_ext {
match &mut xhdu {
XtensionHDU::Image(xhdu) => {
let xtension = xhdu.get_header().get_xtension();
while let Some(Ok(hdu)) = hdu_list.next() {
match hdu {
// skip the primary HDU
HDU::Primary(_) => (),
HDU::XImage(hdu) => {
let xtension = hdu.get_header().get_xtension();

let naxis1 = *xtension.get_naxisn(1).unwrap();
let naxis2 = *xtension.get_naxisn(2).unwrap();

let num_pixels = (naxis2 * naxis1) as usize;

match xhdu.get_data_mut() {
iter::Data::U8(it) => {
match hdu.get_data(&mut hdu_list) {
DataIter::U8(it) => {
let data = it.collect::<Vec<_>>();
assert_eq!(num_pixels, data.len())
},
iter::Data::I16(it) => {
DataIter::I16(it) => {
let data = it.collect::<Vec<_>>();
assert_eq!(num_pixels, data.len())
},
iter::Data::I32(it) => {
DataIter::I32(it) => {
let data = it.collect::<Vec<_>>();
assert_eq!(num_pixels, data.len())
},
iter::Data::I64(it) => {
DataIter::I64(it) => {
let data = it.collect::<Vec<_>>();
assert_eq!(num_pixels, data.len())
},
iter::Data::F32(it) => {
DataIter::F32(it) => {
let data = it.collect::<Vec<_>>();
assert_eq!(num_pixels, data.len())
},
iter::Data::F64(it) => {
DataIter::F64(it) => {
let data = it.collect::<Vec<_>>();
assert_eq!(num_pixels, data.len())
},
}
},
XtensionHDU::BinTable(xhdu) => {
let num_bytes = xhdu.get_header()
HDU::XBinaryTable(hdu) => {
let num_bytes = hdu.get_header()
.get_xtension()
.get_num_bytes_data_block();

let it_bytes = xhdu.get_data_mut();
let it_bytes = hdu.get_data(&mut hdu_list);
let data = it_bytes.collect::<Vec<_>>();
assert_eq!(num_bytes as usize, data.len());
},
XtensionHDU::AsciiTable(xhdu) => {
let num_bytes = xhdu.get_header()
HDU::XASCIITable(hdu) => {
let num_bytes = hdu.get_header()
.get_xtension()
.get_num_bytes_data_block();

let it_bytes = xhdu.get_data_mut();
let it_bytes = hdu.get_data(&mut hdu_list);
let data = it_bytes.collect::<Vec<_>>();
assert_eq!(num_bytes as usize, data.len());
},
}

hdu_ext = xhdu.next();
}
```

For async input readers:

```rust
use fitsrs::{
fits::AsyncFits,
hdu::{
data::stream,
extension::AsyncXtensionHDU
#[tokio::test]
async fn parse_fits_async() {
use std::fs::File;
use std::io::Cursor;
use fitsrs::hdu::AsyncHDU;
use fitsrs::hdu::data::stream::Stream;
use fitsrs::async_fits::AsyncFits;
use fitsrs::hdu::header::extension::Xtension;

use std::io::{BufReader, Read};

// reader needs to implement futures::io::AsyncRead
let f = File::open("samples/fits.gsfc.nasa.gov/EUVE.fits").unwrap();
let reader = BufReader::new(f);

let mut hdu_list = AsyncFits::from_reader(reader);

while let Some(Ok(mut hdu)) = hdu_list.next().await {
match hdu {
AsyncHDU::Primary(_) => (),
AsyncHDU::Image(hdu) => {
let xtension = hdu.get_header().get_xtension();

let naxis1 = *xtension.get_naxisn(1).unwrap() as usize;
let naxis2 = *xtension.get_naxisn(2).unwrap() as usize;

let num_pixels = naxis2 * naxis1;

match hdu.get_data(&mut hdu_list) {
Stream::U8(st) => {
let data = st.collect::<Vec<_>>().await;
assert_eq!(num_pixels, data.len())
},
Stream::I16(st) => {
let data = st.collect::<Vec<_>>().await;
assert_eq!(num_pixels, data.len())
},
Stream::I32(st) => {
let data = st.collect::<Vec<_>>().await;
assert_eq!(num_pixels, data.len())
},
Stream::I64(st) => {
let data = st.collect::<Vec<_>>().await;
assert_eq!(num_pixels, data.len())
},
Stream::F32(st) => {
let data = st.collect::<Vec<_>>().await;
assert_eq!(num_pixels, data.len())
},
Stream::F64(st) => {
let data = st.collect::<Vec<_>>().await;
assert_eq!(num_pixels, data.len())
},
}
},
AsyncHDU::XBinaryTable(hdu) => {
let num_bytes = hdu.get_header()
.get_xtension()
.get_num_bytes_data_block();

let it_bytes = hdu.get_data(&mut hdu_list);
let data = it_bytes.collect::<Vec<_>>().await;
assert_eq!(num_bytes as usize, data.len());
},
AsyncHDU::XASCIITable(hdu) => {
let num_bytes = xhdu.get_header()
.get_xtension()
.get_num_bytes_data_block();

let it_bytes = xhdu.get_data(&mut hdu_list);
let data = it_bytes.collect::<Vec<_>>().await;
assert_eq!(num_bytes as usize, data.len());
},
}
}
};

// reader needs to implement futures::io::AsyncRead
let AsyncFits { hdu } = AsyncFits::from_reader(&mut reader).await.unwrap();

let mut hdu_ext = hdu.next().await;

while let Ok(Some(mut xhdu)) = hdu_ext {
match &mut xhdu {
AsyncXtensionHDU::Image(xhdu) => {
let xtension = xhdu.get_header().get_xtension();

let naxis1 = *xtension.get_naxisn(1).unwrap() as usize;
let naxis2 = *xtension.get_naxisn(2).unwrap() as usize;

let num_pixels = naxis2 * naxis1;

match xhdu.get_data_mut() {
stream::Data::U8(st) => {
let data = st.collect::<Vec<_>>().await;
assert_eq!(num_pixels, data.len())
},
stream::Data::I16(st) => {
let data = st.collect::<Vec<_>>().await;
assert_eq!(num_pixels, data.len())
},
stream::Data::I32(st) => {
let data = st.collect::<Vec<_>>().await;
assert_eq!(num_pixels, data.len())
},
stream::Data::I64(st) => {
let data = st.collect::<Vec<_>>().await;
assert_eq!(num_pixels, data.len())
},
stream::Data::F32(st) => {
let data = st.collect::<Vec<_>>().await;
assert_eq!(num_pixels, data.len())
},
stream::Data::F64(st) => {
let data = st.collect::<Vec<_>>().await;
assert_eq!(num_pixels, data.len())
},
}
},
AsyncXtensionHDU::BinTable(xhdu) => {
let num_bytes = xhdu.get_header()
.get_xtension()
.get_num_bytes_data_block();

let it_bytes = xhdu.get_data_mut();
let data = it_bytes.collect::<Vec<_>>().await;
assert_eq!(num_bytes as usize, data.len());
},
AsyncXtensionHDU::AsciiTable(xhdu) => {
let num_bytes = xhdu.get_header()
.get_xtension()
.get_num_bytes_data_block();

let it_bytes = xhdu.get_data_mut();
let data = it_bytes.collect::<Vec<_>>().await;
assert_eq!(num_bytes as usize, data.len());
},
}

hdu_ext = xhdu.next().await;
}
```
Loading

0 comments on commit 21e2059

Please sign in to comment.