From 7fbc73d32b78eaa0d15764f6804ace6a8fdd5d92 Mon Sep 17 00:00:00 2001 From: bmatthieu3 Date: Tue, 25 Feb 2025 10:36:13 +0100 Subject: [PATCH] allow access to raw bytes on image extension for cursor readers --- README.md | 18 +++--- src/hdu/data/bintable/data.rs | 4 +- src/hdu/data/image.rs | 107 +++++++++++++++++++++++----------- src/hdu/data/iter.rs | 2 +- src/hdu/data/mod.rs | 2 +- src/lib.rs | 91 ++++++++++++++++------------- 6 files changed, 136 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 0fc2ad6..d2d62e7 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ Example ```rust use std::fs::File; use std::io::Cursor; -use fitsrs::{Fits, ImageData, HDU, hdu::header::Xtension}; +use fitsrs::{Fits, ImageData, Pixels, HDU, hdu::header::Xtension}; use fitsrs::wcs::{ImgXY, LonLat}; use std::io::{BufReader, Read}; @@ -110,29 +110,29 @@ while let Some(Ok(hdu)) = hdu_list.next() { assert!((xy.x() - xy_2.x()).abs() <= 1e-9); assert!((xy.y() - xy_2.y()).abs() <= 1e-9); } - - match hdu_list.get_data(&hdu) { - ImageData::U8(it) => { + let image = hdu_list.get_data(&hdu); + match image.pixels() { + Pixels::U8(it) => { let data = it.collect::>(); assert_eq!(num_pixels, data.len()) }, - ImageData::I16(it) => { + Pixels::I16(it) => { let data = it.collect::>(); assert_eq!(num_pixels, data.len()) }, - ImageData::I32(it) => { + Pixels::I32(it) => { let data = it.collect::>(); assert_eq!(num_pixels, data.len()) }, - ImageData::I64(it) => { + Pixels::I64(it) => { let data = it.collect::>(); assert_eq!(num_pixels, data.len()) }, - ImageData::F32(it) => { + Pixels::F32(it) => { let data = it.collect::>(); assert_eq!(num_pixels, data.len()) }, - ImageData::F64(it) => { + Pixels::F64(it) => { let data = it.collect::>(); assert_eq!(num_pixels, data.len()) }, diff --git a/src/hdu/data/bintable/data.rs b/src/hdu/data/bintable/data.rs index 1cb3de5..ef9b9ed 100644 --- a/src/hdu/data/bintable/data.rs +++ b/src/hdu/data/bintable/data.rs @@ -144,7 +144,7 @@ pub struct TableData { heap: bool, } -impl TableData> +impl<'a, R> TableData<&'a mut Cursor> where R: AsRef<[u8]>, { @@ -161,7 +161,7 @@ where use std::io::Cursor; /// Get a reference to the inner reader for in-memory readers -impl TableData> { +impl<'a, R> TableData<&'a mut Cursor> { pub const fn get_ref(&self) -> &R { self.reader.get_ref() } diff --git a/src/hdu/data/image.rs b/src/hdu/data/image.rs index f5050e6..1b9f0d2 100644 --- a/src/hdu/data/image.rs +++ b/src/hdu/data/image.rs @@ -20,34 +20,17 @@ where { type Data = ImageData<&'a mut Self>; - fn read_data_unit(&'a mut self, header: &Header, _start_pos: u64) -> Self::Data { - ImageData::new(header.get_xtension(), self) + fn read_data_unit(&'a mut self, header: &Header, start_pos: u64) -> Self::Data { + ImageData::new(header.get_xtension(), self, start_pos) } } -use super::stream; -#[async_trait(?Send)] -impl<'a, R> AsyncDataBufRead<'a, Image> for futures::io::BufReader -where - R: AsyncReadExt + Debug + 'a + Unpin, -{ - type Data = DataStream<'a, Self>; - fn prepare_data_reading( - ctx: &Image, - num_remaining_bytes_in_cur_hdu: &'a mut usize, - reader: &'a mut Self, - ) -> Self::Data { - let bitpix = ctx.get_bitpix(); - match bitpix { - Bitpix::U8 => DataStream::U8(stream::St::new(reader, num_remaining_bytes_in_cur_hdu)), - Bitpix::I16 => DataStream::I16(stream::St::new(reader, num_remaining_bytes_in_cur_hdu)), - Bitpix::I32 => DataStream::I32(stream::St::new(reader, num_remaining_bytes_in_cur_hdu)), - Bitpix::I64 => DataStream::I64(stream::St::new(reader, num_remaining_bytes_in_cur_hdu)), - Bitpix::F32 => DataStream::F32(stream::St::new(reader, num_remaining_bytes_in_cur_hdu)), - Bitpix::F64 => DataStream::F64(stream::St::new(reader, num_remaining_bytes_in_cur_hdu)), - } - } +#[derive(Serialize, Debug)] +pub struct ImageData { + start_pos: u64, + num_bytes_data_block: u64, + pixels: Pixels, } /// An iterator on the data array @@ -58,7 +41,7 @@ where /// for non in-memory readers (typically BufReader) that ensures /// a file may not fit in memory #[derive(Serialize, Debug)] -pub enum ImageData { +pub enum Pixels { U8(It), I16(It), I32(It), @@ -71,16 +54,72 @@ impl ImageData where R: Read, { - pub(crate) fn new(ctx: &Image, reader: R) -> Self { + pub(crate) fn new(ctx: &Image, reader: R, start_pos: u64) -> Self { let limit = ctx.get_num_bytes_data_block(); - match ctx.get_bitpix() { - Bitpix::U8 => ImageData::U8(It::new(reader, limit)), - Bitpix::I16 => ImageData::I16(It::new(reader, limit)), - Bitpix::I32 => ImageData::I32(It::new(reader, limit)), - Bitpix::I64 => ImageData::I64(It::new(reader, limit)), - Bitpix::F32 => ImageData::F32(It::new(reader, limit)), - Bitpix::F64 => ImageData::F64(It::new(reader, limit)), - } + let pixels = match ctx.get_bitpix() { + Bitpix::U8 => Pixels::U8(It::new(reader, limit)), + Bitpix::I16 => Pixels::I16(It::new(reader, limit)), + Bitpix::I32 => Pixels::I32(It::new(reader, limit)), + Bitpix::I64 => Pixels::I64(It::new(reader, limit)), + Bitpix::F32 => Pixels::F32(It::new(reader, limit)), + Bitpix::F64 => Pixels::F64(It::new(reader, limit)), + }; + + Self { start_pos, num_bytes_data_block: limit, pixels } + } + + /// Get the pixels iterator of the image + pub fn pixels(self) -> Pixels { + self.pixels } } + +use std::io::Cursor; +impl<'a, R> ImageData<&'a mut Cursor> +where + R: AsRef<[u8]>, +{ + /// For in memory buffers, access the raw bytes of the image. + /// You might need to convert the data from big to little endian at some point + pub fn raw_bytes(&self) -> &[u8] { + let inner = match &self.pixels { + Pixels::U8(It { reader, ..}) => reader.get_ref(), + Pixels::I16(It { reader, ..}) => reader.get_ref(), + Pixels::I32(It { reader, ..}) => reader.get_ref(), + Pixels::I64(It { reader, ..}) => reader.get_ref(), + Pixels::F32(It { reader, ..}) => reader.get_ref(), + Pixels::F64(It { reader, ..}) => reader.get_ref(), + }; + let raw_bytes = inner.as_ref(); + + let s = self.start_pos as usize; + let e = s + (self.num_bytes_data_block as usize); + &raw_bytes[s..e] + } +} + +use super::stream; +#[async_trait(?Send)] +impl<'a, R> AsyncDataBufRead<'a, Image> for futures::io::BufReader +where + R: AsyncReadExt + Debug + 'a + Unpin, +{ + type Data = DataStream<'a, Self>; + + fn prepare_data_reading( + ctx: &Image, + num_remaining_bytes_in_cur_hdu: &'a mut usize, + reader: &'a mut Self, + ) -> Self::Data { + let bitpix = ctx.get_bitpix(); + match bitpix { + Bitpix::U8 => DataStream::U8(stream::St::new(reader, num_remaining_bytes_in_cur_hdu)), + Bitpix::I16 => DataStream::I16(stream::St::new(reader, num_remaining_bytes_in_cur_hdu)), + Bitpix::I32 => DataStream::I32(stream::St::new(reader, num_remaining_bytes_in_cur_hdu)), + Bitpix::I64 => DataStream::I64(stream::St::new(reader, num_remaining_bytes_in_cur_hdu)), + Bitpix::F32 => DataStream::F32(stream::St::new(reader, num_remaining_bytes_in_cur_hdu)), + Bitpix::F64 => DataStream::F64(stream::St::new(reader, num_remaining_bytes_in_cur_hdu)), + } + } +} \ No newline at end of file diff --git a/src/hdu/data/iter.rs b/src/hdu/data/iter.rs index 9f5734e..ac6b88f 100644 --- a/src/hdu/data/iter.rs +++ b/src/hdu/data/iter.rs @@ -48,7 +48,7 @@ impl Value for f64 { #[derive(Debug, Serialize)] pub struct It { /// The reader - reader: R, + pub(crate) reader: R, /// The number of items the reader must read num_items: usize, /// Number of item read diff --git a/src/hdu/data/mod.rs b/src/hdu/data/mod.rs index 3c76d18..3f7fcd6 100644 --- a/src/hdu/data/mod.rs +++ b/src/hdu/data/mod.rs @@ -6,7 +6,7 @@ pub mod stream; pub use bintable::BinaryTableData; pub use bintable::TableData; -pub use image::ImageData; +pub use image::{ImageData, Pixels}; pub use iter::It; diff --git a/src/lib.rs b/src/lib.rs index 48f9a32..8f0c097 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,7 +8,7 @@ //! use std::fs::File; //! use std::io::BufReader; //! -//! use fitsrs::{Fits, HDU, ImageData}; +//! use fitsrs::{Fits, HDU, ImageData, Pixels}; //! //! let f = File::open("samples/fits.gsfc.nasa.gov/HST_FOC.fits").unwrap(); //! let reader = BufReader::new(f); @@ -19,7 +19,8 @@ //! let naxis1 = *xtension.get_naxisn(1).unwrap() as usize; //! let naxis2 = *xtension.get_naxisn(2).unwrap() as usize; //! -//! if let ImageData::F32(it) = hdu_list.get_data(&hdu) { +//! let image = hdu_list.get_data(&hdu); +//! if let Pixels::F32(it) = image.pixels() { //! let data = it.collect::>(); //! assert_eq!(data.len(), naxis1 * naxis2); //! } else { @@ -48,7 +49,7 @@ pub use async_fits::AsyncFits; pub use file::FITSFile; pub use fits::Fits; pub use hdu::data::bintable::{BinaryTableData, DataValue, TableData, TableRowData}; -pub use hdu::data::image::ImageData; +pub use hdu::data::image::{ImageData, Pixels}; pub use hdu::data::iter::It; pub use hdu::{AsyncHDU, HDU}; @@ -56,9 +57,10 @@ pub use hdu::{AsyncHDU, HDU}; mod tests { use crate::async_fits::AsyncFits; use crate::fits::Fits; + use crate::hdu::data::image::Pixels; use crate::hdu::data::DataStream; use crate::hdu::AsyncHDU; - use crate::{FITSFile, ImageData}; + use crate::FITSFile; use crate::wcs::ImgXY; use crate::hdu::data::bintable::ColumnId; @@ -152,8 +154,9 @@ mod tests { let header = hdu.get_header(); let num_pixels = header.get_xtension().get_naxisn(1).unwrap() * header.get_xtension().get_naxisn(2).unwrap(); - match hdu_list.get_data(&hdu) { - ImageData::F32(it) => { + let image = hdu_list.get_data(&hdu); + match image.pixels() { + Pixels::F32(it) => { assert!(it.collect::>().len() as u64 == num_pixels); } _ => unreachable!(), @@ -178,8 +181,9 @@ mod tests { let header = hdu.get_header(); let num_pixels = header.get_xtension().get_naxisn(1).unwrap() * header.get_xtension().get_naxisn(2).unwrap(); - match hdu_list.get_data(&hdu) { - ImageData::I16(data) => { + let image = hdu_list.get_data(&hdu); + match image.pixels() { + Pixels::I16(data) => { assert!(data.collect::>().len() as u64 == num_pixels) } _ => unreachable!(), @@ -258,28 +262,29 @@ mod tests { .unwrap(); } - match hdu_list.get_data(&hdu) { - ImageData::I16(it) => { + let image = hdu_list.get_data(&hdu); + match image.pixels() { + Pixels::I16(it) => { let data = it.collect::>(); assert_eq!(data.len(), num_pixels); } - ImageData::U8(it) => { + Pixels::U8(it) => { let data = it.collect::>(); assert_eq!(data.len(), num_pixels); } - ImageData::I32(it) => { + Pixels::I32(it) => { let data = it.collect::>(); assert_eq!(data.len(), num_pixels); } - ImageData::I64(it) => { + Pixels::I64(it) => { let data = it.collect::>(); assert_eq!(data.len(), num_pixels); } - ImageData::F32(it) => { + Pixels::F32(it) => { let data = it.collect::>(); assert_eq!(data.len(), num_pixels); } - ImageData::F64(it) => { + Pixels::F64(it) => { let data = it.collect::>(); assert_eq!(data.len(), num_pixels); } @@ -302,8 +307,9 @@ mod tests { let xtension = hdu.get_header().get_xtension(); let naxis1 = *xtension.get_naxisn(1).unwrap(); let naxis2 = *xtension.get_naxisn(2).unwrap(); - match hdu_list.get_data(&hdu) { - ImageData::F32(data) => { + let image = hdu_list.get_data(&hdu); + match image.pixels() { + Pixels::F32(data) => { assert_eq!(data.collect::>().len(), (naxis1 * naxis2) as usize); } _ => unreachable!(), @@ -374,8 +380,9 @@ mod tests { let naxis1 = *xtension.get_naxisn(1).unwrap(); let naxis2 = *xtension.get_naxisn(2).unwrap(); - match hdu_list.get_data(&hdu) { - ImageData::F32(it) => { + let image = hdu_list.get_data(&hdu); + match image.pixels() { + Pixels::F32(it) => { let c = it .map(|v| (((v - min) / (max - min)) * 255.0) as u8) .collect::>(); @@ -418,24 +425,24 @@ mod tests { (Some(naxis1), Some(naxis2)) => { let num_pixels = (naxis2 * naxis1) as usize; - let data = hdu_list.get_data(&hdu); - match data { - ImageData::U8(it) => { + let image = hdu_list.get_data(&hdu); + match image.pixels() { + Pixels::U8(it) => { assert_eq!(num_pixels, it.collect::>().len()) } - ImageData::I16(it) => { + Pixels::I16(it) => { assert_eq!(num_pixels, it.collect::>().len()) } - ImageData::I32(it) => { + Pixels::I32(it) => { assert_eq!(num_pixels, it.collect::>().len()) } - ImageData::I64(it) => { + Pixels::I64(it) => { assert_eq!(num_pixels, it.collect::>().len()) } - ImageData::F32(it) => { + Pixels::F32(it) => { assert_eq!(num_pixels, it.collect::>().len()) } - ImageData::F64(it) => { + Pixels::F64(it) => { assert_eq!(num_pixels, it.collect::>().len()) } } @@ -480,28 +487,29 @@ mod tests { let num_pixels = naxis2 * naxis1; - match hdu_list.get_data(&hdu) { - ImageData::U8(it) => { + let image = hdu_list.get_data(&hdu); + match image.pixels() { + Pixels::U8(it) => { let data = it.collect::>(); assert_eq!(num_pixels as usize, data.len()) } - ImageData::I16(it) => { + Pixels::I16(it) => { let data = it.collect::>(); assert_eq!(num_pixels as usize, data.len()) } - ImageData::I32(it) => { + Pixels::I32(it) => { let data = it.collect::>(); assert_eq!(num_pixels as usize, data.len()) } - ImageData::I64(it) => { + Pixels::I64(it) => { let data = it.collect::>(); assert_eq!(num_pixels as usize, data.len()) } - ImageData::F32(it) => { + Pixels::F32(it) => { let data = it.collect::>(); assert_eq!(num_pixels as usize, data.len()) } - ImageData::F64(it) => { + Pixels::F64(it) => { let data = it.collect::>(); assert_eq!(num_pixels as usize, data.len()) } @@ -643,28 +651,29 @@ mod tests { let num_pixels = (naxis2 * naxis1) as usize; - match hdu_list.get_data(&hdu) { - ImageData::U8(it) => { + let data = hdu_list.get_data(&hdu); + match data.pixels() { + Pixels::U8(it) => { let data = it.collect::>(); assert_eq!(num_pixels, data.len()) } - ImageData::I16(it) => { + Pixels::I16(it) => { let data = it.collect::>(); assert_eq!(num_pixels, data.len()) } - ImageData::I32(it) => { + Pixels::I32(it) => { let data = it.collect::>(); assert_eq!(num_pixels, data.len()) } - ImageData::I64(it) => { + Pixels::I64(it) => { let data = it.collect::>(); assert_eq!(num_pixels, data.len()) } - ImageData::F32(it) => { + Pixels::F32(it) => { let data = it.collect::>(); assert_eq!(num_pixels, data.len()) } - ImageData::F64(it) => { + Pixels::F64(it) => { let data = it.collect::>(); assert_eq!(num_pixels, data.len()) }