Add 'discard_comment' and 'discard_event' methods for better generici…

…ty; clean a few things
cds-astro · Feb 8, 2024 · d3b24c1 · d3b24c1
1 parent 5a85487
commit d3b24c1
Show file tree

Hide file tree

Showing 53 changed files with 1,104 additions and 1,220 deletions.
diff --git a/README.md b/README.md
@@ -570,9 +570,11 @@ votable:
 * [ ] Add much more tests!
 * [X] Add possibility to convert to/from `TABLEDATA`, `BINARY`, `BINARY2`
     + but still to implement streaming mode in CLI
-* [ ] Enrich `votable::impls::Schema.serialize_seed` (possible bugs when deserializing JSON/TOML/YAML arrays and converting to BINARY or BINARY2)
-* [ ] Write a custom deserializer for `VOTableValue` (look at cargo-expand output for a basis)
 * [ ] Implements `toCSV` (but not `fromCSV`) in CLI
+* [ ] Replace `quick_error` by `anyhow` and `thiserror`.
+* [ ] Bump `quick_xml` version and: 
+    + [ ] Implement parsing without copy from `&[u8]` (when full data in men, e.g. in wasm, or for memmapped files) 
+    + [ ] Implement async
 * ...
 
 

diff --git a/resources/mivot/9/test_9_ok_9.5.xml b/resources/mivot/9/test_9_ok_9.5.xml
@@ -1,5 +1,4 @@
 <!-- Test Case 9.5: dmref + no sourceref + WHERE -->
 <JOIN dmref="aaaaa">
-  <!--<WHERE foreignkey="eee" value="ssss" />-->
-  <WHERE primarykey="eee" value="ssss" />
-</JOIN>
+  <WHERE primarykey="eee" foreignkey="ssss" />
+</JOIN>
diff --git a/src/coosys.rs b/src/coosys.rs
@@ -1,20 +1,25 @@
-use log::{debug, warn};
-use quick_xml::{
-  events::{attributes::Attributes, BytesStart, Event},
-  ElementWriter, Reader, Writer,
-};
+//! Module dedicated to the `COOSYS` tag.
+
 use std::{
   io::{BufRead, Write},
   num::ParseFloatError,
   str::{self, FromStr},
 };
 
+use log::warn;
 use paste::paste;
-
+use quick_xml::{
+  events::{attributes::Attributes, BytesStart, Event},
+  ElementWriter, Reader, Writer,
+};
 use serde;
 
 use crate::{
-  error::VOTableError, fieldref::FieldRef, paramref::ParamRef, timesys::RefPosition,
+  error::VOTableError,
+  fieldref::FieldRef,
+  paramref::ParamRef,
+  timesys::RefPosition,
+  utils::{discard_comment, discard_event},
   QuickXmlReadWrite, TableDataContent, VOTableVisitor,
 };
 
@@ -256,7 +261,8 @@ impl QuickXmlReadWrite for CooSys {
         },
         Event::End(e) if e.local_name() == Self::TAG_BYTES => return Ok(()),
         Event::Eof => return Err(VOTableError::PrematureEOF(Self::TAG)),
-        _ => debug!("Discarded event in {}: {:?}", Self::TAG, event),
+        Event::Comment(e) => discard_comment(e, reader, Self::TAG),
+        _ => discard_event(event, Self::TAG),
       }
     }
   }

diff --git a/src/data/binary.rs b/src/data/binary.rs
@@ -1,18 +1,24 @@
-use log::{debug, warn};
-use quick_xml::events::{BytesEnd, BytesStart};
-use quick_xml::{
-  events::{attributes::Attributes, Event},
-  Reader, Writer,
-};
+//! Module dedicated to the `BINARY` tag.
+
 use std::{
   io::{BufRead, Write},
   str,
 };
 
+use log::warn;
+use quick_xml::{
+  events::{attributes::Attributes, BytesEnd, BytesStart, Event},
+  Reader, Writer,
+};
 use serde;
 
 use super::{
-  super::{error::VOTableError, is_empty, table::TableElem, QuickXmlReadWrite, TableDataContent},
+  super::{
+    error::VOTableError,
+    table::TableElem,
+    utils::{discard_comment, discard_event, is_empty},
+    QuickXmlReadWrite, TableDataContent,
+  },
   stream::Stream,
 };
 
@@ -118,7 +124,8 @@ impl<C: TableDataContent> QuickXmlReadWrite for Binary<C> {
               match &mut event {
                 Event::Text(e) if is_empty(e) => {}
                 Event::End(e) if e.name() == Self::TAG_BYTES => return Ok(()),
-                _ => debug!("Discarded event in {}: {:?}", Self::TAG, event),
+                Event::Comment(e) => discard_comment(e, tmp_reader, Self::TAG),
+                _ => discard_event(event, Self::TAG),
               }
             }
           }
@@ -140,7 +147,8 @@ impl<C: TableDataContent> QuickXmlReadWrite for Binary<C> {
         },
         Event::Text(e) if is_empty(e) => {}
         Event::Eof => return Err(VOTableError::PrematureEOF(Self::TAG)),
-        _ => debug!("Discarded event in {}: {:?}", Self::TAG, event),
+        Event::Comment(e) => discard_comment(e, reader, Self::TAG),
+        _ => discard_event(event, Self::TAG),
       }
     }
   }

diff --git a/src/data/binary2.rs b/src/data/binary2.rs
@@ -1,18 +1,24 @@
-use log::{debug, warn};
-use quick_xml::events::{BytesEnd, BytesStart};
-use quick_xml::{
-  events::{attributes::Attributes, Event},
-  Reader, Writer,
-};
+//! Module dedicated to the `BINARY2` tag.
+
 use std::{
   io::{BufRead, Write},
   str,
 };
 
+use log::warn;
+use quick_xml::{
+  events::{attributes::Attributes, BytesEnd, BytesStart, Event},
+  Reader, Writer,
+};
 use serde;
 
 use super::{
-  super::{error::VOTableError, is_empty, table::TableElem, QuickXmlReadWrite, TableDataContent},
+  super::{
+    error::VOTableError,
+    table::TableElem,
+    utils::{discard_comment, discard_event, is_empty},
+    QuickXmlReadWrite, TableDataContent,
+  },
   stream::Stream,
 };
 
@@ -118,7 +124,8 @@ impl<C: TableDataContent> QuickXmlReadWrite for Binary2<C> {
               match &mut event {
                 Event::Text(e) if is_empty(e) => {}
                 Event::End(e) if e.name() == Self::TAG_BYTES => return Ok(()),
-                _ => debug!("Discarded event in {}: {:?}", Self::TAG, event),
+                Event::Comment(e) => discard_comment(e, tmp_reader, Self::TAG),
+                _ => discard_event(event, Self::TAG),
               }
             }
           }
@@ -140,7 +147,8 @@ impl<C: TableDataContent> QuickXmlReadWrite for Binary2<C> {
         },
         Event::Text(e) if is_empty(e) => {}
         Event::Eof => return Err(VOTableError::PrematureEOF(Self::TAG)),
-        _ => debug!("Discarded event in {}: {:?}", Self::TAG, event),
+        Event::Comment(e) => discard_comment(e, reader, Self::TAG),
+        _ => discard_event(event, Self::TAG),
       }
     }
   }

diff --git a/src/data/fits.rs b/src/data/fits.rs
@@ -1,9 +1,10 @@
+//! Module dedicated to the `FITS` tag.
+
 use std::{
   io::{BufRead, Write},
   str,
 };
 
-use log::debug;
 use paste::paste;
 use quick_xml::{
   events::{attributes::Attributes, BytesStart, Event},
@@ -14,7 +15,11 @@ use serde;
 use crate::impls::mem::VoidTableDataContent;
 
 use super::{
-  super::{error::VOTableError, QuickXmlReadWrite, TableDataContent, VOTableVisitor},
+  super::{
+    error::VOTableError,
+    utils::{discard_comment, discard_event},
+    QuickXmlReadWrite, TableDataContent, VOTableVisitor,
+  },
   stream::Stream,
 };
 
@@ -67,15 +72,26 @@ impl QuickXmlReadWrite for Fits {
   fn read_sub_elements<R: BufRead>(
     &mut self,
     mut reader: Reader<R>,
+    reader_buff: &mut Vec<u8>,
+    context: &Self::Context,
+  ) -> Result<Reader<R>, VOTableError> {
+    self
+      .read_sub_elements_by_ref(&mut reader, reader_buff, context)
+      .map(|()| reader)
+  }
+
+  fn read_sub_elements_by_ref<R: BufRead>(
+    &mut self,
+    mut reader: &mut Reader<R>,
     mut reader_buff: &mut Vec<u8>,
     _context: &Self::Context,
-  ) -> Result<Reader<R>, VOTableError> {
+  ) -> Result<(), VOTableError> {
     loop {
       let mut event = reader.read_event(reader_buff).map_err(VOTableError::Read)?;
       match &mut event {
         Event::Start(ref e) => match e.name() {
           Stream::<VoidTableDataContent>::TAG_BYTES => {
-            self.stream = from_event_start!(Stream, reader, reader_buff, e)
+            self.stream = from_event_start_by_ref!(Stream, reader, reader_buff, e)
           }
           _ => {
             return Err(VOTableError::UnexpectedStartTag(
@@ -93,22 +109,14 @@ impl QuickXmlReadWrite for Fits {
             ))
           }
         },
-        Event::End(e) if e.name() == Self::TAG_BYTES => return Ok(reader),
+        Event::End(e) if e.name() == Self::TAG_BYTES => return Ok(()),
         Event::Eof => return Err(VOTableError::PrematureEOF(Self::TAG)),
-        _ => debug!("Discarded event in {}: {:?}", Self::TAG, event),
+        Event::Comment(e) => discard_comment(e, reader, Self::TAG),
+        _ => discard_event(event, Self::TAG),
       }
     }
   }
 
-  fn read_sub_elements_by_ref<R: BufRead>(
-    &mut self,
-    _reader: &mut Reader<R>,
-    _reader_buff: &mut Vec<u8>,
-    _context: &Self::Context,
-  ) -> Result<(), VOTableError> {
-    todo!()
-  }
-
   fn write<W: Write>(
     &mut self,
     writer: &mut Writer<W>,

diff --git a/src/data/mod.rs b/src/data/mod.rs
@@ -1,26 +1,25 @@
-use log::{debug, warn};
-use quick_xml::{
-  events::{attributes::Attributes, BytesStart, Event},
-  Reader, Writer,
-};
+//! Module dedicated to the `DATA` tag.
 use std::{
   io::{BufRead, Write},
   str,
 };
 
+use log::warn;
 use paste::paste;
-
+use quick_xml::{
+  events::{attributes::Attributes, BytesStart, Event},
+  Reader, Writer,
+};
 use serde;
 
 use super::{
-  error::VOTableError, info::Info, table::TableElem, QuickXmlReadWrite, TableDataContent,
-  VOTableVisitor,
-};
-
-use crate::{
   data::stream::{EncodingType, Stream},
+  error::VOTableError,
   impls::mem::VoidTableDataContent,
-  is_empty,
+  info::Info,
+  table::TableElem,
+  utils::{discard_comment, discard_event, is_empty},
+  QuickXmlReadWrite, TableDataContent, VOTableVisitor,
 };
 
 // Sub modules
@@ -268,7 +267,8 @@ impl<C: TableDataContent> Data<C> {
         Event::Text(e) if is_empty(e) => {}
         Event::End(e) if e.local_name() == Self::TAG_BYTES => return Ok(None),
         Event::Eof => return Err(VOTableError::PrematureEOF(Self::TAG)),
-        _ => debug!("Discarded event in {}: {:?}", Self::TAG, event),
+        Event::Comment(e) => discard_comment(e, reader, Self::TAG),
+        _ => discard_event(event, Self::TAG),
       }
     }
   }
@@ -386,7 +386,7 @@ impl<C: TableDataContent> QuickXmlReadWrite for Data<C> {
         Event::Text(e) if is_empty(e) => {}
         Event::End(e) if e.local_name() == Self::TAG_BYTES => return Ok(()),
         Event::Eof => return Err(VOTableError::PrematureEOF(Self::TAG)),
-        _ => debug!("Discarded event in {}: {:?}", Self::TAG, event),
+        _ => discard_event(event, Self::TAG),
       }
     }
   }

diff --git a/src/data/stream.rs b/src/data/stream.rs
@@ -1,19 +1,22 @@
-use log::debug;
-use quick_xml::{
-  events::{attributes::Attributes, BytesStart, Event},
-  Reader, Writer,
-};
+//! Module dedicated to the `STREAM` tag.
+
 use std::{
   fmt,
   io::{BufRead, Write},
   str::{self, FromStr},
 };
 
 use paste::paste;
+use quick_xml::{
+  events::{attributes::Attributes, BytesStart, Event},
+  Reader, Writer,
+};
 
 use crate::{
-  error::VOTableError, impls::mem::VoidTableDataContent, is_empty, QuickXmlReadWrite,
-  TableDataContent,
+  error::VOTableError,
+  impls::mem::VoidTableDataContent,
+  utils::{discard_comment, discard_event, is_empty},
+  QuickXmlReadWrite, TableDataContent,
 };
 use serde;
 
@@ -177,7 +180,8 @@ impl Stream<VoidTableDataContent> {
         },
         Event::Text(e) if is_empty(e) => {}
         Event::Eof => return Err(VOTableError::PrematureEOF(Self::TAG)),
-        _ => debug!("Discarded event in {}: {:?}", Self::TAG, event),
+        Event::Comment(e) => discard_comment(e, reader, Self::TAG),
+        _ => discard_event(event, Self::TAG),
       }
     }
   }

diff --git a/src/data/tabledata.rs b/src/data/tabledata.rs
@@ -1,3 +1,5 @@
+//! Module dedicated to the `TABLEDATA` tag.
+
 use std::{
   io::{BufRead, Write},
   str,

diff --git a/src/datatype.rs b/src/datatype.rs
@@ -1,3 +1,5 @@
+//! Enum of the possible VOTable datatypes.
+
 use std::{fmt, str::FromStr};
 
 /// See [IVOA spec](https://www.ivoa.net/documents/VOTable/20191021/REC-VOTable-1.4-20191021.html#primitives)

diff --git a/src/definitions.rs b/src/definitions.rs
@@ -1,18 +1,21 @@
+//! Struct dedicated to the `DEFINITIONS` tag.
 use std::{
   io::{BufRead, Write},
   str,
 };
 
-use log::debug;
 use paste::paste;
 use quick_xml::{
   events::{attributes::Attributes, BytesStart, Event},
   Reader, Writer,
 };
 
 use super::{
-  coosys::CooSys, error::VOTableError, param::Param, QuickXmlReadWrite, TableDataContent,
-  VOTableVisitor,
+  coosys::CooSys,
+  error::VOTableError,
+  param::Param,
+  utils::{discard_comment, discard_event},
+  QuickXmlReadWrite, TableDataContent, VOTableVisitor,
 };
 
 #[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
@@ -153,7 +156,8 @@ impl QuickXmlReadWrite for Definitions {
         },
         Event::End(e) if e.local_name() == Self::TAG_BYTES => return Ok(()),
         Event::Eof => return Err(VOTableError::PrematureEOF(Self::TAG)),
-        _ => debug!("Discarded event in {}: {:?}", Self::TAG, event),
+        Event::Comment(e) => discard_comment(e, reader, Self::TAG),
+        _ => discard_event(event, Self::TAG),
       }
     }
   }

diff --git a/src/desc.rs b/src/desc.rs
@@ -1,3 +1,5 @@
+//! Struct dedicated to the `DESCRIPTION` tag.
+
 use std::io::{BufRead, Write};
 
 use log::{debug, warn};