add some doc

src/consumer.rs

@@ -1,8 +1,57 @@
+//! # Data consumers
+//!
+//! The goal of data producers is to parse data depending on the previous result.
+//! It can be used to selectively seek in a file.
+//!
+//! ## Example
+//!
+//! This consumer will take 4 samples from the input, print them, then stop
+//!
+//! ```rust
+//!  use nom::{MemProducer,Consumer,ConsumerState};
+//!  use std::str;
+//!
+//!  struct TestPrintConsumer {
+//!    counter: usize
+//!  }
+//!
+//!  impl TestPrintConsumer {
+//!    fn new() -> TestPrintConsumer {
+//!      TestPrintConsumer { counter: 0 }
+//!    }
+//!  }
+//!
+//!  // Return ConsumerState::Await if it needs more data, or ConsumerDone when it ends
+//!  impl Consumer for TestPrintConsumer {
+//!    fn consume(&mut self, input: &[u8]) -> ConsumerState {
+//!      println!("{} -> {}", self.counter, str::from_utf8(input).unwrap());
+//!      self.counter = self.counter + 1;
+//!      if self.counter <=4 {
+//!        ConsumerState::Await
+//!      } else {
+//!        ConsumerState::ConsumerDone
+//!      }
+//!    }
+//!  }
+//!
+//!  // It can consume data directly from a producer
+//!  let mut p = MemProducer::new("abcdefghijklmnopqrstuvwx".as_bytes(), 4);
+//!  let mut c = TestPrintConsumer::new();
+//!  c.run(&mut p);
+//! ```
+
 use self::ConsumerState::*;
 use producer::Producer;
 use producer::ProducerState::*;
 use internal::Err;
 
+/// Holds the current state of the consumer
+///
+/// * Await if more data is needed
+///
+/// * ConsumerDone if the consumer does not need anymore data to be parsed
+///
+/// * ConsumerError when something went wrong
 #[derive(Show,PartialEq,Eq)]
 pub enum ConsumerState {
   Await,
@@ -11,6 +60,9 @@ pub enum ConsumerState {
   ConsumerError(Err)
 }
 
+/// Implement the consume method, taking a byte array as input and returning a consumer state
+///
+/// The run function takes care of continuing or not
 pub trait Consumer {
   fn consume(&mut self, input: &[u8]) -> ConsumerState;
   fn run(&mut self, producer: &mut Producer) {

src/internal.rs

@@ -1,10 +1,23 @@
 use self::IResult::*;
 
 pub type Err = u32;
-type IResultClosure<'a,I,O> = Box<FnMut(I) -> IResult<I,O> +'a>;
+pub type IResultClosure<'a,I,O> = Box<FnMut(I) -> IResult<I,O> +'a>;
 
 //type IResultClosure<'a,I,O> = |I|:'a -> IResult<'a,I,O>;
 //type IResultClosure<'a,I,O> = Fn<I, IResult<'a,I,O>>;
+
+/// Holds the result of parsing functions
+///
+/// It depends on I, the input types, and O, the output type.
+///
+/// * Done indicates a correct parsing, the first field containing the rest of the unparsed data,
+/// the second field contains the parsed data
+///
+/// * Error is currently a u32, but should be updated to indicate which parser had a problem,
+/// a description, and an eventual stack of parser to know which path failed
+///
+/// * Incomplete will hold the closure used to restart the computation once more data is available.
+/// Current attemps at implementation of Incomplete are progressing, but slowed down by lifetime problems
 #[derive(Show,PartialEq,Eq)]
 pub enum IResult<I,O> {
   Done(I,O),

src/producer.rs

@@ -1,9 +1,53 @@
+//! # Data producers
+//!
+//! The goal of data producers is to parse data as soon as it is generated.
+//!
+//! ## Example
+//!
+//! ```
+//! // invalid code for rustdoc
+//! //#![macro_use] extern crate nom;
+//! // use nom::pusher;
+//! // use std::str;
+//! // fn local_print<'a,T: Debug>(input: T) -> IResult<T, ()> {
+//! //   println!("{:?}", input);
+//! //   Done(input, ())
+//! // }
+//! // // create a data producer from a file
+//! // FileProducer::new("links.txt", 20).map(|producer: FileProducer| {
+//! //   let mut p = producer;
+//!
+//! //   // create the parsing function
+//! //   fn parser(par: IResult<(),&[u8]>) -> IResult<&[u8],()> {
+//! //     par.map_res(str::from_utf8).flat_map(local_print);
+//! //     Done("".as_bytes(), ())
+//! //   }
+//!
+//! //   // adapt the parsing function to the producer
+//! //   pusher!(push, parser);
+//! //   // get started
+//! //   push(&mut p);
+//! // });
+//!
+//! ```
+
 use internal::*;
 use self::ProducerState::*;
 
 use std::io::fs::File;
 use std::io::{IoResult, IoErrorKind};
 
+
+/// Holds the data producer's current state
+///
+/// * Eof indicates all data has been parsed, and contains the parser's result
+///
+/// * Continue indicates that more data is needed and should be available,
+/// but not right now. Parsing should resume at some point.
+///
+/// * Data contains already parsed data
+///
+/// * ProducerError indicates something went wrong
 #[derive(Debug,PartialEq,Eq)]
 pub enum ProducerState<O> {
   Eof(O),
@@ -12,9 +56,14 @@ pub enum ProducerState<O> {
   ProducerError(Err),
 }
 
+/// A producer implements the produce method, currently working with u8 arrays
 pub trait Producer {
   fn produce(&mut self) -> ProducerState<&[u8]>;
 }
+
+/// Can produce data from a file
+///
+/// the size field is the size of v, the internal buffer
 pub struct FileProducer {
   size: usize,
   file: File,
@@ -49,6 +98,15 @@ impl Producer for FileProducer {
   }
 }
 
+/// Can parse data from an already in memory byte array
+///
+/// * buffer holds the reference to the data that must be parsed
+///
+/// * length is the length of that buffer
+///
+/// * index is the position in the buffer
+///
+/// * chunk_size is the quantity of data sent at once
 pub struct MemProducer<'x> {
   buffer: &'x [u8],
   chunk_size: usize,
@@ -86,7 +144,22 @@ impl<'x> Producer for MemProducer<'x> {
   }
 }
 
-
+/// Prepares a parser function for a push pipeline
+///
+/// It creates a function that accepts a producer and immediately starts parsing the data sent
+///
+/// # Example
+///
+/// ```
+/// // invalid code for rustdoc
+/// //fn pr(par: IResult<(),&[u8]>) -> IResult<&[u8],()> {
+/// //  par.flat_map(local_print)
+/// //}
+/// //let mut p = MemProducer::new("abcdefgh".as_bytes(), 8);
+/// //
+/// //pusher!(ps, pr);
+/// //ps(&mut p);
+/// ```
 #[macro_export]
 macro_rules! pusher (
   ($name:ident, $f:expr) => (