Shuffle order of pub interfaces

Author: torymur

src/index.rs

@@ -159,31 +159,11 @@ impl Index {
         })
     }
 
-    /// Lists allowed tokens for a give state ID or `None` if it is not found in `Index`.
-    pub fn allowed_tokens(&self, state: &StateId) -> Option<Vec<TokenId>> {
-        self.transitions
-            .get(state)
-            .map(|res| res.keys().cloned().collect())
-    }
-
-    /// Returns transition state for a given state and token id or `None` otherwise.
-    pub fn next_state(&self, state: &StateId, token_id: &TokenId) -> Option<StateId> {
-        if token_id == &self.eos_token_id {
-            return None;
-        }
-        Some(*self.transitions.get(state)?.get(token_id)?)
-    }
-
     /// Returns the ID of the initial state in the automaton.
     pub fn initial_state(&self) -> StateId {
         self.initial_state
     }
 
-    /// Checks if state is in final states set or not.
-    pub fn is_final_state(&self, state: &StateId) -> bool {
-        self.final_states.contains(state)
-    }
-
     /// Returns set of final states.
     pub fn final_states(&self) -> &HashSet<StateId> {
         &self.final_states
@@ -193,6 +173,26 @@ impl Index {
     pub fn transitions(&self) -> &HashMap<StateId, HashMap<TokenId, StateId>> {
         &self.transitions
     }
+
+    /// Checks if state is in final states set or not.
+    pub fn is_final_state(&self, state: &StateId) -> bool {
+        self.final_states.contains(state)
+    }
+
+    /// Lists allowed tokens for a give state ID or `None` if it is not found in `Index`.
+    pub fn allowed_tokens(&self, state: &StateId) -> Option<Vec<TokenId>> {
+        self.transitions
+            .get(state)
+            .map(|res| res.keys().cloned().collect())
+    }
+
+    /// Returns transition state for a given state and token id or `None` otherwise.
+    pub fn next_state(&self, state: &StateId, token_id: &TokenId) -> Option<StateId> {
+        if token_id == &self.eos_token_id {
+            return None;
+        }
+        Some(*self.transitions.get(state)?.get(token_id)?)
+    }
 }
 
 impl std::fmt::Display for Index {

src/python_bindings/mod.rs

@@ -277,10 +277,6 @@ impl PyVocabulary {
         )))
     }
 
-    fn get_eos_token_id(&self) -> TokenId {
-        self.0.eos_token_id()
-    }
-
     fn get(&self, py: Python<'_>, token: Py<PyAny>) -> PyResult<Option<Vec<TokenId>>> {
         if let Ok(t) = token.extract::<String>(py) {
             return Ok(self.0.token_ids(t.into_bytes()).cloned());
@@ -294,6 +290,10 @@ impl PyVocabulary {
         )))
     }
 
+    fn get_eos_token_id(&self) -> TokenId {
+        self.0.eos_token_id()
+    }
+
     fn __repr__(&self) -> String {
         format!("{:#?}", self.0)
     }

src/vocabulary/mod.rs

@@ -66,22 +66,6 @@ impl Vocabulary {
         }
     }
 
-    /// Inserts a token to the vocabulary with the specified identifier.
-    pub fn try_insert(&mut self, token: impl Into<Token>, id: TokenId) -> Result<(), Error> {
-        if id == self.eos_token_id {
-            return Err(Error::EOSTokenDisallowed);
-        }
-        let token = token.into();
-        self.tokens.entry(token).or_default().push(id);
-        Ok(())
-    }
-
-    /// Removes given token from the vocabulary.
-    pub fn remove(&mut self, token: impl Into<Token>) {
-        let token = token.into();
-        self.tokens.remove(&token);
-    }
-
     /// Creates the vocabulary of pre-trained model from Hugging Face Hub.
     pub fn from_pretrained(
         model: &str,
@@ -148,6 +132,22 @@ impl Vocabulary {
         self.eos_token_id
     }
 
+    /// Inserts a token to the vocabulary with the specified identifier.
+    pub fn try_insert(&mut self, token: impl Into<Token>, id: TokenId) -> Result<(), Error> {
+        if id == self.eos_token_id {
+            return Err(Error::EOSTokenDisallowed);
+        }
+        let token = token.into();
+        self.tokens.entry(token).or_default().push(id);
+        Ok(())
+    }
+
+    /// Removes given token from the vocabulary.
+    pub fn remove(&mut self, token: impl Into<Token>) {
+        let token = token.into();
+        self.tokens.remove(&token);
+    }
+
     /// Filters out `Prepend` kind of tokenizer's normalizers.
     fn filter_prepend_normalizers(tokenizer: &mut Tokenizer) {
         // Main concern is prepend normalizers, for example https://github.com/google/sentencepiece