diff --git a/README.md b/README.md index d30f5787..3d23a8cb 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,7 @@ Below are some examples of the currently supported models: | 49 | 0.5B | XLM-RoBERTa | [intfloat/multilingual-e5-large-instruct](https://hf.co/intfloat/multilingual-e5-large-instruct) | | N/A | 0.4B | Alibaba GTE | [Alibaba-NLP/gte-large-en-v1.5](https://hf.co/Alibaba-NLP/gte-large-en-v1.5) | | N/A | 0.4B | ModernBERT | [answerdotai/ModernBERT-large](https://hf.co/answerdotai/ModernBERT-large) | +| N/A | 0.3B | NomicBert | [nomic-ai/nomic-embed-text-v2-moe](https://hf.co/nomic-ai/nomic-embed-text-v2-moe) | | N/A | 0.1B | NomicBert | [nomic-ai/nomic-embed-text-v1](https://hf.co/nomic-ai/nomic-embed-text-v1) | | N/A | 0.1B | NomicBert | [nomic-ai/nomic-embed-text-v1.5](https://hf.co/nomic-ai/nomic-embed-text-v1.5) | | N/A | 0.1B | JinaBERT | [jinaai/jina-embeddings-v2-base-en](https://hf.co/jinaai/jina-embeddings-v2-base-en) | diff --git a/backends/candle/src/layers/linear.rs b/backends/candle/src/layers/linear.rs index fc8af1dd..4c875a89 100644 --- a/backends/candle/src/layers/linear.rs +++ b/backends/candle/src/layers/linear.rs @@ -11,6 +11,16 @@ pub enum HiddenAct { Swiglu, } +impl HiddenAct { + pub fn forward(&self, x: &Tensor) -> Result { + match self { + Self::Gelu => x.gelu(), + Self::Relu => x.relu(), + Self::Swiglu => candle_nn::ops::swiglu(x), + } + } +} + #[derive(Debug)] pub struct Linear { weight: Tensor, diff --git a/backends/candle/src/models/flash_nomic.rs b/backends/candle/src/models/flash_nomic.rs index 5683a86c..32cd31b6 100644 --- a/backends/candle/src/models/flash_nomic.rs +++ b/backends/candle/src/models/flash_nomic.rs @@ -1,6 +1,6 @@ use crate::flash_attn::flash_attn_varlen; use crate::layers::{get_cos_sin, get_inv_freqs, LayerNorm, Linear}; -use crate::models::nomic::{NomicBertEmbeddings, NomicBertGatedMLP}; +use crate::models::nomic::{NomicBertEmbeddings, NomicMLP}; use crate::models::{Model, NomicConfig}; use candle::{DType, Device, IndexOp, Result, Tensor, D}; use candle_nn::VarBuilder; @@ -25,16 +25,25 @@ impl NomicAttention { let attention_head_size = config.n_embd / config.n_head; let hidden_size = config.n_embd; - let qkv_weight = vb.pp("Wqkv").get( - (3 * num_attention_heads * attention_head_size, hidden_size), - "weight", - )?; - let qkv_linear = Linear::new(qkv_weight, None, None); + let qkv_dim = 3 * num_attention_heads * attention_head_size; + + let qkv_weight = vb.pp("Wqkv").get((qkv_dim, hidden_size), "weight")?; + let qkv_bias = if config.qkv_proj_bias { + Some(vb.pp("Wqkv").get((qkv_dim,), "bias")?) + } else { + None + }; + let qkv_linear = Linear::new(qkv_weight, qkv_bias, None); let out_proj_weight = vb .pp("out_proj") .get((hidden_size, hidden_size), "weight")?; - let out_proj = Linear::new(out_proj_weight, None, None); + let out_proj_bias = if config.qkv_proj_bias { + Some(vb.pp("out_proj").get((hidden_size,), "bias")?) + } else { + None + }; + let out_proj = Linear::new(out_proj_weight, out_proj_bias, None); let softmax_scale = (1. / (attention_head_size as f64).sqrt()) as f32; @@ -93,7 +102,7 @@ impl NomicAttention { struct NomicBertBlock { attention: NomicAttention, - mlp: NomicBertGatedMLP, + mlp: NomicMLP, post_attention_layer_norm: LayerNorm, output_layer_norm: LayerNorm, @@ -101,9 +110,10 @@ struct NomicBertBlock { } impl NomicBertBlock { - pub fn load(vb: VarBuilder, config: &NomicConfig) -> Result { + pub fn load(vb: VarBuilder, index: usize, config: &NomicConfig) -> Result { let attention = NomicAttention::load(vb.pp("attn"), config)?; - let mlp = NomicBertGatedMLP::load(vb.pp("mlp"), config)?; + + let mlp = NomicMLP::load(vb.pp("mlp"), index, config)?; let post_attention_layer_norm = LayerNorm::load(vb.pp("norm1"), config.n_embd, config.layer_norm_epsilon)?; @@ -132,6 +142,7 @@ impl NomicBertBlock { let attn_output = self .attention .forward(&hidden_states, cu_seqlens, cos, sin, max_s)?; + let hidden_states = self .post_attention_layer_norm .forward(&hidden_states, Some(&attn_output))?; @@ -145,13 +156,14 @@ impl NomicBertBlock { struct NomicBertEncoder { layers: Vec, + span: tracing::Span, } impl NomicBertEncoder { pub fn load(vb: VarBuilder, config: &NomicConfig) -> Result { let layers = (0..config.n_layer) - .map(|index| NomicBertBlock::load(vb.pp(format!("layers.{index}")), config)) + .map(|index| NomicBertBlock::load(vb.pp(format!("layers.{index}")), index, config)) .collect::>>()?; let span = tracing::span!(tracing::Level::TRACE, "encoder"); @@ -170,7 +182,6 @@ impl NomicBertEncoder { let mut hidden_states = hidden_states.clone(); - // Use a loop rather than a fold as it's easier to modify when adding debug/... for layer in self.layers.iter() { hidden_states = layer.forward(&hidden_states, cu_seqlens, cos, sin, max_s)? } @@ -419,6 +430,7 @@ impl Model for FlashNomicBertModel { fn is_padded(&self) -> bool { false } + fn embed(&self, batch: Batch) -> Result<(Option, Option)> { self.forward(batch) } diff --git a/backends/candle/src/models/nomic.rs b/backends/candle/src/models/nomic.rs index c860cfbc..8748db38 100644 --- a/backends/candle/src/models/nomic.rs +++ b/backends/candle/src/models/nomic.rs @@ -4,6 +4,7 @@ use crate::layers::{ use crate::models::Model; use candle::{DType, Device, IndexOp, Module, Result, Tensor, D}; use candle_nn::{Embedding, VarBuilder}; +use candle_transformers::models::deepseek2::{BincountOp, NonZeroOp, TopKLastDimOp, TopKOutput}; use serde::Deserialize; use text_embeddings_backend_core::{Batch, ModelType, Pool}; @@ -20,6 +21,11 @@ pub struct NomicConfig { #[serde(default = "default_max_trained_positions")] pub max_trained_positions: usize, + pub moe_every_n_layers: Option, + pub moe_normalize_expert_weights: Option, + pub moe_top_k: Option, + pub num_experts: Option, + pub n_embd: usize, pub n_head: usize, pub n_inner: usize, @@ -42,12 +48,8 @@ impl NomicConfig { pub fn valid(&self) -> bool { !self.prenorm && self.rotary_emb_fraction == 1.0 - && !self.qkv_proj_bias && !self.rotary_emb_interleaved - && !self.mlp_fc1_bias - && !self.mlp_fc2_bias && self.type_vocab_size > 0 - && self.activation_function == HiddenAct::Swiglu } } @@ -56,6 +58,7 @@ pub struct NomicBertEmbeddings { word_embeddings: Embedding, token_type_embeddings: Embedding, layer_norm: LayerNorm, + span: tracing::Span, } @@ -92,8 +95,8 @@ impl NomicBertEmbeddings { } pub struct NomicBertGatedMLP { - gate_up_proj: Linear, - down_proj: Linear, + fc1: Linear, + fc2: Linear, span: tracing::Span, } @@ -102,29 +105,91 @@ impl NomicBertGatedMLP { pub fn load(vb: VarBuilder, config: &NomicConfig) -> Result { let intermediate_size = config.n_inner; - let gate_proj_weight = vb + let fc11_weight = vb + .pp("fc11") + .get((intermediate_size, config.n_embd), "weight")?; + let fc12_weight = vb .pp("fc12") .get((intermediate_size, config.n_embd), "weight")?; + let fc1_weight = Tensor::cat(&[fc12_weight, fc11_weight], 0)?; - let up_proj_weight = vb - .pp("fc11") - .get((intermediate_size, config.n_embd), "weight")?; + let fc1_bias = if config.mlp_fc1_bias { + let fc11_bias = vb.pp("fc11").get((intermediate_size,), "bias")?; + let fc12_bias = vb.pp("fc12").get((intermediate_size,), "bias")?; + Some(Tensor::cat(&[fc12_bias, fc11_bias], 0)?) + } else { + None + }; - let gate_up_proj_weight = Tensor::cat(&[&gate_proj_weight, &up_proj_weight], 0)?; - let gate_up_proj = Linear::new( - gate_up_proj_weight, - None, + let fc1 = Linear::new( + fc1_weight, + fc1_bias, Some(config.activation_function.clone()), ); - let down_proj_weight = vb + let fc2_weight = vb .pp("fc2") .get((config.n_embd, intermediate_size), "weight")?; - let down_proj = Linear::new(down_proj_weight, None, None); + let fc2_bias = if config.mlp_fc2_bias { + Some(vb.pp("fc2").get((config.n_embd,), "bias")?) + } else { + None + }; + let fc2 = Linear::new(fc2_weight, fc2_bias, None); Ok(Self { - gate_up_proj, - down_proj, + fc1, + fc2, + span: tracing::span!(tracing::Level::TRACE, "gated_mlp"), + }) + } + + pub fn forward(&self, hidden_states: &Tensor) -> Result { + let _enter = self.span.enter(); + + let gate_up_states = self.fc1.forward(hidden_states)?; + self.fc2.forward(&gate_up_states) + } +} + +pub struct NomicBertMLP { + fc1: Linear, + fc2: Linear, + + span: tracing::Span, +} + +impl NomicBertMLP { + pub fn load(vb: VarBuilder, config: &NomicConfig) -> Result { + let intermediate_size = config.n_inner; + + let fc1_weight = vb + .pp("fc1") + .get((intermediate_size, config.n_embd), "weight")?; + let fc1_bias = if config.mlp_fc1_bias { + Some(vb.pp("fc1").get((intermediate_size,), "bias")?) + } else { + None + }; + let fc1 = Linear::new( + fc1_weight, + fc1_bias, + Some(config.activation_function.clone()), + ); + + let fc2_weight = vb + .pp("fc2") + .get((config.n_embd, intermediate_size), "weight")?; + let fc2_bias = if config.mlp_fc2_bias { + Some(vb.pp("fc2").get((config.n_embd,), "bias")?) + } else { + None + }; + let fc2 = Linear::new(fc2_weight, fc2_bias, None); + + Ok(Self { + fc1, + fc2, span: tracing::span!(tracing::Level::TRACE, "mlp"), }) } @@ -132,8 +197,226 @@ impl NomicBertGatedMLP { pub fn forward(&self, hidden_states: &Tensor) -> Result { let _enter = self.span.enter(); - let gate_up_states = self.gate_up_proj.forward(hidden_states)?; - self.down_proj.forward(&gate_up_states) + let y = self.fc1.forward(hidden_states)?; + self.fc2.forward(&y) + } +} + +pub struct NomicRouter { + layer: Linear, + moe_top_k: usize, + + span: tracing::Span, +} + +impl NomicRouter { + pub fn load(vb: VarBuilder, config: &NomicConfig) -> Result { + let num_experts = config.num_experts.unwrap(); + let moe_top_k = config.moe_top_k.unwrap(); + + let layer_weight = vb.pp("layer").get((num_experts, config.n_embd), "weight")?; + let layer = Linear::new(layer_weight, None, None); + + Ok(Self { + layer, + moe_top_k, + span: tracing::span!(tracing::Level::TRACE, "router"), + }) + } + + pub fn forward(&self, hidden_states: &Tensor) -> Result<(Tensor, Tensor)> { + let _enter = self.span.enter(); + + let weights = hidden_states.reshape(((), hidden_states.dim(D::Minus1)?))?; + let weights = self.layer.forward(&weights)?.to_dtype(DType::F32)?; + let weights = candle_nn::ops::softmax_last_dim(&weights)?; + + let TopKOutput { values, indices } = weights.topk(self.moe_top_k)?; + + let values = values.to_dtype(hidden_states.dtype())?; + + Ok((values, indices)) + } +} + +pub struct NomicExpertMLP { + w1: Tensor, + w2: Tensor, + activation: HiddenAct, + + span: tracing::Span, +} + +impl NomicExpertMLP { + pub fn load(vb: VarBuilder, config: &NomicConfig) -> Result { + let hidden_size = config.n_embd; + let ffn_hidden_size = config.n_inner; + let moe_num_experts = config.num_experts.unwrap(); + let activation = config.activation_function.clone(); + + let w1 = vb + .get((moe_num_experts * ffn_hidden_size, hidden_size), "w1")? + .reshape((moe_num_experts, ffn_hidden_size, hidden_size))?; + let w2 = vb + .get((moe_num_experts * ffn_hidden_size, hidden_size), "w2")? + .reshape((moe_num_experts, ffn_hidden_size, hidden_size))?; + + Ok(Self { + w1, + w2, + activation, + span: tracing::span!(tracing::Level::TRACE, "expert_mlp"), + }) + } + + pub fn forward(&self, hidden_states: &Tensor, expert_idx: usize) -> Result { + let _enter = self.span.enter(); + + let expert_w1 = self.w1.narrow(0, expert_idx, 1)?.squeeze(0)?.t()?; + let expert_w2 = self.w2.narrow(0, expert_idx, 1)?.squeeze(0)?; + + let hidden_states = hidden_states.broadcast_matmul(&expert_w1)?; + let hidden_states = self.activation.forward(&hidden_states)?; + + hidden_states.broadcast_matmul(&expert_w2) + } +} + +pub struct NomicExperts { + moe_num_experts: usize, + mlp: NomicExpertMLP, + bias: Tensor, + + span: tracing::Span, +} + +impl NomicExperts { + pub fn load(vb: VarBuilder, config: &NomicConfig) -> Result { + let moe_num_experts = config.num_experts.unwrap(); + + let mlp = NomicExpertMLP::load(vb.pp("mlp"), config)?; + + let bias = vb.get((config.n_embd,), "bias")?; + + Ok(Self { + moe_num_experts, + mlp, + bias, + span: tracing::span!(tracing::Level::TRACE, "experts"), + }) + } + + pub fn forward( + &self, + hidden_states: &Tensor, + top_weights: &Tensor, + top_experts: &Tensor, + ) -> Result { + let _enter = self.span.enter(); + + let dims = hidden_states.dims(); + let ndim = dims.len(); + + let (bs, seq_len, hidden_size) = match ndim { + 3 => (dims[0], dims[1], dims[2]), + 2 => (1, dims[0], dims[1]), + _ => unreachable!(), + }; + + let hidden_states = hidden_states.reshape(((), hidden_size))?; + + let mut out = Tensor::zeros_like(&hidden_states)?; + + let counts = top_experts + .flatten_all()? + .bincount(self.moe_num_experts as u32)?; + + for (expert_idx, &count) in counts.iter().enumerate().take(self.moe_num_experts) { + if count == 0u32 { + continue; + } + + let idx_top = top_experts.eq(expert_idx as f64)?.nonzero()?.t()?; + let idx = &idx_top.i(0)?.contiguous()?; + let top = &idx_top.i(1)?.contiguous()?; + + let expert_out = self + .mlp + .forward(&hidden_states.index_select(idx, 0)?, expert_idx)? + .broadcast_mul( + &top_weights + .index_select(idx, 0)? + .gather(&top.unsqueeze(1)?, 1)? + .squeeze(1)? + .unsqueeze(D::Minus1)? + .to_dtype(hidden_states.dtype())?, + )?; + + out = out.index_add(idx, &expert_out, 0)?; + } + + if ndim == 3 { + out = out.reshape((bs, seq_len, hidden_size))?; + } + + out.broadcast_add(&self.bias) + } +} + +pub struct NomicMoELayer { + router: NomicRouter, + experts: NomicExperts, + + span: tracing::Span, +} + +impl NomicMoELayer { + pub fn load(vb: VarBuilder, config: &NomicConfig) -> Result { + let router = NomicRouter::load(vb.pp("router"), config)?; + let experts = NomicExperts::load(vb.pp("experts"), config)?; + + Ok(Self { + router, + experts, + span: tracing::span!(tracing::Level::TRACE, "moe"), + }) + } + + pub fn forward(&self, hidden_states: &Tensor) -> Result { + let _enter = self.span.enter(); + + let (top_weights, top_experts) = self.router.forward(hidden_states)?; + + self.experts + .forward(hidden_states, &top_weights, &top_experts) + } +} + +pub enum NomicMLP { + MoE(NomicMoELayer), + GatedMLP(NomicBertGatedMLP), + Mlp(NomicBertMLP), +} + +impl NomicMLP { + pub fn load(vb: VarBuilder, index: usize, config: &NomicConfig) -> Result { + let use_moe = matches!(config.moe_every_n_layers, Some(n) if n > 0 && index % n == 1); + + if use_moe { + Ok(Self::MoE(NomicMoELayer::load(vb, config)?)) + } else if config.activation_function == HiddenAct::Gelu { + Ok(Self::Mlp(NomicBertMLP::load(vb, config)?)) + } else { + Ok(Self::GatedMLP(NomicBertGatedMLP::load(vb, config)?)) + } + } + + pub fn forward(&self, hidden_states: &Tensor) -> Result { + match self { + Self::MoE(layer) => layer.forward(hidden_states), + Self::GatedMLP(layer) => layer.forward(hidden_states), + Self::Mlp(layer) => layer.forward(hidden_states), + } } } @@ -155,16 +438,25 @@ impl NomicAttention { let attention_head_size = config.n_embd / config.n_head; let hidden_size = config.n_embd; - let qkv_weight = vb.pp("Wqkv").get( - (3 * num_attention_heads * attention_head_size, hidden_size), - "weight", - )?; - let qkv_linear = Linear::new(qkv_weight, None, None); + let qkv_dim = 3 * num_attention_heads * attention_head_size; + + let qkv_weight = vb.pp("Wqkv").get((qkv_dim, hidden_size), "weight")?; + let qkv_bias = if config.qkv_proj_bias { + Some(vb.pp("Wqkv").get((qkv_dim,), "bias")?) + } else { + None + }; + let qkv_linear = Linear::new(qkv_weight, qkv_bias, None); let out_proj_weight = vb .pp("out_proj") .get((hidden_size, hidden_size), "weight")?; - let out_proj = Linear::new(out_proj_weight, None, None); + let out_proj_bias = if config.qkv_proj_bias { + Some(vb.pp("out_proj").get((hidden_size,), "bias")?) + } else { + None + }; + let out_proj = Linear::new(out_proj_weight, out_proj_bias, None); let softmax_scale = 1. / (attention_head_size as f64).sqrt(); @@ -282,7 +574,7 @@ impl NomicAttention { struct NomicBertBlock { attention: NomicAttention, - mlp: NomicBertGatedMLP, + mlp: NomicMLP, post_attention_layer_norm: LayerNorm, output_layer_norm: LayerNorm, @@ -290,9 +582,10 @@ struct NomicBertBlock { } impl NomicBertBlock { - pub fn load(vb: VarBuilder, config: &NomicConfig) -> Result { + pub fn load(vb: VarBuilder, index: usize, config: &NomicConfig) -> Result { let attention = NomicAttention::load(vb.pp("attn"), config)?; - let mlp = NomicBertGatedMLP::load(vb.pp("mlp"), config)?; + + let mlp = NomicMLP::load(vb.pp("mlp"), index, config)?; let post_attention_layer_norm = LayerNorm::load(vb.pp("norm1"), config.n_embd, config.layer_norm_epsilon)?; @@ -320,6 +613,7 @@ impl NomicBertBlock { let attn_output = self .attention .forward(hidden_states, attention_bias, cos, sin)?; + let hidden_states = self .post_attention_layer_norm .forward(hidden_states, Some(&attn_output))?; @@ -333,13 +627,14 @@ impl NomicBertBlock { struct NomicBertEncoder { layers: Vec, + span: tracing::Span, } impl NomicBertEncoder { pub fn load(vb: VarBuilder, config: &NomicConfig) -> Result { let layers = (0..config.n_layer) - .map(|index| NomicBertBlock::load(vb.pp(format!("layers.{index}")), config)) + .map(|index| NomicBertBlock::load(vb.pp(format!("layers.{index}")), index, config)) .collect::>>()?; let span = tracing::span!(tracing::Level::TRACE, "encoder"); @@ -357,7 +652,6 @@ impl NomicBertEncoder { let mut hidden_states = hidden_states.clone(); - // Use a loop rather than a fold as it's easier to modify when adding debug/... for layer in self.layers.iter() { hidden_states = layer.forward(&hidden_states, attention_bias, cos, sin)? } @@ -444,6 +738,7 @@ impl NomicBertModel { span: tracing::span!(tracing::Level::TRACE, "model"), }) } + pub fn forward(&self, batch: Batch) -> Result<(Option, Option)> { let _enter = self.span.enter(); diff --git a/backends/candle/tests/snapshots/test_flash_nomic__nomic_moe_batch.snap b/backends/candle/tests/snapshots/test_flash_nomic__nomic_moe_batch.snap new file mode 100644 index 00000000..a7a46ce6 --- /dev/null +++ b/backends/candle/tests/snapshots/test_flash_nomic__nomic_moe_batch.snap @@ -0,0 +1,2308 @@ +--- +source: backends/candle/tests/test_flash_nomic.rs +expression: embeddings_batch +--- +- - -0.7705078 + - 0.06677246 + - -0.7470703 + - -0.13134766 + - 0.04748535 + - -0.71972656 + - 0.03213501 + - 0.069885254 + - -0.5288086 + - 1.6503906 + - -0.06713867 + - 0.18188477 + - -0.31958008 + - 1.5625 + - -0.2854004 + - 0.17663574 + - -0.107910156 + - -0.19873047 + - 0.53125 + - -0.09716797 + - -0.25170898 + - 0.20239258 + - -0.5 + - -0.5239258 + - -0.8730469 + - -0.26391602 + - -0.015304565 + - -0.34887695 + - 1.1074219 + - -0.11254883 + - -0.15429688 + - 0.24023438 + - -0.24291992 + - 0.48583984 + - -0.30273438 + - 0.50634766 + - -0.05923462 + - -0.07110596 + - 0.8364258 + - 0.056274414 + - -0.11968994 + - -0.037322998 + - 0.36865234 + - 0.6660156 + - -0.06542969 + - 0.5522461 + - 0.113464355 + - -0.085632324 + - 0.2763672 + - 0.09802246 + - -0.25317383 + - 0.41870117 + - 0.8491211 + - 0.044281006 + - 0.68847656 + - -0.2064209 + - -0.51123047 + - -0.4501953 + - 1.1611328 + - -0.65478516 + - -0.21289063 + - 0.44213867 + - 0.52685547 + - 0.23937988 + - 0.37109375 + - -1.0966797 + - 0.01638794 + - -0.57373047 + - -1.140625 + - 0.23571777 + - 0.014472961 + - -0.28515625 + - -0.06097412 + - -0.048461914 + - -0.72802734 + - 0.34228516 + - 0.34545898 + - -0.008087158 + - 0.6113281 + - -0.0914917 + - 0.076660156 + - -0.578125 + - 0.16308594 + - -0.33276367 + - -0.11755371 + - -0.8203125 + - -0.2590332 + - 0.26538086 + - -0.84472656 + - -0.5878906 + - 0.30517578 + - -0.20214844 + - 0.29736328 + - 0.58447266 + - 0.030792236 + - 0.0770874 + - 0.17492676 + - -0.47802734 + - 0.23095703 + - 0.5683594 + - -0.079833984 + - -0.30786133 + - -1.2099609 + - -0.5004883 + - -0.37963867 + - -0.06011963 + - -1.0009766 + - -0.39257813 + - -1.1767578 + - -0.20251465 + - -0.43408203 + - 0.30737305 + - -0.17541504 + - -0.8847656 + - 0.5498047 + - 0.23352051 + - 0.5107422 + - -0.13867188 + - 0.5058594 + - 0.734375 + - 0.42407227 + - -0.19702148 + - -0.20776367 + - 0.9238281 + - -0.34765625 + - -0.69970703 + - -0.3803711 + - 0.7866211 + - -0.36328125 + - -0.01499176 + - -0.21582031 + - 0.23181152 + - 0.21704102 + - 0.2824707 + - -0.5834961 + - -0.7910156 + - 0.21789551 + - 0.3137207 + - -0.7006836 + - -0.68603516 + - 0.5288086 + - 0.71875 + - 0.05545044 + - -0.15856934 + - -0.2163086 + - -0.060394287 + - 0.14733887 + - 0.30639648 + - 0.15148926 + - 0.19628906 + - -0.31640625 + - -0.15429688 + - -1.2783203 + - 0.15258789 + - -1.0078125 + - -0.0016040802 + - -0.037872314 + - -0.109069824 + - 0.84814453 + - -0.47875977 + - 1.1435547 + - 0.6303711 + - 0.13696289 + - 0.37426758 + - -0.19885254 + - 0.64453125 + - -0.19311523 + - 0.46899414 + - -0.16430664 + - -0.5205078 + - 0.13134766 + - -0.4423828 + - 0.4675293 + - 0.2763672 + - -0.40307617 + - -0.7910156 + - -0.08544922 + - -1.5429688 + - -0.5102539 + - 0.2142334 + - -0.71533203 + - 0.5605469 + - -0.6171875 + - 0.46557617 + - 0.9326172 + - 0.1430664 + - -0.7163086 + - 0.11743164 + - -0.24536133 + - 0.6455078 + - -0.32250977 + - 0.9404297 + - -0.36645508 + - 1.1279297 + - -0.85302734 + - 1.2333984 + - -0.45947266 + - 0.5175781 + - 0.5292969 + - -0.21606445 + - -0.5488281 + - -0.3395996 + - -0.17675781 + - -0.1661377 + - 0.71191406 + - -0.4128418 + - 0.084106445 + - 0.71972656 + - -0.2692871 + - 0.24963379 + - -0.72753906 + - -0.47192383 + - -0.92089844 + - 0.59375 + - -0.18041992 + - 0.2734375 + - 0.5541992 + - -1.2207031 + - 1.7294922 + - 0.055541992 + - 0.09539795 + - 0.5205078 + - 0.44604492 + - -1.1337891 + - 0.18603516 + - 0.38330078 + - 0.7949219 + - -0.5307617 + - -0.5830078 + - 0.21618652 + - 0.45629883 + - -0.42089844 + - -0.068359375 + - 0.9116211 + - 0.46191406 + - 0.3251953 + - 0.6845703 + - 0.15905762 + - -0.66748047 + - 0.5839844 + - -1.0595703 + - 0.4362793 + - -0.60009766 + - -0.30517578 + - 0.0076789856 + - -0.32421875 + - -0.052886963 + - -1.5175781 + - -0.5600586 + - 0.8183594 + - 0.20031738 + - 1.0556641 + - -0.3671875 + - 0.121398926 + - 0.19885254 + - 0.39941406 + - 0.42504883 + - -0.7895508 + - -0.23852539 + - -0.92871094 + - 0.65234375 + - -0.47583008 + - -0.48828125 + - 0.13574219 + - -0.015090942 + - -0.27270508 + - -0.38500977 + - -0.9160156 + - 1.4101563 + - 0.06616211 + - -0.54345703 + - 0.1907959 + - -0.91259766 + - -0.06915283 + - 0.2915039 + - 0.87353516 + - -0.66552734 + - 0.33007813 + - -0.70751953 + - -0.77001953 + - -0.6503906 + - -0.3071289 + - -0.93066406 + - -0.32592773 + - 0.72509766 + - -0.75 + - 0.08282471 + - 0.58251953 + - -1.2255859 + - 0.056152344 + - 0.46606445 + - 0.77001953 + - -0.5371094 + - -0.4519043 + - 0.10131836 + - -0.078063965 + - 0.009971619 + - -0.1763916 + - 0.004043579 + - -0.14074707 + - -0.025894165 + - 0.40039063 + - -0.30004883 + - 0.2290039 + - 0.1373291 + - 0.42871094 + - 0.39746094 + - -1.015625 + - 0.1451416 + - -0.24230957 + - 0.84375 + - 0.69628906 + - 0.048736572 + - -0.15625 + - 0.43579102 + - 0.6225586 + - 0.016616821 + - -0.78222656 + - -0.009765625 + - 0.7631836 + - -0.38110352 + - -0.19897461 + - 0.80908203 + - -0.011230469 + - 0.6586914 + - -0.34228516 + - -0.0045661926 + - -0.3725586 + - -0.5605469 + - 0.6269531 + - -0.06109619 + - -0.36767578 + - 0.0859375 + - 0.625 + - -0.65722656 + - 0.29345703 + - -1.1689453 + - 0.38745117 + - -0.1508789 + - -0.46777344 + - 0.8232422 + - 0.22314453 + - 0.39941406 + - 0.47509766 + - 0.32006836 + - -0.12158203 + - -0.36547852 + - 0.91503906 + - 0.16149902 + - 0.7216797 + - 0.006275177 + - -0.7446289 + - -0.484375 + - 0.19213867 + - 0.047058105 + - 1.8378906 + - 0.62841797 + - -0.101379395 + - 0.33764648 + - -0.009132385 + - 0.12548828 + - -1.3408203 + - -0.3527832 + - -0.23022461 + - -0.17346191 + - -1.0097656 + - -0.78564453 + - 0.85009766 + - -0.30737305 + - 0.5463867 + - -0.10083008 + - 0.48583984 + - 0.2019043 + - -0.40429688 + - 0.26586914 + - -0.5683594 + - 0.18823242 + - 0.27197266 + - -0.58154297 + - 1.0634766 + - -0.36035156 + - 1.0019531 + - 0.054229736 + - 0.578125 + - -0.9272461 + - -1.0791016 + - 0.6191406 + - -0.27246094 + - 0.1105957 + - 0.0006971359 + - -0.10699463 + - 0.1977539 + - 0.072509766 + - -0.19433594 + - 0.6928711 + - 0.6928711 + - 0.57666016 + - -1.1015625 + - 0.42456055 + - -0.5854492 + - -0.51171875 + - 0.18579102 + - -0.17785645 + - 0.3503418 + - -0.44091797 + - -0.20922852 + - -0.14465332 + - 0.08996582 + - 0.17663574 + - 0.5605469 + - 0.24658203 + - 0.4189453 + - -0.09814453 + - 0.7207031 + - 0.2902832 + - -0.25732422 + - 0.46875 + - -0.60498047 + - 0.31958008 + - -1.0341797 + - -0.72265625 + - 0.036590576 + - -0.13977051 + - 0.057373047 + - -0.4855957 + - 0.8413086 + - 0.29882813 + - -0.123413086 + - -0.07745361 + - 0.9003906 + - 0.27026367 + - 0.34033203 + - -0.74121094 + - 0.98046875 + - -0.16296387 + - -0.20385742 + - -0.022628784 + - -0.24487305 + - 0.4453125 + - 0.62353516 + - -0.43286133 + - -0.10321045 + - 0.3503418 + - -0.008575439 + - 0.69140625 + - 0.13427734 + - -0.52734375 + - 1.0117188 + - 0.091796875 + - 0.64208984 + - -0.068115234 + - -0.3095703 + - -0.55029297 + - 0.0970459 + - 0.18981934 + - -0.40161133 + - -0.07232666 + - 0.67822266 + - -0.20385742 + - 0.1977539 + - 0.9086914 + - 0.02482605 + - -0.28930664 + - 0.1348877 + - -0.4050293 + - 0.125 + - -0.050231934 + - 0.16809082 + - -0.6694336 + - 0.41210938 + - 0.04269409 + - 0.29541016 + - -0.57373047 + - -0.25854492 + - 1.1113281 + - 0.107788086 + - 0.36401367 + - -0.5600586 + - -0.036956787 + - 0.053466797 + - -0.12084961 + - -0.30297852 + - 0.5756836 + - 0.66259766 + - 0.42895508 + - 0.50390625 + - 0.21557617 + - 0.46801758 + - -0.49560547 + - 0.43847656 + - -0.4819336 + - -0.0036258698 + - 0.8105469 + - 0.2565918 + - -0.19348145 + - 0.9067383 + - -0.024169922 + - 0.31176758 + - 0.5683594 + - 0.24353027 + - 0.08831787 + - -0.07647705 + - 0.05206299 + - 0.484375 + - -0.5288086 + - 0.2565918 + - 0.23852539 + - -0.34765625 + - 0.1385498 + - -0.16430664 + - 0.5786133 + - 0.17822266 + - -1.0146484 + - -0.6660156 + - 0.60253906 + - 0.48339844 + - -0.71777344 + - -0.56103516 + - -0.043670654 + - -0.027313232 + - 0.5761719 + - -0.47021484 + - -0.43554688 + - -0.72753906 + - -0.7207031 + - -0.29296875 + - 0.0011854172 + - -0.50390625 + - -0.107543945 + - -1.0048828 + - -0.13342285 + - -0.70410156 + - -0.3239746 + - -0.7685547 + - -0.33203125 + - -0.014854431 + - -0.49438477 + - 1.0507813 + - 0.51953125 + - 0.25927734 + - 0.99853516 + - -0.35986328 + - 0.27612305 + - -0.45581055 + - -0.24609375 + - -0.015602112 + - -0.75097656 + - 0.072753906 + - 0.17822266 + - -0.5175781 + - 0.02204895 + - -0.022598267 + - 0.15820313 + - 0.61279297 + - 0.09082031 + - -0.75927734 + - -0.69384766 + - 0.7294922 + - 0.115722656 + - 0.107421875 + - -0.50390625 + - -0.74609375 + - -0.08081055 + - -0.13293457 + - -0.80322266 + - -0.043121338 + - 0.2692871 + - -0.13793945 + - -0.5830078 + - 0.105529785 + - 0.65185547 + - 0.13977051 + - 0.22387695 + - -0.20947266 + - -0.93066406 + - -1.0966797 + - 0.40649414 + - -0.052246094 + - -0.097839355 + - -0.055419922 + - -0.12011719 + - -0.3696289 + - 0.38671875 + - -0.10040283 + - 0.609375 + - -0.69677734 + - 0.4638672 + - -0.7080078 + - 1.3359375 + - 0.15576172 + - 0.04937744 + - 1.1611328 + - -0.7758789 + - -0.032470703 + - 0.2878418 + - 0.5180664 + - -0.21606445 + - -1.484375 + - 0.18261719 + - -0.053894043 + - -0.35864258 + - -0.4741211 + - 0.20581055 + - -0.05279541 + - -0.23022461 + - 0.9794922 + - 0.41918945 + - 0.31030273 + - 0.5493164 + - -0.038848877 + - -0.6430664 + - 0.41137695 + - -0.09442139 + - -0.57910156 + - -0.05834961 + - -0.35253906 + - -0.21386719 + - -0.21398926 + - 0.44360352 + - -0.21704102 + - -1.0146484 + - 0.14355469 + - 0.57910156 + - 0.609375 + - 0.63427734 + - -0.5205078 + - -0.27783203 + - 0.9116211 + - -0.18701172 + - -0.5258789 + - -0.9741211 + - -0.18432617 + - 0.7553711 + - -0.38305664 + - -1.171875 + - 0.58251953 + - 0.40942383 + - -0.19213867 + - 0.37646484 + - -0.40478516 + - -0.2763672 + - -0.84033203 + - -0.06427002 + - 1.4375 + - 0.13598633 + - 0.020401001 + - 0.1796875 + - -0.7763672 + - -0.49438477 + - -0.022628784 + - 0.39697266 + - -0.013198853 + - -0.6616211 + - 0.0025100708 + - 0.42236328 + - -0.36279297 + - -0.140625 + - 0.2998047 + - -0.67626953 + - -1.2607422 + - 0.5913086 + - 0.45776367 + - 0.13586426 + - 0.5913086 + - -0.07293701 + - -0.28344727 + - -0.060455322 + - -0.7109375 + - 0.122680664 + - 1.0273438 + - -0.5600586 + - 0.6020508 + - -0.48364258 + - -0.4404297 + - 0.26708984 + - 0.27246094 + - 0.44018555 + - -0.49829102 + - 0.070251465 + - -0.85595703 + - -0.092163086 + - -0.031707764 + - -0.99902344 + - 0.4765625 + - -0.07537842 + - -0.17993164 + - 0.6455078 + - 0.31518555 + - -0.14685059 + - -0.60498047 + - 0.6245117 + - 0.7734375 + - -0.58203125 + - 0.16564941 + - -0.39404297 + - 0.4921875 + - 0.60498047 + - -0.29785156 + - 1.5048828 + - 0.24853516 + - 0.75878906 + - 0.049682617 + - -0.2277832 + - -1.0244141 + - -0.18103027 + - 0.0119018555 + - -0.7949219 + - -0.20959473 + - 0.8955078 + - 0.04159546 + - -0.26391602 + - 0.24291992 + - 1.0224609 + - -0.7890625 + - 0.70654297 + - -0.7973633 + - 0.21679688 + - -0.33251953 + - 0.828125 + - -0.042938232 + - 0.6772461 + - 0.0259552 + - -0.070007324 + - 0.33789063 + - -0.3642578 + - 0.71728516 + - 0.2697754 + - -0.24584961 + - 1.03125 + - 0.000767231 + - 0.85009766 + - 0.67822266 + - 0.20043945 + - -0.088378906 + - -0.7597656 + - 0.16503906 + - -0.06088257 + - -0.43408203 + - 0.2770996 + - 0.3503418 + - 0.0014648438 + - 0.7709961 + - -0.9296875 + - -0.28027344 + - -0.29223633 + - -0.22912598 + - 0.5991211 + - 0.6899414 + - 0.30395508 + - 0.058502197 + - -0.7553711 + - -0.39990234 + - -0.02243042 + - -0.87597656 + - -0.9248047 + - 1.2861328 + - -0.14318848 + - 0.62939453 + - -0.25390625 + - -0.27905273 + - -0.5600586 + - 0.030822754 + - 0.43847656 + - -0.7451172 + - 0.8696289 + - 0.2697754 + - 0.3244629 + - -0.4272461 +- - -0.86376953 + - 0.10498047 + - -0.80859375 + - 0.109375 + - -0.15795898 + - -1.2734375 + - 0.053863525 + - -0.10333252 + - -0.3544922 + - 1.4140625 + - -0.2939453 + - 0.20715332 + - -0.31982422 + - 1.84375 + - 0.17443848 + - 0.22192383 + - -0.27392578 + - -0.6899414 + - 0.95996094 + - -0.009353638 + - -0.24121094 + - -0.33520508 + - -0.60791016 + - -0.0074043274 + - -1.0742188 + - -0.31152344 + - -0.16723633 + - -0.32861328 + - 0.84472656 + - -0.41333008 + - 0.12158203 + - -0.23828125 + - -0.55908203 + - 0.9160156 + - 0.25341797 + - 0.8876953 + - 0.29858398 + - -0.56152344 + - 0.6118164 + - 0.08117676 + - 0.49951172 + - 0.26660156 + - 0.057250977 + - 0.8378906 + - -0.0043525696 + - 0.6665039 + - -0.10949707 + - 0.35864258 + - -0.15722656 + - -0.043640137 + - -0.61816406 + - 0.5136719 + - 0.9375 + - -0.42285156 + - 0.61328125 + - 0.027496338 + - -0.7314453 + - 0.5786133 + - 0.80371094 + - -0.58203125 + - -0.77197266 + - 0.5966797 + - 0.4934082 + - 0.53808594 + - 0.8984375 + - -0.8378906 + - -0.002603531 + - -0.60595703 + - -0.18322754 + - -0.3010254 + - -0.058746338 + - -0.5957031 + - -0.21801758 + - 0.44970703 + - -1.0253906 + - 0.03010559 + - 0.5966797 + - 0.2915039 + - 0.59033203 + - -0.59033203 + - -0.21801758 + - -0.3383789 + - 0.26953125 + - -0.29711914 + - 0.29907227 + - -0.51660156 + - 0.1015625 + - 0.74072266 + - -1.2207031 + - -0.4309082 + - 0.050445557 + - -0.60058594 + - 0.7890625 + - 0.14550781 + - -0.11047363 + - 0.43017578 + - -0.6743164 + - -0.55859375 + - -0.1640625 + - 0.71191406 + - -0.4399414 + - -0.14367676 + - -1.6689453 + - -0.37817383 + - -0.10681152 + - 0.17443848 + - -0.2668457 + - 0.08544922 + - -1.1855469 + - -0.23986816 + - -0.734375 + - -0.6015625 + - -0.359375 + - -0.8984375 + - 0.5864258 + - -0.11779785 + - 0.37109375 + - -0.4086914 + - 0.67871094 + - 0.057800293 + - 0.7373047 + - 0.11230469 + - 0.02444458 + - -0.07330322 + - 0.0657959 + - -1.0058594 + - -0.9482422 + - 1.3339844 + - -0.73828125 + - 0.6816406 + - -0.49536133 + - 0.33203125 + - 0.13098145 + - -0.18029785 + - -0.14880371 + - -0.46728516 + - -0.40722656 + - 0.017242432 + - -0.81152344 + - -1.2460938 + - 0.5517578 + - 0.52246094 + - 0.055358887 + - -0.16210938 + - 0.22460938 + - 0.64160156 + - -0.36279297 + - 0.58203125 + - 0.18127441 + - -0.1340332 + - -1.3203125 + - -0.1574707 + - -1.2675781 + - -0.033935547 + - -0.25634766 + - 0.18322754 + - 0.024810791 + - -0.031280518 + - 0.41455078 + - -0.6464844 + - 1.265625 + - 0.28051758 + - 0.30322266 + - 0.39892578 + - -0.46435547 + - 1.0439453 + - -0.2919922 + - 0.47973633 + - 0.15441895 + - -0.24023438 + - -0.21313477 + - -0.70751953 + - 0.5 + - -0.2956543 + - 0.6152344 + - -0.77001953 + - -0.34765625 + - -1.3535156 + - -1.0195313 + - 0.35351563 + - -0.8808594 + - 0.67822266 + - -0.4609375 + - 0.31982422 + - 0.9555664 + - 0.045166016 + - -0.56152344 + - 0.4416504 + - 0.2626953 + - 0.38208008 + - -0.041046143 + - 0.7680664 + - -0.3930664 + - 0.7919922 + - -0.3798828 + - 1.5283203 + - -0.34472656 + - 0.87597656 + - 0.453125 + - -0.47583008 + - -0.42700195 + - -0.011474609 + - -0.27001953 + - 0.328125 + - 0.22619629 + - -0.3544922 + - -0.13110352 + - 0.98828125 + - -0.34960938 + - 0.36767578 + - -0.7636719 + - -0.40234375 + - -0.3671875 + - 0.5708008 + - -0.4765625 + - 0.48828125 + - 0.14001465 + - -1.2890625 + - 1.7734375 + - -0.47583008 + - 0.1817627 + - 0.2783203 + - 0.25830078 + - -0.37304688 + - 0.07861328 + - 0.106933594 + - 1.0556641 + - 0.12060547 + - -0.23876953 + - -0.21936035 + - 0.703125 + - -0.47680664 + - -0.60839844 + - 1.1640625 + - 0.6064453 + - 0.5253906 + - 0.16577148 + - 0.49023438 + - -1.0859375 + - 0.74902344 + - -0.84375 + - 0.7392578 + - -0.43481445 + - 0.5527344 + - -0.44458008 + - 0.28271484 + - -0.60595703 + - -0.6855469 + - -0.4152832 + - 0.11010742 + - 0.6191406 + - 1.1367188 + - 0.017700195 + - 0.06768799 + - -0.61816406 + - -0.083984375 + - 0.4243164 + - -0.81884766 + - 0.17797852 + - -1.0449219 + - 0.7578125 + - -0.48242188 + - -0.12487793 + - 0.22070313 + - 0.061035156 + - -0.35595703 + - -0.2154541 + - -0.5239258 + - 0.8383789 + - 0.5810547 + - -0.08477783 + - 0.2783203 + - -1.0117188 + - -0.7998047 + - -0.24377441 + - 1.109375 + - -0.07849121 + - 0.1348877 + - -0.54345703 + - 0.0045547485 + - -0.4140625 + - -0.58496094 + - -0.88134766 + - -0.35205078 + - 1.0429688 + - -0.7739258 + - 0.55126953 + - -0.2697754 + - -1.2001953 + - 0.5698242 + - 0.18811035 + - 0.28515625 + - -0.38671875 + - -0.17346191 + - -0.11779785 + - 0.035339355 + - 0.0413208 + - 0.0036201477 + - -0.5756836 + - 0.74072266 + - -0.6743164 + - 0.57958984 + - -0.37890625 + - -0.04901123 + - 0.515625 + - 0.82470703 + - 0.1328125 + - -1.0107422 + - 0.53271484 + - -0.12890625 + - 0.9111328 + - 0.8125 + - 0.39575195 + - -0.5473633 + - 0.3095703 + - 0.34228516 + - 0.01171875 + - -0.30126953 + - 0.23242188 + - 0.1862793 + - 0.23632813 + - 0.4074707 + - 0.61816406 + - 0.030227661 + - 0.9638672 + - -0.8798828 + - 0.049438477 + - -0.23706055 + - -0.49072266 + - 0.52441406 + - -0.21850586 + - -0.4584961 + - 0.23791504 + - 0.9609375 + - -0.9580078 + - 1.1142578 + - -1.3300781 + - 0.59228516 + - -0.5366211 + - -0.71972656 + - 0.4194336 + - -0.58984375 + - 0.46728516 + - -0.04034424 + - -0.10455322 + - 0.010231018 + - -0.08502197 + - 0.98291016 + - 0.025466919 + - 0.96484375 + - 0.7861328 + - -0.734375 + - -0.32543945 + - -0.4321289 + - -0.6513672 + - 0.7895508 + - 0.29248047 + - 0.25439453 + - 0.45263672 + - 0.042663574 + - -0.27124023 + - -0.7441406 + - -0.73095703 + - 0.13171387 + - 0.5185547 + - -0.98876953 + - -0.69384766 + - 0.86572266 + - -0.20080566 + - 0.3317871 + - -0.2746582 + - 0.26611328 + - 0.671875 + - -0.11669922 + - 0.5576172 + - -0.8486328 + - 0.024780273 + - 0.7060547 + - -0.25390625 + - 0.8515625 + - -0.36376953 + - 0.51416016 + - -0.10510254 + - 0.58447266 + - -0.82910156 + - -0.5517578 + - 0.5883789 + - -0.33544922 + - -0.07366943 + - -0.059326172 + - 0.019363403 + - 0.14868164 + - 0.65283203 + - -0.71191406 + - 1.3476563 + - -0.2512207 + - 0.32714844 + - -0.38989258 + - 0.21936035 + - -0.26000977 + - -0.671875 + - -0.31298828 + - -0.4399414 + - 0.30810547 + - -0.828125 + - -0.5390625 + - -0.3544922 + - 0.1842041 + - 0.29052734 + - 0.42089844 + - 0.21435547 + - 0.22424316 + - -0.17932129 + - 0.017166138 + - 0.092041016 + - -0.015655518 + - -0.31347656 + - 0.49414063 + - 0.68847656 + - -1.4863281 + - -0.4423828 + - 0.21386719 + - 0.40429688 + - -0.02192688 + - -0.50439453 + - 0.9946289 + - 0.068481445 + - 0.37304688 + - 0.04800415 + - 0.70751953 + - -0.5864258 + - 0.30981445 + - -0.47509766 + - 0.69921875 + - 0.022415161 + - 0.08721924 + - -0.26879883 + - -0.23828125 + - 0.49365234 + - 0.3996582 + - -0.6279297 + - -0.014892578 + - 0.05596924 + - 0.062408447 + - 0.32373047 + - -0.1550293 + - -0.9140625 + - 0.62109375 + - -0.29614258 + - 1.3125 + - 0.095947266 + - -0.33520508 + - -0.44580078 + - -0.32470703 + - 0.48779297 + - -0.3869629 + - -0.0075683594 + - 0.5566406 + - -0.16015625 + - 0.76953125 + - 0.18945313 + - -0.018463135 + - -0.07635498 + - 0.32617188 + - -0.37451172 + - -0.10986328 + - 0.27441406 + - 0.4296875 + - -0.58496094 + - 0.6254883 + - -0.1784668 + - 0.1616211 + - -0.37353516 + - 0.15625 + - 0.7836914 + - 0.22583008 + - 0.3330078 + - 0.61328125 + - -0.13427734 + - -0.48950195 + - -0.04296875 + - 0.18713379 + - 1.0449219 + - 0.29882813 + - 0.74316406 + - 0.7998047 + - 0.2163086 + - 0.30273438 + - -0.98828125 + - 0.12670898 + - -0.46923828 + - -0.5498047 + - 0.8203125 + - 0.2607422 + - -0.045196533 + - 0.41796875 + - -0.18652344 + - 0.7265625 + - 0.6220703 + - 0.30908203 + - 0.23413086 + - -0.28710938 + - 0.60253906 + - 0.54003906 + - -0.8129883 + - 0.49072266 + - 0.953125 + - -1.2167969 + - 0.030471802 + - -0.44970703 + - 0.81347656 + - 0.7001953 + - -0.91259766 + - -0.44970703 + - 0.34106445 + - 0.765625 + - -0.079711914 + - -0.23596191 + - -0.38671875 + - -0.061767578 + - 0.94921875 + - -0.18725586 + - -0.76660156 + - -0.9707031 + - -0.7602539 + - -0.047973633 + - 0.32080078 + - -0.7290039 + - -0.7817383 + - -0.76953125 + - 0.19628906 + - -0.3701172 + - -0.30395508 + - -0.21398926 + - 0.28051758 + - 0.43798828 + - 0.2548828 + - 1.2285156 + - 0.72265625 + - -0.036254883 + - 1.0625 + - -0.7314453 + - 0.21020508 + - -0.45507813 + - 0.40576172 + - 0.24414063 + - 0.18017578 + - -0.2409668 + - -0.012893677 + - -0.30371094 + - 0.16259766 + - -0.2626953 + - 0.11773682 + - -0.26879883 + - 0.44262695 + - -0.51464844 + - -0.14990234 + - 0.25805664 + - 0.043518066 + - -0.19311523 + - -0.3293457 + - -0.71875 + - 0.2076416 + - 0.115722656 + - -0.546875 + - 0.81689453 + - 0.35180664 + - -0.27368164 + - -0.103881836 + - 0.18713379 + - 1.1572266 + - 0.33520508 + - -0.20507813 + - 0.36523438 + - -0.7783203 + - -0.578125 + - -0.46679688 + - 0.002319336 + - -0.31323242 + - -0.099243164 + - -0.3857422 + - -0.93359375 + - 0.48535156 + - 0.04345703 + - 0.78222656 + - -0.6821289 + - -0.17773438 + - -0.43798828 + - 1.0546875 + - -0.08770752 + - 0.15625 + - 0.7636719 + - -0.55859375 + - 0.14160156 + - 0.91015625 + - -0.31591797 + - -0.14221191 + - -1.46875 + - 0.5571289 + - 0.06225586 + - -0.06188965 + - -0.7426758 + - 0.05731201 + - -0.9008789 + - -1.1503906 + - 0.31689453 + - 0.27856445 + - 0.5722656 + - -0.14526367 + - -0.35864258 + - -0.328125 + - 0.42114258 + - -0.14123535 + - -0.58203125 + - -0.05908203 + - 0.22167969 + - -0.4658203 + - 0.1706543 + - 0.3076172 + - -0.17504883 + - -0.72314453 + - -0.3935547 + - 0.7841797 + - 1.0078125 + - 0.44189453 + - -0.12976074 + - -0.12963867 + - 0.9423828 + - -0.035186768 + - -0.8930664 + - -0.6064453 + - 0.20715332 + - 1.0644531 + - -0.48266602 + - -1.1357422 + - 0.08721924 + - -0.08721924 + - -0.24658203 + - 0.0284729 + - -0.32470703 + - -0.296875 + - -0.87939453 + - 0.43481445 + - 0.3215332 + - -0.28759766 + - 0.73095703 + - 0.31274414 + - -0.25170898 + - -0.2668457 + - -0.5644531 + - 0.5620117 + - 0.23168945 + - -0.18518066 + - -0.20263672 + - 0.61621094 + - -0.021026611 + - 0.31835938 + - 0.19494629 + - -1.2089844 + - -0.8383789 + - 1.1962891 + - 0.89160156 + - 0.15649414 + - 0.9892578 + - -0.07446289 + - 0.09942627 + - 0.20507813 + - -0.45507813 + - -0.050201416 + - 0.4975586 + - 0.18676758 + - 0.37402344 + - -0.4345703 + - -1.0390625 + - -0.18554688 + - -0.8149414 + - 0.2890625 + - -0.6953125 + - -0.119628906 + - -0.04421997 + - 0.34228516 + - -0.68603516 + - -0.81103516 + - 0.17016602 + - 0.05960083 + - -0.5078125 + - 0.31591797 + - -0.07086182 + - -0.29541016 + - -0.3100586 + - 0.30664063 + - 0.8251953 + - -0.21386719 + - -0.016113281 + - -0.12475586 + - 0.8203125 + - 0.4309082 + - 0.11767578 + - 1.3037109 + - 0.06665039 + - 0.10119629 + - 0.5595703 + - -0.048858643 + - 0.07318115 + - 0.28955078 + - 0.21948242 + - -0.9404297 + - -0.37402344 + - 0.671875 + - -0.23535156 + - 0.1685791 + - 0.7734375 + - 0.62939453 + - -0.47485352 + - 0.25512695 + - -0.049316406 + - -0.035949707 + - -0.2775879 + - 0.8935547 + - -0.10900879 + - 0.7290039 + - 0.6040039 + - -0.77978516 + - 1.0917969 + - -0.3100586 + - 0.5058594 + - 0.21923828 + - -0.34301758 + - 0.96728516 + - 0.28564453 + - 0.0050849915 + - 0.32128906 + - 0.034942627 + - -0.13208008 + - -0.67578125 + - -0.27978516 + - -0.13452148 + - -0.28442383 + - 0.021972656 + - 0.50097656 + - -0.27172852 + - -0.103027344 + - -1.328125 + - -0.4194336 + - -0.1517334 + - 0.019317627 + - 0.1887207 + - 0.4501953 + - 0.40283203 + - -0.71191406 + - -0.19824219 + - -0.28515625 + - -0.5541992 + - -0.6879883 + - -0.33203125 + - 0.23095703 + - 0.0703125 + - 0.21398926 + - -0.5883789 + - 0.057617188 + - -0.4855957 + - 0.08532715 + - 0.96484375 + - -0.62402344 + - 0.68603516 + - 0.4399414 + - 0.1508789 + - -0.021881104 +- - -0.7705078 + - 0.06677246 + - -0.7470703 + - -0.13134766 + - 0.04748535 + - -0.71972656 + - 0.03213501 + - 0.069885254 + - -0.5288086 + - 1.6503906 + - -0.06713867 + - 0.18188477 + - -0.31958008 + - 1.5625 + - -0.2854004 + - 0.17663574 + - -0.107910156 + - -0.19873047 + - 0.53125 + - -0.09716797 + - -0.25170898 + - 0.20239258 + - -0.5 + - -0.5239258 + - -0.8730469 + - -0.26391602 + - -0.015304565 + - -0.34887695 + - 1.1074219 + - -0.11254883 + - -0.15429688 + - 0.24023438 + - -0.24291992 + - 0.48583984 + - -0.30273438 + - 0.50634766 + - -0.05923462 + - -0.07110596 + - 0.8364258 + - 0.056274414 + - -0.11968994 + - -0.037322998 + - 0.36865234 + - 0.6660156 + - -0.06542969 + - 0.5522461 + - 0.113464355 + - -0.085632324 + - 0.2763672 + - 0.09802246 + - -0.25317383 + - 0.41870117 + - 0.8491211 + - 0.044281006 + - 0.68847656 + - -0.2064209 + - -0.51123047 + - -0.4501953 + - 1.1611328 + - -0.65478516 + - -0.21289063 + - 0.44213867 + - 0.52685547 + - 0.23937988 + - 0.37109375 + - -1.0966797 + - 0.01638794 + - -0.57373047 + - -1.140625 + - 0.23571777 + - 0.014472961 + - -0.28515625 + - -0.06097412 + - -0.048461914 + - -0.72802734 + - 0.34228516 + - 0.34545898 + - -0.008087158 + - 0.6113281 + - -0.0914917 + - 0.076660156 + - -0.578125 + - 0.16308594 + - -0.33276367 + - -0.11755371 + - -0.8203125 + - -0.2590332 + - 0.26538086 + - -0.84472656 + - -0.5878906 + - 0.30517578 + - -0.20214844 + - 0.29736328 + - 0.58447266 + - 0.030792236 + - 0.0770874 + - 0.17492676 + - -0.47802734 + - 0.23095703 + - 0.5683594 + - -0.079833984 + - -0.30786133 + - -1.2099609 + - -0.5004883 + - -0.37963867 + - -0.06011963 + - -1.0009766 + - -0.39257813 + - -1.1767578 + - -0.20251465 + - -0.43408203 + - 0.30737305 + - -0.17541504 + - -0.8847656 + - 0.5498047 + - 0.23352051 + - 0.5107422 + - -0.13867188 + - 0.5058594 + - 0.734375 + - 0.42407227 + - -0.19702148 + - -0.20776367 + - 0.9238281 + - -0.34765625 + - -0.69970703 + - -0.3803711 + - 0.7866211 + - -0.36328125 + - -0.01499176 + - -0.21582031 + - 0.23181152 + - 0.21704102 + - 0.2824707 + - -0.5834961 + - -0.7910156 + - 0.21789551 + - 0.3137207 + - -0.7006836 + - -0.68603516 + - 0.5288086 + - 0.71875 + - 0.05545044 + - -0.15856934 + - -0.2163086 + - -0.060394287 + - 0.14733887 + - 0.30639648 + - 0.15148926 + - 0.19628906 + - -0.31640625 + - -0.15429688 + - -1.2783203 + - 0.15258789 + - -1.0078125 + - -0.0016040802 + - -0.037872314 + - -0.109069824 + - 0.84814453 + - -0.47875977 + - 1.1435547 + - 0.6303711 + - 0.13696289 + - 0.37426758 + - -0.19885254 + - 0.64453125 + - -0.19311523 + - 0.46899414 + - -0.16430664 + - -0.5205078 + - 0.13134766 + - -0.4423828 + - 0.4675293 + - 0.2763672 + - -0.40307617 + - -0.7910156 + - -0.08544922 + - -1.5429688 + - -0.5102539 + - 0.2142334 + - -0.71533203 + - 0.5605469 + - -0.6171875 + - 0.46557617 + - 0.9326172 + - 0.1430664 + - -0.7163086 + - 0.11743164 + - -0.24536133 + - 0.6455078 + - -0.32250977 + - 0.9404297 + - -0.36645508 + - 1.1279297 + - -0.85302734 + - 1.2333984 + - -0.45947266 + - 0.5175781 + - 0.5292969 + - -0.21606445 + - -0.5488281 + - -0.3395996 + - -0.17675781 + - -0.1661377 + - 0.71191406 + - -0.4128418 + - 0.084106445 + - 0.71972656 + - -0.2692871 + - 0.24963379 + - -0.72753906 + - -0.47192383 + - -0.92089844 + - 0.59375 + - -0.18041992 + - 0.2734375 + - 0.5541992 + - -1.2207031 + - 1.7294922 + - 0.055541992 + - 0.09539795 + - 0.5205078 + - 0.44604492 + - -1.1337891 + - 0.18603516 + - 0.38330078 + - 0.7949219 + - -0.5307617 + - -0.5830078 + - 0.21618652 + - 0.45629883 + - -0.42089844 + - -0.068359375 + - 0.9116211 + - 0.46191406 + - 0.3251953 + - 0.6845703 + - 0.15905762 + - -0.66748047 + - 0.5839844 + - -1.0595703 + - 0.4362793 + - -0.60009766 + - -0.30517578 + - 0.0076789856 + - -0.32421875 + - -0.052886963 + - -1.5175781 + - -0.5600586 + - 0.8183594 + - 0.20031738 + - 1.0556641 + - -0.3671875 + - 0.121398926 + - 0.19885254 + - 0.39941406 + - 0.42504883 + - -0.7895508 + - -0.23852539 + - -0.92871094 + - 0.65234375 + - -0.47583008 + - -0.48828125 + - 0.13574219 + - -0.015090942 + - -0.27270508 + - -0.38500977 + - -0.9160156 + - 1.4101563 + - 0.06616211 + - -0.54345703 + - 0.1907959 + - -0.91259766 + - -0.06915283 + - 0.2915039 + - 0.87353516 + - -0.66552734 + - 0.33007813 + - -0.70751953 + - -0.77001953 + - -0.6503906 + - -0.3071289 + - -0.93066406 + - -0.32592773 + - 0.72509766 + - -0.75 + - 0.08282471 + - 0.58251953 + - -1.2255859 + - 0.056152344 + - 0.46606445 + - 0.77001953 + - -0.5371094 + - -0.4519043 + - 0.10131836 + - -0.078063965 + - 0.009971619 + - -0.1763916 + - 0.004043579 + - -0.14074707 + - -0.025894165 + - 0.40039063 + - -0.30004883 + - 0.2290039 + - 0.1373291 + - 0.42871094 + - 0.39746094 + - -1.015625 + - 0.1451416 + - -0.24230957 + - 0.84375 + - 0.69628906 + - 0.048736572 + - -0.15625 + - 0.43579102 + - 0.6225586 + - 0.016616821 + - -0.78222656 + - -0.009765625 + - 0.7631836 + - -0.38110352 + - -0.19897461 + - 0.80908203 + - -0.011230469 + - 0.6586914 + - -0.34228516 + - -0.0045661926 + - -0.3725586 + - -0.5605469 + - 0.6269531 + - -0.06109619 + - -0.36767578 + - 0.0859375 + - 0.625 + - -0.65722656 + - 0.29345703 + - -1.1689453 + - 0.38745117 + - -0.1508789 + - -0.46777344 + - 0.8232422 + - 0.22314453 + - 0.39941406 + - 0.47509766 + - 0.32006836 + - -0.12158203 + - -0.36547852 + - 0.91503906 + - 0.16149902 + - 0.7216797 + - 0.006275177 + - -0.7446289 + - -0.484375 + - 0.19213867 + - 0.047058105 + - 1.8378906 + - 0.62841797 + - -0.101379395 + - 0.33764648 + - -0.009132385 + - 0.12548828 + - -1.3408203 + - -0.3527832 + - -0.23022461 + - -0.17346191 + - -1.0097656 + - -0.78564453 + - 0.85009766 + - -0.30737305 + - 0.5463867 + - -0.10083008 + - 0.48583984 + - 0.2019043 + - -0.40429688 + - 0.26586914 + - -0.5683594 + - 0.18823242 + - 0.27197266 + - -0.58154297 + - 1.0634766 + - -0.36035156 + - 1.0019531 + - 0.054229736 + - 0.578125 + - -0.9272461 + - -1.0791016 + - 0.6191406 + - -0.27246094 + - 0.1105957 + - 0.0006971359 + - -0.10699463 + - 0.1977539 + - 0.072509766 + - -0.19433594 + - 0.6928711 + - 0.6928711 + - 0.57666016 + - -1.1015625 + - 0.42456055 + - -0.5854492 + - -0.51171875 + - 0.18579102 + - -0.17785645 + - 0.3503418 + - -0.44091797 + - -0.20922852 + - -0.14465332 + - 0.08996582 + - 0.17663574 + - 0.5605469 + - 0.24658203 + - 0.4189453 + - -0.09814453 + - 0.7207031 + - 0.2902832 + - -0.25732422 + - 0.46875 + - -0.60498047 + - 0.31958008 + - -1.0341797 + - -0.72265625 + - 0.036590576 + - -0.13977051 + - 0.057373047 + - -0.4855957 + - 0.8413086 + - 0.29882813 + - -0.123413086 + - -0.07745361 + - 0.9003906 + - 0.27026367 + - 0.34033203 + - -0.74121094 + - 0.98046875 + - -0.16296387 + - -0.20385742 + - -0.022628784 + - -0.24487305 + - 0.4453125 + - 0.62353516 + - -0.43286133 + - -0.10321045 + - 0.3503418 + - -0.008575439 + - 0.69140625 + - 0.13427734 + - -0.52734375 + - 1.0117188 + - 0.091796875 + - 0.64208984 + - -0.068115234 + - -0.3095703 + - -0.55029297 + - 0.0970459 + - 0.18981934 + - -0.40161133 + - -0.07232666 + - 0.67822266 + - -0.20385742 + - 0.1977539 + - 0.9086914 + - 0.02482605 + - -0.28930664 + - 0.1348877 + - -0.4050293 + - 0.125 + - -0.050231934 + - 0.16809082 + - -0.6694336 + - 0.41210938 + - 0.04269409 + - 0.29541016 + - -0.57373047 + - -0.25854492 + - 1.1113281 + - 0.107788086 + - 0.36401367 + - -0.5600586 + - -0.036956787 + - 0.053466797 + - -0.12084961 + - -0.30297852 + - 0.5756836 + - 0.66259766 + - 0.42895508 + - 0.50390625 + - 0.21557617 + - 0.46801758 + - -0.49560547 + - 0.43847656 + - -0.4819336 + - -0.0036258698 + - 0.8105469 + - 0.2565918 + - -0.19348145 + - 0.9067383 + - -0.024169922 + - 0.31176758 + - 0.5683594 + - 0.24353027 + - 0.08831787 + - -0.07647705 + - 0.05206299 + - 0.484375 + - -0.5288086 + - 0.2565918 + - 0.23852539 + - -0.34765625 + - 0.1385498 + - -0.16430664 + - 0.5786133 + - 0.17822266 + - -1.0146484 + - -0.6660156 + - 0.60253906 + - 0.48339844 + - -0.71777344 + - -0.56103516 + - -0.043670654 + - -0.027313232 + - 0.5761719 + - -0.47021484 + - -0.43554688 + - -0.72753906 + - -0.7207031 + - -0.29296875 + - 0.0011854172 + - -0.50390625 + - -0.107543945 + - -1.0048828 + - -0.13342285 + - -0.70410156 + - -0.3239746 + - -0.7685547 + - -0.33203125 + - -0.014854431 + - -0.49438477 + - 1.0507813 + - 0.51953125 + - 0.25927734 + - 0.99853516 + - -0.35986328 + - 0.27612305 + - -0.45581055 + - -0.24609375 + - -0.015602112 + - -0.75097656 + - 0.072753906 + - 0.17822266 + - -0.5175781 + - 0.02204895 + - -0.022598267 + - 0.15820313 + - 0.61279297 + - 0.09082031 + - -0.75927734 + - -0.69384766 + - 0.7294922 + - 0.115722656 + - 0.107421875 + - -0.50390625 + - -0.74609375 + - -0.08081055 + - -0.13293457 + - -0.80322266 + - -0.043121338 + - 0.2692871 + - -0.13793945 + - -0.5830078 + - 0.105529785 + - 0.65185547 + - 0.13977051 + - 0.22387695 + - -0.20947266 + - -0.93066406 + - -1.0966797 + - 0.40649414 + - -0.052246094 + - -0.097839355 + - -0.055419922 + - -0.12011719 + - -0.3696289 + - 0.38671875 + - -0.10040283 + - 0.609375 + - -0.69677734 + - 0.4638672 + - -0.7080078 + - 1.3359375 + - 0.15576172 + - 0.04937744 + - 1.1611328 + - -0.7758789 + - -0.032470703 + - 0.2878418 + - 0.5180664 + - -0.21606445 + - -1.484375 + - 0.18261719 + - -0.053894043 + - -0.35864258 + - -0.4741211 + - 0.20581055 + - -0.05279541 + - -0.23022461 + - 0.9794922 + - 0.41918945 + - 0.31030273 + - 0.5493164 + - -0.038848877 + - -0.6430664 + - 0.41137695 + - -0.09442139 + - -0.57910156 + - -0.05834961 + - -0.35253906 + - -0.21386719 + - -0.21398926 + - 0.44360352 + - -0.21704102 + - -1.0146484 + - 0.14355469 + - 0.57910156 + - 0.609375 + - 0.63427734 + - -0.5205078 + - -0.27783203 + - 0.9116211 + - -0.18701172 + - -0.5258789 + - -0.9741211 + - -0.18432617 + - 0.7553711 + - -0.38305664 + - -1.171875 + - 0.58251953 + - 0.40942383 + - -0.19213867 + - 0.37646484 + - -0.40478516 + - -0.2763672 + - -0.84033203 + - -0.06427002 + - 1.4375 + - 0.13598633 + - 0.020401001 + - 0.1796875 + - -0.7763672 + - -0.49438477 + - -0.022628784 + - 0.39697266 + - -0.013198853 + - -0.6616211 + - 0.0025100708 + - 0.42236328 + - -0.36279297 + - -0.140625 + - 0.2998047 + - -0.67626953 + - -1.2607422 + - 0.5913086 + - 0.45776367 + - 0.13586426 + - 0.5913086 + - -0.07293701 + - -0.28344727 + - -0.060455322 + - -0.7109375 + - 0.122680664 + - 1.0273438 + - -0.5600586 + - 0.6020508 + - -0.48364258 + - -0.4404297 + - 0.26708984 + - 0.27246094 + - 0.44018555 + - -0.49829102 + - 0.070251465 + - -0.85595703 + - -0.092163086 + - -0.031707764 + - -0.99902344 + - 0.4765625 + - -0.07537842 + - -0.17993164 + - 0.6455078 + - 0.31518555 + - -0.14685059 + - -0.60498047 + - 0.6245117 + - 0.7734375 + - -0.58203125 + - 0.16564941 + - -0.39404297 + - 0.4921875 + - 0.60498047 + - -0.29785156 + - 1.5048828 + - 0.24853516 + - 0.75878906 + - 0.049682617 + - -0.2277832 + - -1.0244141 + - -0.18103027 + - 0.0119018555 + - -0.7949219 + - -0.20959473 + - 0.8955078 + - 0.04159546 + - -0.26391602 + - 0.24291992 + - 1.0224609 + - -0.7890625 + - 0.70654297 + - -0.7973633 + - 0.21679688 + - -0.33251953 + - 0.828125 + - -0.042938232 + - 0.6772461 + - 0.0259552 + - -0.070007324 + - 0.33789063 + - -0.3642578 + - 0.71728516 + - 0.2697754 + - -0.24584961 + - 1.03125 + - 0.000767231 + - 0.85009766 + - 0.67822266 + - 0.20043945 + - -0.088378906 + - -0.7597656 + - 0.16503906 + - -0.06088257 + - -0.43408203 + - 0.2770996 + - 0.3503418 + - 0.0014648438 + - 0.7709961 + - -0.9296875 + - -0.28027344 + - -0.29223633 + - -0.22912598 + - 0.5991211 + - 0.6899414 + - 0.30395508 + - 0.058502197 + - -0.7553711 + - -0.39990234 + - -0.02243042 + - -0.87597656 + - -0.9248047 + - 1.2861328 + - -0.14318848 + - 0.62939453 + - -0.25390625 + - -0.27905273 + - -0.5600586 + - 0.030822754 + - 0.43847656 + - -0.7451172 + - 0.8696289 + - 0.2697754 + - 0.3244629 + - -0.4272461 diff --git a/backends/candle/tests/snapshots/test_flash_nomic__nomic_moe_single.snap b/backends/candle/tests/snapshots/test_flash_nomic__nomic_moe_single.snap new file mode 100644 index 00000000..2c0b518d --- /dev/null +++ b/backends/candle/tests/snapshots/test_flash_nomic__nomic_moe_single.snap @@ -0,0 +1,772 @@ +--- +source: backends/candle/tests/test_flash_nomic.rs +expression: embeddings_single +--- +- - -0.77001953 + - 0.06573486 + - -0.7475586 + - -0.13171387 + - 0.04763794 + - -0.7192383 + - 0.03164673 + - 0.068481445 + - -0.5292969 + - 1.6494141 + - -0.06665039 + - 0.18225098 + - -0.3190918 + - 1.5625 + - -0.28515625 + - 0.1763916 + - -0.10852051 + - -0.19958496 + - 0.5307617 + - -0.09790039 + - -0.25097656 + - 0.20275879 + - -0.5004883 + - -0.5239258 + - -0.8725586 + - -0.26367188 + - -0.014892578 + - -0.34936523 + - 1.1064453 + - -0.11279297 + - -0.1538086 + - 0.23986816 + - -0.24353027 + - 0.48583984 + - -0.3017578 + - 0.5058594 + - -0.058654785 + - -0.07183838 + - 0.8364258 + - 0.05621338 + - -0.119384766 + - -0.037231445 + - 0.36938477 + - 0.6660156 + - -0.064453125 + - 0.5522461 + - 0.11340332 + - -0.086242676 + - 0.2763672 + - 0.097717285 + - -0.2529297 + - 0.4182129 + - 0.8491211 + - 0.044158936 + - 0.6879883 + - -0.2076416 + - -0.5097656 + - -0.4501953 + - 1.1601563 + - -0.6538086 + - -0.21166992 + - 0.4423828 + - 0.52783203 + - 0.23937988 + - 0.37036133 + - -1.0966797 + - 0.016113281 + - -0.57373047 + - -1.140625 + - 0.23547363 + - 0.014053345 + - -0.2854004 + - -0.0602417 + - -0.04815674 + - -0.72802734 + - 0.34204102 + - 0.3449707 + - -0.0076026917 + - 0.61083984 + - -0.0914917 + - 0.076293945 + - -0.578125 + - 0.1640625 + - -0.3322754 + - -0.11785889 + - -0.81884766 + - -0.25878906 + - 0.26538086 + - -0.8457031 + - -0.58691406 + - 0.30517578 + - -0.20178223 + - 0.296875 + - 0.58447266 + - 0.031402588 + - 0.077941895 + - 0.17468262 + - -0.47680664 + - 0.23144531 + - 0.5678711 + - -0.079711914 + - -0.3088379 + - -1.2099609 + - -0.5004883 + - -0.37939453 + - -0.060180664 + - -1.0009766 + - -0.39208984 + - -1.1757813 + - -0.20178223 + - -0.4338379 + - 0.3076172 + - -0.17578125 + - -0.8847656 + - 0.55078125 + - 0.23364258 + - 0.5107422 + - -0.13891602 + - 0.5058594 + - 0.734375 + - 0.42456055 + - -0.19763184 + - -0.20751953 + - 0.92285156 + - -0.34692383 + - -0.7001953 + - -0.38012695 + - 0.7871094 + - -0.36376953 + - -0.014328003 + - -0.21606445 + - 0.23181152 + - 0.2175293 + - 0.28295898 + - -0.5839844 + - -0.7910156 + - 0.21789551 + - 0.31347656 + - -0.70166016 + - -0.68603516 + - 0.5283203 + - 0.7192383 + - 0.05596924 + - -0.15856934 + - -0.2163086 + - -0.060546875 + - 0.14648438 + - 0.30737305 + - 0.15075684 + - 0.19641113 + - -0.3166504 + - -0.15454102 + - -1.2783203 + - 0.15246582 + - -1.0078125 + - -0.00090646744 + - -0.038391113 + - -0.109436035 + - 0.84765625 + - -0.4790039 + - 1.1425781 + - 0.6298828 + - 0.13696289 + - 0.37426758 + - -0.19836426 + - 0.64453125 + - -0.19189453 + - 0.46801758 + - -0.16455078 + - -0.5209961 + - 0.13146973 + - -0.4423828 + - 0.46704102 + - 0.27685547 + - -0.40283203 + - -0.7919922 + - -0.08569336 + - -1.5439453 + - -0.5097656 + - 0.21459961 + - -0.71533203 + - 0.56103516 + - -0.6171875 + - 0.46533203 + - 0.93359375 + - 0.14282227 + - -0.71533203 + - 0.116882324 + - -0.24487305 + - 0.64501953 + - -0.3227539 + - 0.9404297 + - -0.3659668 + - 1.1279297 + - -0.85302734 + - 1.234375 + - -0.45996094 + - 0.5180664 + - 0.5283203 + - -0.21618652 + - -0.54833984 + - -0.33984375 + - -0.17773438 + - -0.16564941 + - 0.71191406 + - -0.4140625 + - 0.084228516 + - 0.71972656 + - -0.26953125 + - 0.24963379 + - -0.7285156 + - -0.47265625 + - -0.92041016 + - 0.59375 + - -0.17993164 + - 0.27319336 + - 0.5541992 + - -1.2207031 + - 1.7294922 + - 0.055725098 + - 0.09442139 + - 0.5205078 + - 0.44555664 + - -1.1337891 + - 0.18518066 + - 0.38354492 + - 0.7949219 + - -0.5307617 + - -0.58251953 + - 0.21557617 + - 0.45629883 + - -0.42211914 + - -0.06829834 + - 0.9116211 + - 0.46166992 + - 0.32470703 + - 0.68408203 + - 0.15856934 + - -0.66748047 + - 0.5839844 + - -1.0595703 + - 0.43579102 + - -0.60009766 + - -0.3046875 + - 0.0076904297 + - -0.32348633 + - -0.053741455 + - -1.5175781 + - -0.5600586 + - 0.81884766 + - 0.20043945 + - 1.0556641 + - -0.36645508 + - 0.12194824 + - 0.19921875 + - 0.39990234 + - 0.42529297 + - -0.79003906 + - -0.23901367 + - -0.9301758 + - 0.65234375 + - -0.47583008 + - -0.48828125 + - 0.13537598 + - -0.014663696 + - -0.27172852 + - -0.38476563 + - -0.9160156 + - 1.4101563 + - 0.06665039 + - -0.54296875 + - 0.19104004 + - -0.91259766 + - -0.06878662 + - 0.29125977 + - 0.87353516 + - -0.66552734 + - 0.32910156 + - -0.70703125 + - -0.77001953 + - -0.6508789 + - -0.30688477 + - -0.93066406 + - -0.32617188 + - 0.72509766 + - -0.75097656 + - 0.08300781 + - 0.5834961 + - -1.2255859 + - 0.057250977 + - 0.46557617 + - 0.7685547 + - -0.5371094 + - -0.45214844 + - 0.10168457 + - -0.07824707 + - 0.009796143 + - -0.17541504 + - 0.003835678 + - -0.14050293 + - -0.025787354 + - 0.40039063 + - -0.30004883 + - 0.22949219 + - 0.13684082 + - 0.42871094 + - 0.3972168 + - -1.015625 + - 0.14538574 + - -0.2421875 + - 0.84521484 + - 0.69628906 + - 0.04864502 + - -0.15637207 + - 0.43603516 + - 0.6225586 + - 0.017120361 + - -0.78222656 + - -0.010284424 + - 0.7631836 + - -0.3815918 + - -0.19836426 + - 0.80908203 + - -0.010810852 + - 0.65966797 + - -0.34301758 + - -0.0039405823 + - -0.37329102 + - -0.5605469 + - 0.6269531 + - -0.06121826 + - -0.36767578 + - 0.08642578 + - 0.6254883 + - -0.65771484 + - 0.29296875 + - -1.1689453 + - 0.3876953 + - -0.15148926 + - -0.46801758 + - 0.8232422 + - 0.22302246 + - 0.39990234 + - 0.4753418 + - 0.31958008 + - -0.121398926 + - -0.36547852 + - 0.9160156 + - 0.16149902 + - 0.72314453 + - 0.0065193176 + - -0.7451172 + - -0.48364258 + - 0.19213867 + - 0.046539307 + - 1.8369141 + - 0.6269531 + - -0.10192871 + - 0.3371582 + - -0.008926392 + - 0.12536621 + - -1.3408203 + - -0.35253906 + - -0.23010254 + - -0.17382813 + - -1.0097656 + - -0.7866211 + - 0.85058594 + - -0.30737305 + - 0.5463867 + - -0.099609375 + - 0.48657227 + - 0.20166016 + - -0.4038086 + - 0.26635742 + - -0.5683594 + - 0.18847656 + - 0.27197266 + - -0.5800781 + - 1.0634766 + - -0.36035156 + - 1.0019531 + - 0.05496216 + - 0.578125 + - -0.9267578 + - -1.078125 + - 0.62060547 + - -0.2722168 + - 0.11114502 + - 0.0005927086 + - -0.10809326 + - 0.19799805 + - 0.072265625 + - -0.19494629 + - 0.6933594 + - 0.6928711 + - 0.5776367 + - -1.1015625 + - 0.42504883 + - -0.5859375 + - -0.51220703 + - 0.18603516 + - -0.17797852 + - 0.34960938 + - -0.44140625 + - -0.20898438 + - -0.1439209 + - 0.09008789 + - 0.17565918 + - 0.5605469 + - 0.24682617 + - 0.4194336 + - -0.097961426 + - 0.72021484 + - 0.28979492 + - -0.25708008 + - 0.46875 + - -0.6040039 + - 0.31933594 + - -1.0341797 + - -0.7207031 + - 0.03781128 + - -0.13928223 + - 0.05718994 + - -0.48535156 + - 0.8388672 + - 0.29858398 + - -0.1237793 + - -0.07904053 + - 0.89990234 + - 0.27001953 + - 0.3400879 + - -0.74121094 + - 0.98095703 + - -0.16320801 + - -0.20361328 + - -0.023254395 + - -0.24523926 + - 0.44458008 + - 0.6245117 + - -0.43310547 + - -0.1027832 + - 0.34985352 + - -0.008613586 + - 0.69140625 + - 0.13415527 + - -0.52734375 + - 1.0107422 + - 0.09112549 + - 0.6425781 + - -0.06854248 + - -0.30981445 + - -0.5493164 + - 0.09698486 + - 0.19006348 + - -0.40161133 + - -0.07287598 + - 0.6777344 + - -0.20373535 + - 0.19750977 + - 0.9082031 + - 0.024963379 + - -0.2890625 + - 0.13574219 + - -0.40551758 + - 0.124572754 + - -0.049987793 + - 0.1673584 + - -0.6689453 + - 0.41137695 + - 0.042755127 + - 0.29516602 + - -0.5722656 + - -0.2590332 + - 1.1113281 + - 0.107788086 + - 0.36376953 + - -0.5600586 + - -0.036468506 + - 0.05316162 + - -0.11987305 + - -0.30273438 + - 0.5756836 + - 0.66259766 + - 0.42871094 + - 0.50390625 + - 0.21643066 + - 0.4663086 + - -0.4963379 + - 0.43774414 + - -0.4819336 + - -0.0029296875 + - 0.81103516 + - 0.25683594 + - -0.19494629 + - 0.90625 + - -0.024551392 + - 0.31274414 + - 0.5683594 + - 0.24377441 + - 0.08886719 + - -0.0769043 + - 0.05206299 + - 0.48486328 + - -0.52783203 + - 0.25585938 + - 0.23852539 + - -0.34765625 + - 0.13769531 + - -0.16503906 + - 0.57910156 + - 0.17822266 + - -1.0146484 + - -0.66503906 + - 0.60253906 + - 0.48388672 + - -0.71875 + - -0.5605469 + - -0.04348755 + - -0.027282715 + - 0.5751953 + - -0.4699707 + - -0.43652344 + - -0.72753906 + - -0.7211914 + - -0.2932129 + - 0.0013942719 + - -0.50390625 + - -0.10675049 + - -1.0039063 + - -0.13305664 + - -0.70410156 + - -0.32299805 + - -0.7675781 + - -0.33081055 + - -0.014381409 + - -0.49365234 + - 1.0507813 + - 0.52001953 + - 0.25878906 + - 0.99902344 + - -0.35986328 + - 0.27612305 + - -0.45581055 + - -0.24682617 + - -0.015274048 + - -0.75097656 + - 0.072265625 + - 0.17822266 + - -0.5175781 + - 0.022613525 + - -0.022628784 + - 0.15856934 + - 0.61279297 + - 0.089538574 + - -0.75927734 + - -0.6933594 + - 0.7285156 + - 0.11541748 + - 0.10614014 + - -0.50439453 + - -0.7451172 + - -0.08050537 + - -0.13342285 + - -0.8027344 + - -0.043762207 + - 0.2685547 + - -0.1373291 + - -0.5830078 + - 0.1048584 + - 0.65185547 + - 0.13916016 + - 0.22399902 + - -0.20935059 + - -0.9296875 + - -1.0957031 + - 0.40649414 + - -0.052246094 + - -0.09814453 + - -0.05517578 + - -0.12017822 + - -0.3696289 + - 0.38720703 + - -0.10040283 + - 0.6088867 + - -0.69628906 + - 0.46411133 + - -0.7080078 + - 1.3349609 + - 0.15515137 + - 0.0491333 + - 1.1611328 + - -0.7758789 + - -0.031341553 + - 0.28833008 + - 0.5180664 + - -0.2163086 + - -1.484375 + - 0.1821289 + - -0.05307007 + - -0.35864258 + - -0.47460938 + - 0.20483398 + - -0.052337646 + - -0.23010254 + - 0.98046875 + - 0.41992188 + - 0.31030273 + - 0.5488281 + - -0.0390625 + - -0.64404297 + - 0.41210938 + - -0.09527588 + - -0.5786133 + - -0.058258057 + - -0.35205078 + - -0.21484375 + - -0.21447754 + - 0.44360352 + - -0.21618652 + - -1.0146484 + - 0.14355469 + - 0.578125 + - 0.609375 + - 0.63427734 + - -0.5205078 + - -0.2775879 + - 0.9116211 + - -0.1875 + - -0.5258789 + - -0.97509766 + - -0.18518066 + - 0.7558594 + - -0.3828125 + - -1.1728516 + - 0.5830078 + - 0.40893555 + - -0.19165039 + - 0.37548828 + - -0.40356445 + - -0.27563477 + - -0.84033203 + - -0.0647583 + - 1.4394531 + - 0.13647461 + - 0.020248413 + - 0.18005371 + - -0.7758789 + - -0.49365234 + - -0.022491455 + - 0.39672852 + - -0.012168884 + - -0.6616211 + - 0.0029296875 + - 0.42260742 + - -0.3635254 + - -0.14038086 + - 0.3005371 + - -0.67626953 + - -1.2587891 + - 0.5913086 + - 0.4580078 + - 0.13598633 + - 0.5913086 + - -0.0725708 + - -0.28344727 + - -0.06060791 + - -0.7104492 + - 0.1227417 + - 1.0283203 + - -0.55908203 + - 0.6015625 + - -0.48486328 + - -0.44091797 + - 0.26708984 + - 0.27246094 + - 0.44067383 + - -0.49829102 + - 0.070373535 + - -0.8564453 + - -0.092163086 + - -0.031188965 + - -0.99853516 + - 0.4765625 + - -0.07489014 + - -0.18017578 + - 0.64501953 + - 0.3149414 + - -0.14746094 + - -0.6040039 + - 0.6245117 + - 0.7734375 + - -0.58154297 + - 0.1661377 + - -0.39453125 + - 0.49267578 + - 0.60498047 + - -0.29858398 + - 1.5039063 + - 0.24829102 + - 0.7597656 + - 0.04916382 + - -0.22729492 + - -1.0244141 + - -0.18127441 + - 0.012062073 + - -0.79541016 + - -0.20910645 + - 0.8955078 + - 0.0413208 + - -0.26416016 + - 0.24291992 + - 1.0224609 + - -0.7890625 + - 0.7050781 + - -0.7973633 + - 0.21679688 + - -0.33203125 + - 0.828125 + - -0.042816162 + - 0.6772461 + - 0.026565552 + - -0.06921387 + - 0.33764648 + - -0.36376953 + - 0.71728516 + - 0.27026367 + - -0.2467041 + - 1.03125 + - 0.0013074875 + - 0.8496094 + - 0.67822266 + - 0.2006836 + - -0.08807373 + - -0.7597656 + - 0.16491699 + - -0.060791016 + - -0.43359375 + - 0.27661133 + - 0.34936523 + - 0.0023708344 + - 0.7709961 + - -0.92871094 + - -0.28125 + - -0.29223633 + - -0.22961426 + - 0.5986328 + - 0.6894531 + - 0.3046875 + - 0.058380127 + - -0.7553711 + - -0.39990234 + - -0.02229309 + - -0.87597656 + - -0.9248047 + - 1.2880859 + - -0.14294434 + - 0.62939453 + - -0.25463867 + - -0.27856445 + - -0.5605469 + - 0.030273438 + - 0.43847656 + - -0.7446289 + - 0.8701172 + - 0.2692871 + - 0.32421875 + - -0.42700195 diff --git a/backends/candle/tests/snapshots/test_nomic__nomic_moe_batch.snap b/backends/candle/tests/snapshots/test_nomic__nomic_moe_batch.snap new file mode 100644 index 00000000..3e9423a0 --- /dev/null +++ b/backends/candle/tests/snapshots/test_nomic__nomic_moe_batch.snap @@ -0,0 +1,2308 @@ +--- +source: backends/candle/tests/test_nomic.rs +expression: embeddings_batch +--- +- - -0.77039415 + - 0.06603869 + - -0.7481627 + - -0.13090512 + - 0.047572974 + - -0.7196526 + - 0.03157639 + - 0.06883626 + - -0.5306753 + - 1.6497533 + - -0.06626414 + - 0.18275733 + - -0.31899172 + - 1.5625474 + - -0.28502676 + - 0.17592163 + - -0.10744914 + - -0.19912067 + - 0.5304716 + - -0.09754831 + - -0.25156677 + - 0.20254421 + - -0.50044435 + - -0.5243958 + - -0.8732222 + - -0.26390445 + - -0.014966733 + - -0.34825927 + - 1.1057283 + - -0.11190004 + - -0.1537848 + - 0.24017277 + - -0.2439278 + - 0.48591775 + - -0.30278283 + - 0.5074822 + - -0.058676906 + - -0.07136222 + - 0.8357766 + - 0.057142794 + - -0.12005273 + - -0.037646633 + - 0.3695325 + - 0.6662965 + - -0.06519522 + - 0.55348885 + - 0.11334873 + - -0.08590865 + - 0.27785024 + - 0.09716725 + - -0.253027 + - 0.4178844 + - 0.8498044 + - 0.044411708 + - 0.6873774 + - -0.20685801 + - -0.51111424 + - -0.45068893 + - 1.1606588 + - -0.65466064 + - -0.21233405 + - 0.44229627 + - 0.5280446 + - 0.23785159 + - 0.3712364 + - -1.0964439 + - 0.016722241 + - -0.5736524 + - -1.1417898 + - 0.23557658 + - 0.014350355 + - -0.28558 + - -0.06097868 + - -0.048328485 + - -0.72823924 + - 0.34208012 + - 0.34606266 + - -0.0084145265 + - 0.61085355 + - -0.091765486 + - 0.07675646 + - -0.5773393 + - 0.1638734 + - -0.33433697 + - -0.117930256 + - -0.8185136 + - -0.25872475 + - 0.2649337 + - -0.8452235 + - -0.5882134 + - 0.30627438 + - -0.20208707 + - 0.29648748 + - 0.5842015 + - 0.031331632 + - 0.07708081 + - 0.1742084 + - -0.47708175 + - 0.23076515 + - 0.5692555 + - -0.08020617 + - -0.30914536 + - -1.209993 + - -0.50007707 + - -0.37780604 + - -0.0598822 + - -1.0006276 + - -0.39157873 + - -1.1754516 + - -0.20212841 + - -0.4339072 + - 0.3072314 + - -0.1764683 + - -0.8870701 + - 0.55130166 + - 0.23388444 + - 0.5114897 + - -0.13851677 + - 0.5070149 + - 0.73393387 + - 0.42486447 + - -0.19826415 + - -0.2080145 + - 0.92281014 + - -0.347401 + - -0.7014914 + - -0.3797622 + - 0.786996 + - -0.36280105 + - -0.0140447 + - -0.21591327 + - 0.23173876 + - 0.21724789 + - 0.28368858 + - -0.5846553 + - -0.7910965 + - 0.21771987 + - 0.31363276 + - -0.7016487 + - -0.6870707 + - 0.5295185 + - 0.7203898 + - 0.055928916 + - -0.15740962 + - -0.21706663 + - -0.058954835 + - 0.14719231 + - 0.30693802 + - 0.15094434 + - 0.19656686 + - -0.3160408 + - -0.15512034 + - -1.2784603 + - 0.15233088 + - -1.0080236 + - -0.000720237 + - -0.038671248 + - -0.10807742 + - 0.8479633 + - -0.47854975 + - 1.1430643 + - 0.6298817 + - 0.13775352 + - 0.3741119 + - -0.19794297 + - 0.6444861 + - -0.19297078 + - 0.46881422 + - -0.16479525 + - -0.5200434 + - 0.13135895 + - -0.44248644 + - 0.46925932 + - 0.27678004 + - -0.403421 + - -0.7909126 + - -0.0855446 + - -1.542924 + - -0.5085412 + - 0.21385814 + - -0.715042 + - 0.56146306 + - -0.6167425 + - 0.46498665 + - 0.93407756 + - 0.14437045 + - -0.716309 + - 0.116511896 + - -0.24489665 + - 0.6450039 + - -0.32258338 + - 0.9406323 + - -0.36568338 + - 1.1295815 + - -0.8524075 + - 1.2329407 + - -0.46062726 + - 0.5174305 + - 0.52787596 + - -0.21639708 + - -0.54905576 + - -0.33936718 + - -0.17682505 + - -0.1648933 + - 0.71241367 + - -0.4136393 + - 0.08463591 + - 0.719956 + - -0.26940387 + - 0.25011218 + - -0.72873074 + - -0.4730632 + - -0.92148495 + - 0.5936796 + - -0.17978978 + - 0.27298036 + - 0.5542751 + - -1.2200156 + - 1.729739 + - 0.05547877 + - 0.09480818 + - 0.5203032 + - 0.44673485 + - -1.1334459 + - 0.18587185 + - 0.38202763 + - 0.79516286 + - -0.53067493 + - -0.58314437 + - 0.21462007 + - 0.45758018 + - -0.42067245 + - -0.069073476 + - 0.9129353 + - 0.4624258 + - 0.32548332 + - 0.6848193 + - 0.15837775 + - -0.6672235 + - 0.5835604 + - -1.061395 + - 0.4369138 + - -0.5998701 + - -0.30487373 + - 0.0070526814 + - -0.32386285 + - -0.053613268 + - -1.518894 + - -0.55960566 + - 0.8191565 + - 0.20065679 + - 1.0555451 + - -0.36626932 + - 0.12234365 + - 0.19914 + - 0.39978758 + - 0.42502803 + - -0.789423 + - -0.23936573 + - -0.92901933 + - 0.65265113 + - -0.47609827 + - -0.48883986 + - 0.13586016 + - -0.015904961 + - -0.27119645 + - -0.3843823 + - -0.9164764 + - 1.4105637 + - 0.06634376 + - -0.54394716 + - 0.19188714 + - -0.91326475 + - -0.06878029 + - 0.29199034 + - 0.87362385 + - -0.66567427 + - 0.32954842 + - -0.70667595 + - -0.76947385 + - -0.65125513 + - -0.30774787 + - -0.9317023 + - -0.32665515 + - 0.72586393 + - -0.75036126 + - 0.082919516 + - 0.58263296 + - -1.2264864 + - 0.057144426 + - 0.46637884 + - 0.7696191 + - -0.5372799 + - -0.45179513 + - 0.10078671 + - -0.07869655 + - 0.0096345795 + - -0.175501 + - 0.004681237 + - -0.1404487 + - -0.026245907 + - 0.4005693 + - -0.30025175 + - 0.22953668 + - 0.1375416 + - 0.428519 + - 0.39739782 + - -1.0153538 + - 0.1460948 + - -0.2423155 + - 0.8443164 + - 0.69656485 + - 0.04835316 + - -0.15647319 + - 0.43619663 + - 0.62418705 + - 0.01624326 + - -0.7810276 + - -0.010477738 + - 0.76305914 + - -0.38192648 + - -0.19829656 + - 0.8082138 + - -0.010607832 + - 0.6590785 + - -0.34272116 + - -0.0046392796 + - -0.3724959 + - -0.5596653 + - 0.6276135 + - -0.061553206 + - -0.36661062 + - 0.08592417 + - 0.625833 + - -0.6575443 + - 0.2936597 + - -1.1686789 + - 0.38760704 + - -0.15154664 + - -0.46800917 + - 0.8223411 + - 0.22256187 + - 0.3998943 + - 0.47397232 + - 0.3206088 + - -0.121969394 + - -0.36568233 + - 0.91466475 + - 0.16137335 + - 0.7220207 + - 0.006176531 + - -0.7444068 + - -0.4841184 + - 0.19201957 + - 0.047102544 + - 1.8374208 + - 0.6271223 + - -0.10150748 + - 0.336917 + - -0.0090131145 + - 0.12568006 + - -1.3409635 + - -0.35267213 + - -0.23100877 + - -0.1741425 + - -1.0097998 + - -0.7858335 + - 0.8512763 + - -0.30711672 + - 0.5460912 + - -0.0992859 + - 0.4857912 + - 0.20211148 + - -0.40408227 + - 0.26641554 + - -0.56869894 + - 0.1889131 + - 0.27093658 + - -0.5806636 + - 1.0632486 + - -0.36107498 + - 1.0024666 + - 0.05458025 + - 0.5792241 + - -0.9272068 + - -1.0779783 + - 0.62059027 + - -0.27215776 + - 0.11107396 + - 0.0011022836 + - -0.10702718 + - 0.19780949 + - 0.072179064 + - -0.19438739 + - 0.69316137 + - 0.6943341 + - 0.57687086 + - -1.1014831 + - 0.4255194 + - -0.58607 + - -0.512174 + - 0.18606424 + - -0.17768177 + - 0.35027477 + - -0.44148716 + - -0.20854093 + - -0.14539136 + - 0.08981536 + - 0.1758918 + - 0.5609762 + - 0.24660097 + - 0.41906935 + - -0.09754385 + - 0.7212798 + - 0.2904664 + - -0.2576649 + - 0.4682982 + - -0.60406595 + - 0.32014373 + - -1.0344557 + - -0.72187984 + - 0.037176646 + - -0.13975306 + - 0.057623375 + - -0.48614138 + - 0.8404527 + - 0.29917234 + - -0.12325492 + - -0.07839739 + - 0.9007334 + - 0.27021715 + - 0.34056118 + - -0.74217767 + - 0.980236 + - -0.1634147 + - -0.20316283 + - -0.023410259 + - -0.2450067 + - 0.4445401 + - 0.6236366 + - -0.43281594 + - -0.10284812 + - 0.3502709 + - -0.009207113 + - 0.6916483 + - 0.13299449 + - -0.5267056 + - 1.0120075 + - 0.09034828 + - 0.64179677 + - -0.06884713 + - -0.3096659 + - -0.54968584 + - 0.0969636 + - 0.18965879 + - -0.40163526 + - -0.07182622 + - 0.68074423 + - -0.20377555 + - 0.19715635 + - 0.9080299 + - 0.024624022 + - -0.2899889 + - 0.1345473 + - -0.40604243 + - 0.12468854 + - -0.051036913 + - 0.16812155 + - -0.66958195 + - 0.411691 + - 0.04175844 + - 0.29515392 + - -0.5747508 + - -0.25879157 + - 1.1121367 + - 0.1084293 + - 0.36287305 + - -0.5590087 + - -0.03730988 + - 0.052817043 + - -0.11966555 + - -0.30272052 + - 0.57527596 + - 0.663233 + - 0.42750266 + - 0.5033948 + - 0.21663116 + - 0.46812102 + - -0.49616772 + - 0.43858975 + - -0.48220062 + - -0.003080981 + - 0.8108525 + - 0.25699288 + - -0.19481191 + - 0.9055214 + - -0.024103131 + - 0.3134053 + - 0.5684294 + - 0.24340783 + - 0.088701926 + - -0.07711979 + - 0.05121962 + - 0.48422125 + - -0.52892333 + - 0.25582233 + - 0.23758161 + - -0.3485996 + - 0.13731018 + - -0.16480787 + - 0.5783073 + - 0.17884552 + - -1.0146426 + - -0.66681993 + - 0.60188895 + - 0.4832905 + - -0.7189223 + - -0.5613885 + - -0.04321099 + - -0.027440922 + - 0.5754654 + - -0.4696408 + - -0.43667746 + - -0.7276407 + - -0.721124 + - -0.29358408 + - 0.0015935706 + - -0.50316346 + - -0.107116826 + - -1.0039939 + - -0.13279264 + - -0.70395243 + - -0.32447532 + - -0.7673587 + - -0.33175752 + - -0.015077448 + - -0.4932336 + - 1.050733 + - 0.52110904 + - 0.25942072 + - 0.9989553 + - -0.36023852 + - 0.2752299 + - -0.4552475 + - -0.24631326 + - -0.015422392 + - -0.7509097 + - 0.07263742 + - 0.17731163 + - -0.5174938 + - 0.022233691 + - -0.02268611 + - 0.15864547 + - 0.6128295 + - 0.091073506 + - -0.7602731 + - -0.6943347 + - 0.728763 + - 0.11569264 + - 0.107082024 + - -0.50501287 + - -0.74649143 + - -0.081207104 + - -0.1334263 + - -0.803402 + - -0.04353917 + - 0.26879165 + - -0.13799252 + - -0.5837927 + - 0.10597579 + - 0.6506158 + - 0.13967927 + - 0.22376712 + - -0.20936072 + - -0.92874765 + - -1.0964315 + - 0.4067096 + - -0.05295724 + - -0.097517624 + - -0.054456163 + - -0.11999136 + - -0.36898333 + - 0.3871025 + - -0.101102784 + - 0.6096284 + - -0.6967617 + - 0.46454057 + - -0.70933527 + - 1.3312901 + - 0.15540215 + - 0.0498251 + - 1.16235 + - -0.77596414 + - -0.032434605 + - 0.28740075 + - 0.5178262 + - -0.21571197 + - -1.4859997 + - 0.18299317 + - -0.0539183 + - -0.3600645 + - -0.47527483 + - 0.20545302 + - -0.05270893 + - -0.2302539 + - 0.97856325 + - 0.41979322 + - 0.3101934 + - 0.54819953 + - -0.038455732 + - -0.6436342 + - 0.41197804 + - -0.09605109 + - -0.5797915 + - -0.05929953 + - -0.3528435 + - -0.21341577 + - -0.21503763 + - 0.4439845 + - -0.21653391 + - -1.0152841 + - 0.14308086 + - 0.5781706 + - 0.60956335 + - 0.6356166 + - -0.5208988 + - -0.27639788 + - 0.9113303 + - -0.18646738 + - -0.5261109 + - -0.9746021 + - -0.1857376 + - 0.7562443 + - -0.38231412 + - -1.1720121 + - 0.58257335 + - 0.40958118 + - -0.19192274 + - 0.37512276 + - -0.4043483 + - -0.27556118 + - -0.84106874 + - -0.063658975 + - 1.4388419 + - 0.13686071 + - 0.019977408 + - 0.1799779 + - -0.77664363 + - -0.49418116 + - -0.021396754 + - 0.39700666 + - -0.012493999 + - -0.6617178 + - 0.002628369 + - 0.4220344 + - -0.3625256 + - -0.1397155 + - 0.29987186 + - -0.67692703 + - -1.2585524 + - 0.5911011 + - 0.4588466 + - 0.13478583 + - 0.59147155 + - -0.071932904 + - -0.2834884 + - -0.061164934 + - -0.7110289 + - 0.122869596 + - 1.0284764 + - -0.5590403 + - 0.60044175 + - -0.48428366 + - -0.44081566 + - 0.26646322 + - 0.27263248 + - 0.44107136 + - -0.49937 + - 0.070135154 + - -0.8562865 + - -0.09105257 + - -0.033030003 + - -0.9996763 + - 0.47625217 + - -0.07518137 + - -0.18114424 + - 0.64506584 + - 0.3145451 + - -0.14525168 + - -0.60409915 + - 0.62508523 + - 0.772962 + - -0.5830134 + - 0.16459994 + - -0.39414343 + - 0.49310187 + - 0.6046758 + - -0.29815552 + - 1.505423 + - 0.2486844 + - 0.7593564 + - 0.049756315 + - -0.22769424 + - -1.0260012 + - -0.18081093 + - 0.012249806 + - -0.79507667 + - -0.20898518 + - 0.89436257 + - 0.040965617 + - -0.26393288 + - 0.24376093 + - 1.0225409 + - -0.7895722 + - 0.7053042 + - -0.7965911 + - 0.21728574 + - -0.3322699 + - 0.8294584 + - -0.04298741 + - 0.67676455 + - 0.026964834 + - -0.07045836 + - 0.3374315 + - -0.36341825 + - 0.71740615 + - 0.27053744 + - -0.2469431 + - 1.0310892 + - 0.0010731986 + - 0.85153055 + - 0.67878973 + - 0.20049824 + - -0.088589296 + - -0.7608825 + - 0.16530535 + - -0.060886715 + - -0.43441933 + - 0.27762952 + - 0.3499589 + - 0.002231119 + - 0.7714241 + - -0.929573 + - -0.28016073 + - -0.2926352 + - -0.22947843 + - 0.59893095 + - 0.6900483 + - 0.30413526 + - 0.058415525 + - -0.7559086 + - -0.40068153 + - -0.02234279 + - -0.8754982 + - -0.92537254 + - 1.2867967 + - -0.14284074 + - 0.6293071 + - -0.25400048 + - -0.27964205 + - -0.5605269 + - 0.030744681 + - 0.43876863 + - -0.74606466 + - 0.8694149 + - 0.2686188 + - 0.32451838 + - -0.4265612 +- - -0.86308616 + - 0.10464997 + - -0.80808467 + - 0.109152414 + - -0.15828495 + - -1.2738355 + - 0.05375433 + - -0.10310792 + - -0.3548497 + - 1.4138002 + - -0.29403353 + - 0.20758511 + - -0.32001764 + - 1.843133 + - 0.17455177 + - 0.2209413 + - -0.2732478 + - -0.6900029 + - 0.96186703 + - -0.0105450405 + - -0.2412575 + - -0.3360454 + - -0.6082248 + - -0.0072297417 + - -1.0747737 + - -0.31050614 + - -0.16715746 + - -0.32832184 + - 0.8434579 + - -0.41397455 + - 0.122941874 + - -0.2372269 + - -0.56031376 + - 0.9161398 + - 0.25382963 + - 0.8885782 + - 0.29874283 + - -0.5619264 + - 0.6109048 + - 0.08201033 + - 0.49943748 + - 0.26624915 + - 0.05867671 + - 0.8383686 + - -0.0047096624 + - 0.66836816 + - -0.108843766 + - 0.35794806 + - -0.15595329 + - -0.043658007 + - -0.6172099 + - 0.51299053 + - 0.93706584 + - -0.42387018 + - 0.6119756 + - 0.027609877 + - -0.73200893 + - 0.57787377 + - 0.8033046 + - -0.581951 + - -0.7731883 + - 0.5967651 + - 0.49375257 + - 0.53678864 + - 0.8993954 + - -0.8382513 + - -0.0029070543 + - -0.60662335 + - -0.18346624 + - -0.30130884 + - -0.05985644 + - -0.59661436 + - -0.21889071 + - 0.44902173 + - -1.0257684 + - 0.029930225 + - 0.59746224 + - 0.291006 + - 0.5898661 + - -0.5916474 + - -0.21837401 + - -0.33867082 + - 0.27094448 + - -0.296837 + - 0.29892462 + - -0.51586765 + - 0.10145957 + - 0.7413406 + - -1.2221884 + - -0.4309417 + - 0.05030532 + - -0.6016666 + - 0.7891275 + - 0.14696632 + - -0.111191295 + - 0.4298819 + - -0.67413133 + - -0.5593469 + - -0.16509695 + - 0.71434355 + - -0.44043005 + - -0.14371319 + - -1.6683999 + - -0.37712744 + - -0.10542617 + - 0.17461449 + - -0.26690745 + - 0.085009225 + - -1.1847318 + - -0.23936786 + - -0.733366 + - -0.60240996 + - -0.35942093 + - -0.9000824 + - 0.5874027 + - -0.118941985 + - 0.3715794 + - -0.40944728 + - 0.67932075 + - 0.057728305 + - 0.7374434 + - 0.1117717 + - 0.023495674 + - -0.07271193 + - 0.06676536 + - -1.006733 + - -0.9478982 + - 1.3349262 + - -0.737581 + - 0.68273497 + - -0.4956067 + - 0.33197424 + - 0.13116665 + - -0.1798827 + - -0.14980775 + - -0.46851328 + - -0.40833887 + - 0.016854903 + - -0.8115403 + - -1.2469426 + - 0.55281264 + - 0.5221022 + - 0.05622865 + - -0.16240837 + - 0.22340138 + - 0.6435249 + - -0.36334816 + - 0.58208853 + - 0.1815416 + - -0.13326487 + - -1.3188778 + - -0.1581764 + - -1.2683326 + - -0.0339574 + - -0.25547576 + - 0.18386304 + - 0.024977917 + - -0.030794144 + - 0.41509464 + - -0.64639676 + - 1.2658962 + - 0.28116882 + - 0.30553403 + - 0.40011147 + - -0.4649098 + - 1.0439867 + - -0.2923201 + - 0.47910428 + - 0.1544699 + - -0.24046157 + - -0.21272773 + - -0.7077553 + - 0.5009159 + - -0.2960263 + - 0.61571294 + - -0.7693715 + - -0.34947774 + - -1.3543078 + - -1.0186294 + - 0.35498118 + - -0.8818839 + - 0.6798013 + - -0.46030948 + - 0.3195989 + - 0.9565864 + - 0.047423895 + - -0.5621652 + - 0.44032264 + - 0.26237497 + - 0.38165414 + - -0.040655926 + - 0.7697478 + - -0.39147845 + - 0.79319954 + - -0.37920806 + - 1.5294644 + - -0.3448595 + - 0.87782663 + - 0.45272708 + - -0.47513387 + - -0.42728475 + - -0.011001346 + - -0.2705562 + - 0.32977 + - 0.22692628 + - -0.354846 + - -0.1310609 + - 0.9872984 + - -0.3510219 + - 0.3671743 + - -0.7636153 + - -0.4028881 + - -0.36776432 + - 0.5721725 + - -0.4784616 + - 0.48841962 + - 0.14041027 + - -1.2891492 + - 1.7725534 + - -0.47616413 + - 0.18200104 + - 0.2769574 + - 0.25855923 + - -0.37332287 + - 0.07835276 + - 0.10667632 + - 1.0563878 + - 0.12068716 + - -0.23853846 + - -0.22017126 + - 0.7035559 + - -0.47703043 + - -0.6087915 + - 1.1650358 + - 0.6085283 + - 0.52696735 + - 0.16584232 + - 0.49026525 + - -1.0868301 + - 0.74892825 + - -0.8436255 + - 0.7385864 + - -0.43516207 + - 0.5527646 + - -0.4448839 + - 0.2818894 + - -0.6056908 + - -0.685338 + - -0.4145106 + - 0.11028206 + - 0.61945087 + - 1.1363989 + - 0.018605823 + - 0.06742182 + - -0.6173551 + - -0.08472548 + - 0.4248147 + - -0.8183348 + - 0.17736025 + - -1.0447739 + - 0.75808483 + - -0.48231182 + - -0.1243229 + - 0.22086062 + - 0.06127879 + - -0.35528985 + - -0.21681786 + - -0.523696 + - 0.8380065 + - 0.5820177 + - -0.08492304 + - 0.27881643 + - -1.0124083 + - -0.7998883 + - -0.24380971 + - 1.1094896 + - -0.07898712 + - 0.1359468 + - -0.5433409 + - 0.0049818307 + - -0.41290048 + - -0.58589584 + - -0.88302493 + - -0.3521521 + - 1.043377 + - -0.77324086 + - 0.55165845 + - -0.27075413 + - -1.202274 + - 0.5694114 + - 0.18791 + - 0.28535804 + - -0.3861871 + - -0.17400663 + - -0.11858777 + - 0.034684636 + - 0.04005277 + - 0.0039117136 + - -0.5758771 + - 0.74151605 + - -0.6760163 + - 0.58027405 + - -0.37907383 + - -0.049797684 + - 0.5154967 + - 0.82435125 + - 0.13353427 + - -1.0112203 + - 0.532788 + - -0.12729378 + - 0.91297394 + - 0.813425 + - 0.3960221 + - -0.5477232 + - 0.30966398 + - 0.341457 + - 0.0120184375 + - -0.30117288 + - 0.23168226 + - 0.18655288 + - 0.23527318 + - 0.40739408 + - 0.61692345 + - 0.029422304 + - 0.9650593 + - -0.8804412 + - 0.050632652 + - -0.23598003 + - -0.49167705 + - 0.5251496 + - -0.21965893 + - -0.45838204 + - 0.23766638 + - 0.9608357 + - -0.95765513 + - 1.1148227 + - -1.3307385 + - 0.5944306 + - -0.5378292 + - -0.71914643 + - 0.41962424 + - -0.59007645 + - 0.46740794 + - -0.042504076 + - -0.10402564 + - 0.012016277 + - -0.08533021 + - 0.98233074 + - 0.025441095 + - 0.9648552 + - 0.786962 + - -0.7344964 + - -0.32460403 + - -0.43257454 + - -0.6504927 + - 0.78977853 + - 0.2919153 + - 0.2535027 + - 0.4524282 + - 0.04249085 + - -0.2710552 + - -0.7433588 + - -0.7306842 + - 0.13140512 + - 0.51823956 + - -0.98799545 + - -0.6945293 + - 0.866034 + - -0.2000788 + - 0.33214685 + - -0.27409673 + - 0.2661763 + - 0.67158175 + - -0.11686953 + - 0.55837214 + - -0.84951836 + - 0.025794843 + - 0.70500857 + - -0.25203836 + - 0.8520818 + - -0.36496553 + - 0.51388377 + - -0.10496068 + - 0.58563095 + - -0.82840794 + - -0.55039555 + - 0.58924216 + - -0.3359559 + - -0.07322076 + - -0.058717564 + - 0.018051272 + - 0.14977045 + - 0.6528148 + - -0.71195716 + - 1.3481878 + - -0.25131366 + - 0.3270264 + - -0.38991675 + - 0.22034688 + - -0.26104045 + - -0.67168117 + - -0.31336293 + - -0.44053257 + - 0.30768892 + - -0.8275053 + - -0.53929037 + - -0.35630056 + - 0.18504627 + - 0.29028675 + - 0.42018428 + - 0.21417886 + - 0.22473858 + - -0.17941523 + - 0.01764693 + - 0.0932853 + - -0.0155034615 + - -0.3128794 + - 0.49276984 + - 0.6897387 + - -1.4886831 + - -0.44153237 + - 0.21322094 + - 0.40350547 + - -0.02185589 + - -0.50499696 + - 0.99464726 + - 0.06740898 + - 0.37285092 + - 0.04863594 + - 0.7077425 + - -0.5864583 + - 0.309001 + - -0.47615814 + - 0.69945574 + - 0.021937171 + - 0.08646644 + - -0.2709289 + - -0.23897363 + - 0.4927977 + - 0.40029457 + - -0.6280861 + - -0.0143659385 + - 0.05625758 + - 0.06321309 + - 0.32563493 + - -0.1552612 + - -0.9132417 + - 0.6204322 + - -0.29666343 + - 1.3124477 + - 0.094627135 + - -0.33526766 + - -0.44622967 + - -0.3237933 + - 0.48829868 + - -0.38729754 + - -0.006328776 + - 0.5578793 + - -0.16057178 + - 0.77103406 + - 0.18943106 + - -0.018942509 + - -0.07714457 + - 0.32512534 + - -0.3751991 + - -0.10964618 + - 0.2745956 + - 0.43049303 + - -0.58570474 + - 0.6261323 + - -0.17841522 + - 0.16165124 + - -0.37555477 + - 0.1557496 + - 0.78425884 + - 0.22643082 + - 0.33282852 + - 0.61374354 + - -0.13492967 + - -0.48917207 + - -0.042952914 + - 0.18701614 + - 1.0441581 + - 0.29983467 + - 0.74226505 + - 0.80055046 + - 0.21692531 + - 0.30234173 + - -0.9891658 + - 0.12650566 + - -0.46991107 + - -0.5512108 + - 0.82059526 + - 0.26155686 + - -0.045447152 + - 0.4180746 + - -0.18579914 + - 0.7279482 + - 0.6228181 + - 0.3093444 + - 0.23444466 + - -0.2871354 + - 0.6030943 + - 0.5404321 + - -0.813701 + - 0.49120298 + - 0.95289963 + - -1.2181894 + - 0.030967312 + - -0.4496815 + - 0.81202173 + - 0.70029426 + - -0.91281444 + - -0.45131722 + - 0.34122682 + - 0.7654457 + - -0.07829732 + - -0.2362618 + - -0.38699183 + - -0.06236643 + - 0.9480641 + - -0.18549691 + - -0.7685716 + - -0.9714551 + - -0.75964475 + - -0.04847738 + - 0.3208382 + - -0.7281963 + - -0.7822334 + - -0.7692704 + - 0.19750637 + - -0.3697712 + - -0.30289134 + - -0.21314788 + - 0.27933517 + - 0.4371023 + - 0.25537527 + - 1.2279454 + - 0.72324425 + - -0.03689779 + - 1.0635711 + - -0.7319136 + - 0.20963591 + - -0.45486727 + - 0.40675625 + - 0.24463814 + - 0.1811815 + - -0.24058974 + - -0.013899483 + - -0.30323437 + - 0.16242926 + - -0.2635223 + - 0.11749282 + - -0.2688166 + - 0.44207844 + - -0.51439023 + - -0.15018623 + - 0.2577937 + - 0.043446664 + - -0.19307633 + - -0.33019257 + - -0.7190415 + - 0.20666404 + - 0.11624887 + - -0.54698485 + - 0.8176004 + - 0.35207722 + - -0.27402192 + - -0.1035634 + - 0.1875834 + - 1.1572824 + - 0.33517662 + - -0.2041079 + - 0.3648822 + - -0.77751106 + - -0.577764 + - -0.46833587 + - 0.002186878 + - -0.31261548 + - -0.097573526 + - -0.38575947 + - -0.93398434 + - 0.48450616 + - 0.04237418 + - 0.7813783 + - -0.68283325 + - -0.17771558 + - -0.43963957 + - 1.0524217 + - -0.08786762 + - 0.15726765 + - 0.76370144 + - -0.5579179 + - 0.14207757 + - 0.91035765 + - -0.31653282 + - -0.14191888 + - -1.4689041 + - 0.5574543 + - 0.061259825 + - -0.062857 + - -0.7440378 + - 0.05597469 + - -0.8998406 + - -1.1509421 + - 0.31721145 + - 0.27878776 + - 0.5718953 + - -0.14807907 + - -0.35857674 + - -0.3281373 + - 0.42133775 + - -0.142298 + - -0.58175737 + - -0.05921154 + - 0.22090296 + - -0.46445766 + - 0.17104305 + - 0.30820233 + - -0.17526521 + - -0.72379845 + - -0.39376578 + - 0.7832683 + - 1.007903 + - 0.4426504 + - -0.12980466 + - -0.12940495 + - 0.9409998 + - -0.033661637 + - -0.8928602 + - -0.6068395 + - 0.20674825 + - 1.0652231 + - -0.48214188 + - -1.1357911 + - 0.08617532 + - -0.08829456 + - -0.24711962 + - 0.028374488 + - -0.3249136 + - -0.29695013 + - -0.8791 + - 0.43543783 + - 0.3225098 + - -0.28647906 + - 0.7309591 + - 0.31440485 + - -0.25180957 + - -0.26686728 + - -0.56468207 + - 0.5628607 + - 0.23223303 + - -0.18633276 + - -0.2024215 + - 0.6169545 + - -0.021312451 + - 0.3180363 + - 0.19440426 + - -1.2102475 + - -0.83707213 + - 1.1973891 + - 0.89213324 + - 0.15434153 + - 0.9890812 + - -0.07407461 + - 0.10018677 + - 0.2048974 + - -0.45621386 + - -0.049087737 + - 0.49791932 + - 0.1879405 + - 0.3733906 + - -0.43503758 + - -1.0403999 + - -0.18433239 + - -0.814481 + - 0.28998625 + - -0.6942032 + - -0.1196345 + - -0.043878317 + - 0.3437682 + - -0.68714887 + - -0.8128009 + - 0.17070629 + - 0.0587905 + - -0.5077273 + - 0.3158691 + - -0.07015352 + - -0.29545602 + - -0.31041715 + - 0.30650797 + - 0.8237688 + - -0.21543162 + - -0.016031196 + - -0.12330439 + - 0.81942815 + - 0.43117514 + - 0.117400885 + - 1.3050174 + - 0.06559616 + - 0.101292156 + - 0.5597823 + - -0.049042773 + - 0.07320682 + - 0.28808483 + - 0.21869534 + - -0.9404915 + - -0.3743296 + - 0.6700518 + - -0.23625839 + - 0.168872 + - 0.77333885 + - 0.62934524 + - -0.47554758 + - 0.25385 + - -0.04778995 + - -0.037380733 + - -0.27748594 + - 0.8931497 + - -0.1096187 + - 0.7286039 + - 0.60384846 + - -0.78108215 + - 1.092566 + - -0.31005505 + - 0.5053589 + - 0.2190065 + - -0.34338525 + - 0.9687538 + - 0.28556177 + - 0.006131967 + - 0.32191113 + - 0.035777707 + - -0.13235757 + - -0.67691225 + - -0.28016526 + - -0.13505925 + - -0.28522038 + - 0.02211368 + - 0.5016585 + - -0.27227917 + - -0.10245315 + - -1.3294158 + - -0.42046413 + - -0.15157798 + - 0.018878272 + - 0.18859847 + - 0.44895697 + - 0.40317628 + - -0.710586 + - -0.19861041 + - -0.28604063 + - -0.5556029 + - -0.68761843 + - -0.3311822 + - 0.23267186 + - 0.07165238 + - 0.21401186 + - -0.58837694 + - 0.0583059 + - -0.48591343 + - 0.0843542 + - 0.96501607 + - -0.62620467 + - 0.6857476 + - 0.43923378 + - 0.15171519 + - -0.022041125 +- - -0.77039415 + - 0.06603869 + - -0.7481627 + - -0.13090512 + - 0.047572974 + - -0.7196526 + - 0.03157639 + - 0.06883626 + - -0.5306753 + - 1.6497533 + - -0.06626414 + - 0.18275733 + - -0.31899172 + - 1.5625474 + - -0.28502676 + - 0.17592163 + - -0.10744914 + - -0.19912067 + - 0.5304716 + - -0.09754831 + - -0.25156677 + - 0.20254421 + - -0.50044435 + - -0.5243958 + - -0.8732222 + - -0.26390445 + - -0.014966733 + - -0.34825927 + - 1.1057283 + - -0.11190004 + - -0.1537848 + - 0.24017277 + - -0.2439278 + - 0.48591775 + - -0.30278283 + - 0.5074822 + - -0.058676906 + - -0.07136222 + - 0.8357766 + - 0.057142794 + - -0.12005273 + - -0.037646633 + - 0.3695325 + - 0.6662965 + - -0.06519522 + - 0.55348885 + - 0.11334873 + - -0.08590865 + - 0.27785024 + - 0.09716725 + - -0.253027 + - 0.4178844 + - 0.8498044 + - 0.044411708 + - 0.6873774 + - -0.20685801 + - -0.51111424 + - -0.45068893 + - 1.1606588 + - -0.65466064 + - -0.21233405 + - 0.44229627 + - 0.5280446 + - 0.23785159 + - 0.3712364 + - -1.0964439 + - 0.016722241 + - -0.5736524 + - -1.1417898 + - 0.23557658 + - 0.014350355 + - -0.28558 + - -0.06097868 + - -0.048328485 + - -0.72823924 + - 0.34208012 + - 0.34606266 + - -0.0084145265 + - 0.61085355 + - -0.091765486 + - 0.07675646 + - -0.5773393 + - 0.1638734 + - -0.33433697 + - -0.117930256 + - -0.8185136 + - -0.25872475 + - 0.2649337 + - -0.8452235 + - -0.5882134 + - 0.30627438 + - -0.20208707 + - 0.29648748 + - 0.5842015 + - 0.031331632 + - 0.07708081 + - 0.1742084 + - -0.47708175 + - 0.23076515 + - 0.5692555 + - -0.08020617 + - -0.30914536 + - -1.209993 + - -0.50007707 + - -0.37780604 + - -0.0598822 + - -1.0006276 + - -0.39157873 + - -1.1754516 + - -0.20212841 + - -0.4339072 + - 0.3072314 + - -0.1764683 + - -0.8870701 + - 0.55130166 + - 0.23388444 + - 0.5114897 + - -0.13851677 + - 0.5070149 + - 0.73393387 + - 0.42486447 + - -0.19826415 + - -0.2080145 + - 0.92281014 + - -0.347401 + - -0.7014914 + - -0.3797622 + - 0.786996 + - -0.36280105 + - -0.0140447 + - -0.21591327 + - 0.23173876 + - 0.21724789 + - 0.28368858 + - -0.5846553 + - -0.7910965 + - 0.21771987 + - 0.31363276 + - -0.7016487 + - -0.6870707 + - 0.5295185 + - 0.7203898 + - 0.055928916 + - -0.15740962 + - -0.21706663 + - -0.058954835 + - 0.14719231 + - 0.30693802 + - 0.15094434 + - 0.19656686 + - -0.3160408 + - -0.15512034 + - -1.2784603 + - 0.15233088 + - -1.0080236 + - -0.000720237 + - -0.038671248 + - -0.10807742 + - 0.8479633 + - -0.47854975 + - 1.1430643 + - 0.6298817 + - 0.13775352 + - 0.3741119 + - -0.19794297 + - 0.6444861 + - -0.19297078 + - 0.46881422 + - -0.16479525 + - -0.5200434 + - 0.13135895 + - -0.44248644 + - 0.46925932 + - 0.27678004 + - -0.403421 + - -0.7909126 + - -0.0855446 + - -1.542924 + - -0.5085412 + - 0.21385814 + - -0.715042 + - 0.56146306 + - -0.6167425 + - 0.46498665 + - 0.93407756 + - 0.14437045 + - -0.716309 + - 0.116511896 + - -0.24489665 + - 0.6450039 + - -0.32258338 + - 0.9406323 + - -0.36568338 + - 1.1295815 + - -0.8524075 + - 1.2329407 + - -0.46062726 + - 0.5174305 + - 0.52787596 + - -0.21639708 + - -0.54905576 + - -0.33936718 + - -0.17682505 + - -0.1648933 + - 0.71241367 + - -0.4136393 + - 0.08463591 + - 0.719956 + - -0.26940387 + - 0.25011218 + - -0.72873074 + - -0.4730632 + - -0.92148495 + - 0.5936796 + - -0.17978978 + - 0.27298036 + - 0.5542751 + - -1.2200156 + - 1.729739 + - 0.05547877 + - 0.09480818 + - 0.5203032 + - 0.44673485 + - -1.1334459 + - 0.18587185 + - 0.38202763 + - 0.79516286 + - -0.53067493 + - -0.58314437 + - 0.21462007 + - 0.45758018 + - -0.42067245 + - -0.069073476 + - 0.9129353 + - 0.4624258 + - 0.32548332 + - 0.6848193 + - 0.15837775 + - -0.6672235 + - 0.5835604 + - -1.061395 + - 0.4369138 + - -0.5998701 + - -0.30487373 + - 0.0070526814 + - -0.32386285 + - -0.053613268 + - -1.518894 + - -0.55960566 + - 0.8191565 + - 0.20065679 + - 1.0555451 + - -0.36626932 + - 0.12234365 + - 0.19914 + - 0.39978758 + - 0.42502803 + - -0.789423 + - -0.23936573 + - -0.92901933 + - 0.65265113 + - -0.47609827 + - -0.48883986 + - 0.13586016 + - -0.015904961 + - -0.27119645 + - -0.3843823 + - -0.9164764 + - 1.4105637 + - 0.06634376 + - -0.54394716 + - 0.19188714 + - -0.91326475 + - -0.06878029 + - 0.29199034 + - 0.87362385 + - -0.66567427 + - 0.32954842 + - -0.70667595 + - -0.76947385 + - -0.65125513 + - -0.30774787 + - -0.9317023 + - -0.32665515 + - 0.72586393 + - -0.75036126 + - 0.082919516 + - 0.58263296 + - -1.2264864 + - 0.057144426 + - 0.46637884 + - 0.7696191 + - -0.5372799 + - -0.45179513 + - 0.10078671 + - -0.07869655 + - 0.0096345795 + - -0.175501 + - 0.004681237 + - -0.1404487 + - -0.026245907 + - 0.4005693 + - -0.30025175 + - 0.22953668 + - 0.1375416 + - 0.428519 + - 0.39739782 + - -1.0153538 + - 0.1460948 + - -0.2423155 + - 0.8443164 + - 0.69656485 + - 0.04835316 + - -0.15647319 + - 0.43619663 + - 0.62418705 + - 0.01624326 + - -0.7810276 + - -0.010477738 + - 0.76305914 + - -0.38192648 + - -0.19829656 + - 0.8082138 + - -0.010607832 + - 0.6590785 + - -0.34272116 + - -0.0046392796 + - -0.3724959 + - -0.5596653 + - 0.6276135 + - -0.061553206 + - -0.36661062 + - 0.08592417 + - 0.625833 + - -0.6575443 + - 0.2936597 + - -1.1686789 + - 0.38760704 + - -0.15154664 + - -0.46800917 + - 0.8223411 + - 0.22256187 + - 0.3998943 + - 0.47397232 + - 0.3206088 + - -0.121969394 + - -0.36568233 + - 0.91466475 + - 0.16137335 + - 0.7220207 + - 0.006176531 + - -0.7444068 + - -0.4841184 + - 0.19201957 + - 0.047102544 + - 1.8374208 + - 0.6271223 + - -0.10150748 + - 0.336917 + - -0.0090131145 + - 0.12568006 + - -1.3409635 + - -0.35267213 + - -0.23100877 + - -0.1741425 + - -1.0097998 + - -0.7858335 + - 0.8512763 + - -0.30711672 + - 0.5460912 + - -0.0992859 + - 0.4857912 + - 0.20211148 + - -0.40408227 + - 0.26641554 + - -0.56869894 + - 0.1889131 + - 0.27093658 + - -0.5806636 + - 1.0632486 + - -0.36107498 + - 1.0024666 + - 0.05458025 + - 0.5792241 + - -0.9272068 + - -1.0779783 + - 0.62059027 + - -0.27215776 + - 0.11107396 + - 0.0011022836 + - -0.10702718 + - 0.19780949 + - 0.072179064 + - -0.19438739 + - 0.69316137 + - 0.6943341 + - 0.57687086 + - -1.1014831 + - 0.4255194 + - -0.58607 + - -0.512174 + - 0.18606424 + - -0.17768177 + - 0.35027477 + - -0.44148716 + - -0.20854093 + - -0.14539136 + - 0.08981536 + - 0.1758918 + - 0.5609762 + - 0.24660097 + - 0.41906935 + - -0.09754385 + - 0.7212798 + - 0.2904664 + - -0.2576649 + - 0.4682982 + - -0.60406595 + - 0.32014373 + - -1.0344557 + - -0.72187984 + - 0.037176646 + - -0.13975306 + - 0.057623375 + - -0.48614138 + - 0.8404527 + - 0.29917234 + - -0.12325492 + - -0.07839739 + - 0.9007334 + - 0.27021715 + - 0.34056118 + - -0.74217767 + - 0.980236 + - -0.1634147 + - -0.20316283 + - -0.023410259 + - -0.2450067 + - 0.4445401 + - 0.6236366 + - -0.43281594 + - -0.10284812 + - 0.3502709 + - -0.009207113 + - 0.6916483 + - 0.13299449 + - -0.5267056 + - 1.0120075 + - 0.09034828 + - 0.64179677 + - -0.06884713 + - -0.3096659 + - -0.54968584 + - 0.0969636 + - 0.18965879 + - -0.40163526 + - -0.07182622 + - 0.68074423 + - -0.20377555 + - 0.19715635 + - 0.9080299 + - 0.024624022 + - -0.2899889 + - 0.1345473 + - -0.40604243 + - 0.12468854 + - -0.051036913 + - 0.16812155 + - -0.66958195 + - 0.411691 + - 0.04175844 + - 0.29515392 + - -0.5747508 + - -0.25879157 + - 1.1121367 + - 0.1084293 + - 0.36287305 + - -0.5590087 + - -0.03730988 + - 0.052817043 + - -0.11966555 + - -0.30272052 + - 0.57527596 + - 0.663233 + - 0.42750266 + - 0.5033948 + - 0.21663116 + - 0.46812102 + - -0.49616772 + - 0.43858975 + - -0.48220062 + - -0.003080981 + - 0.8108525 + - 0.25699288 + - -0.19481191 + - 0.9055214 + - -0.024103131 + - 0.3134053 + - 0.5684294 + - 0.24340783 + - 0.088701926 + - -0.07711979 + - 0.05121962 + - 0.48422125 + - -0.52892333 + - 0.25582233 + - 0.23758161 + - -0.3485996 + - 0.13731018 + - -0.16480787 + - 0.5783073 + - 0.17884552 + - -1.0146426 + - -0.66681993 + - 0.60188895 + - 0.4832905 + - -0.7189223 + - -0.5613885 + - -0.04321099 + - -0.027440922 + - 0.5754654 + - -0.4696408 + - -0.43667746 + - -0.7276407 + - -0.721124 + - -0.29358408 + - 0.0015935706 + - -0.50316346 + - -0.107116826 + - -1.0039939 + - -0.13279264 + - -0.70395243 + - -0.32447532 + - -0.7673587 + - -0.33175752 + - -0.015077448 + - -0.4932336 + - 1.050733 + - 0.52110904 + - 0.25942072 + - 0.9989553 + - -0.36023852 + - 0.2752299 + - -0.4552475 + - -0.24631326 + - -0.015422392 + - -0.7509097 + - 0.07263742 + - 0.17731163 + - -0.5174938 + - 0.022233691 + - -0.02268611 + - 0.15864547 + - 0.6128295 + - 0.091073506 + - -0.7602731 + - -0.6943347 + - 0.728763 + - 0.11569264 + - 0.107082024 + - -0.50501287 + - -0.74649143 + - -0.081207104 + - -0.1334263 + - -0.803402 + - -0.04353917 + - 0.26879165 + - -0.13799252 + - -0.5837927 + - 0.10597579 + - 0.6506158 + - 0.13967927 + - 0.22376712 + - -0.20936072 + - -0.92874765 + - -1.0964315 + - 0.4067096 + - -0.05295724 + - -0.097517624 + - -0.054456163 + - -0.11999136 + - -0.36898333 + - 0.3871025 + - -0.101102784 + - 0.6096284 + - -0.6967617 + - 0.46454057 + - -0.70933527 + - 1.3312901 + - 0.15540215 + - 0.0498251 + - 1.16235 + - -0.77596414 + - -0.032434605 + - 0.28740075 + - 0.5178262 + - -0.21571197 + - -1.4859997 + - 0.18299317 + - -0.0539183 + - -0.3600645 + - -0.47527483 + - 0.20545302 + - -0.05270893 + - -0.2302539 + - 0.97856325 + - 0.41979322 + - 0.3101934 + - 0.54819953 + - -0.038455732 + - -0.6436342 + - 0.41197804 + - -0.09605109 + - -0.5797915 + - -0.05929953 + - -0.3528435 + - -0.21341577 + - -0.21503763 + - 0.4439845 + - -0.21653391 + - -1.0152841 + - 0.14308086 + - 0.5781706 + - 0.60956335 + - 0.6356166 + - -0.5208988 + - -0.27639788 + - 0.9113303 + - -0.18646738 + - -0.5261109 + - -0.9746021 + - -0.1857376 + - 0.7562443 + - -0.38231412 + - -1.1720121 + - 0.58257335 + - 0.40958118 + - -0.19192274 + - 0.37512276 + - -0.4043483 + - -0.27556118 + - -0.84106874 + - -0.063658975 + - 1.4388419 + - 0.13686071 + - 0.019977408 + - 0.1799779 + - -0.77664363 + - -0.49418116 + - -0.021396754 + - 0.39700666 + - -0.012493999 + - -0.6617178 + - 0.002628369 + - 0.4220344 + - -0.3625256 + - -0.1397155 + - 0.29987186 + - -0.67692703 + - -1.2585524 + - 0.5911011 + - 0.4588466 + - 0.13478583 + - 0.59147155 + - -0.071932904 + - -0.2834884 + - -0.061164934 + - -0.7110289 + - 0.122869596 + - 1.0284764 + - -0.5590403 + - 0.60044175 + - -0.48428366 + - -0.44081566 + - 0.26646322 + - 0.27263248 + - 0.44107136 + - -0.49937 + - 0.070135154 + - -0.8562865 + - -0.09105257 + - -0.033030003 + - -0.9996763 + - 0.47625217 + - -0.07518137 + - -0.18114424 + - 0.64506584 + - 0.3145451 + - -0.14525168 + - -0.60409915 + - 0.62508523 + - 0.772962 + - -0.5830134 + - 0.16459994 + - -0.39414343 + - 0.49310187 + - 0.6046758 + - -0.29815552 + - 1.505423 + - 0.2486844 + - 0.7593564 + - 0.049756315 + - -0.22769424 + - -1.0260012 + - -0.18081093 + - 0.012249806 + - -0.79507667 + - -0.20898518 + - 0.89436257 + - 0.040965617 + - -0.26393288 + - 0.24376093 + - 1.0225409 + - -0.7895722 + - 0.7053042 + - -0.7965911 + - 0.21728574 + - -0.3322699 + - 0.8294584 + - -0.04298741 + - 0.67676455 + - 0.026964834 + - -0.07045836 + - 0.3374315 + - -0.36341825 + - 0.71740615 + - 0.27053744 + - -0.2469431 + - 1.0310892 + - 0.0010731986 + - 0.85153055 + - 0.67878973 + - 0.20049824 + - -0.088589296 + - -0.7608825 + - 0.16530535 + - -0.060886715 + - -0.43441933 + - 0.27762952 + - 0.3499589 + - 0.002231119 + - 0.7714241 + - -0.929573 + - -0.28016073 + - -0.2926352 + - -0.22947843 + - 0.59893095 + - 0.6900483 + - 0.30413526 + - 0.058415525 + - -0.7559086 + - -0.40068153 + - -0.02234279 + - -0.8754982 + - -0.92537254 + - 1.2867967 + - -0.14284074 + - 0.6293071 + - -0.25400048 + - -0.27964205 + - -0.5605269 + - 0.030744681 + - 0.43876863 + - -0.74606466 + - 0.8694149 + - 0.2686188 + - 0.32451838 + - -0.4265612 diff --git a/backends/candle/tests/snapshots/test_nomic__nomic_moe_single.snap b/backends/candle/tests/snapshots/test_nomic__nomic_moe_single.snap new file mode 100644 index 00000000..e8561a50 --- /dev/null +++ b/backends/candle/tests/snapshots/test_nomic__nomic_moe_single.snap @@ -0,0 +1,772 @@ +--- +source: backends/candle/tests/test_nomic.rs +expression: embeddings_single +--- +- - -0.7703948 + - 0.06603849 + - -0.74816215 + - -0.13090475 + - 0.047573686 + - -0.7196515 + - 0.031576473 + - 0.06883613 + - -0.53067523 + - 1.6497536 + - -0.066263415 + - 0.18275724 + - -0.3189916 + - 1.5625484 + - -0.28502634 + - 0.17592178 + - -0.107449055 + - -0.1991202 + - 0.53047216 + - -0.09754828 + - -0.25156704 + - 0.20254405 + - -0.5004441 + - -0.5243957 + - -0.8732222 + - -0.26390472 + - -0.014966825 + - -0.34825882 + - 1.1057272 + - -0.11190005 + - -0.15378438 + - 0.24017313 + - -0.24392812 + - 0.48591736 + - -0.3027828 + - 0.50748223 + - -0.058677085 + - -0.07136195 + - 0.83577645 + - 0.05714326 + - -0.120052375 + - -0.037646864 + - 0.36953232 + - 0.6662963 + - -0.065195195 + - 0.55348945 + - 0.11334801 + - -0.085908584 + - 0.27785012 + - 0.09716708 + - -0.253027 + - 0.41788417 + - 0.84980404 + - 0.044411607 + - 0.68737745 + - -0.20685814 + - -0.5111138 + - -0.45068842 + - 1.1606585 + - -0.65466064 + - -0.21233349 + - 0.4422962 + - 0.5280449 + - 0.23785187 + - 0.3712372 + - -1.0964442 + - 0.016722301 + - -0.5736521 + - -1.1417893 + - 0.23557632 + - 0.014350495 + - -0.2855798 + - -0.060978733 + - -0.048328497 + - -0.72823864 + - 0.34208003 + - 0.34606233 + - -0.008414639 + - 0.6108536 + - -0.091765694 + - 0.07675648 + - -0.5773389 + - 0.16387363 + - -0.33433706 + - -0.11792964 + - -0.81851333 + - -0.25872478 + - 0.26493293 + - -0.84522337 + - -0.58821297 + - 0.30627465 + - -0.20208721 + - 0.2964886 + - 0.5842018 + - 0.031332072 + - 0.07708082 + - 0.17420901 + - -0.47708127 + - 0.23076515 + - 0.56925577 + - -0.08020631 + - -0.3091448 + - -1.2099931 + - -0.5000771 + - -0.37780625 + - -0.059882045 + - -1.000628 + - -0.39157888 + - -1.1754519 + - -0.20212774 + - -0.43390736 + - 0.30723074 + - -0.176468 + - -0.8870705 + - 0.5513014 + - 0.23388405 + - 0.5114903 + - -0.1385168 + - 0.507015 + - 0.7339341 + - 0.42486444 + - -0.19826467 + - -0.20801544 + - 0.9228097 + - -0.34740096 + - -0.7014914 + - -0.37976262 + - 0.78699666 + - -0.36280194 + - -0.014044191 + - -0.21591327 + - 0.2317385 + - 0.21724835 + - 0.28368837 + - -0.58465487 + - -0.79109603 + - 0.21772003 + - 0.31363297 + - -0.70164907 + - -0.68707085 + - 0.5295186 + - 0.7203898 + - 0.055929065 + - -0.15741046 + - -0.2170672 + - -0.05895475 + - 0.1471919 + - 0.30693787 + - 0.1509443 + - 0.19656661 + - -0.31604093 + - -0.1551203 + - -1.2784597 + - 0.15233077 + - -1.0080247 + - -0.0007201433 + - -0.03867173 + - -0.10807766 + - 0.8479634 + - -0.4785497 + - 1.1430641 + - 0.6298814 + - 0.13775377 + - 0.3741122 + - -0.19794323 + - 0.6444861 + - -0.19297123 + - 0.46881464 + - -0.16479567 + - -0.5200432 + - 0.13135923 + - -0.4424868 + - 0.46925932 + - 0.27678028 + - -0.40342093 + - -0.7909128 + - -0.08554523 + - -1.5429238 + - -0.5085409 + - 0.21385789 + - -0.71504176 + - 0.5614625 + - -0.6167426 + - 0.46498674 + - 0.93407744 + - 0.14437051 + - -0.7163092 + - 0.116512164 + - -0.24489634 + - 0.645004 + - -0.32258278 + - 0.94063216 + - -0.36568317 + - 1.1295812 + - -0.8524073 + - 1.2329413 + - -0.4606274 + - 0.5174302 + - 0.52787626 + - -0.21639742 + - -0.549056 + - -0.33936682 + - -0.1768247 + - -0.16489355 + - 0.7124136 + - -0.41363987 + - 0.084636174 + - 0.71995646 + - -0.269404 + - 0.25011197 + - -0.72873056 + - -0.47306314 + - -0.92148453 + - 0.59367985 + - -0.17978935 + - 0.2729802 + - 0.5542758 + - -1.2200159 + - 1.729739 + - 0.0554798 + - 0.09480786 + - 0.52030283 + - 0.44673485 + - -1.1334461 + - 0.18587159 + - 0.38202813 + - 0.7951632 + - -0.53067535 + - -0.5831443 + - 0.21462004 + - 0.45758048 + - -0.42067245 + - -0.06907367 + - 0.91293555 + - 0.4624264 + - 0.32548374 + - 0.68481934 + - 0.15837803 + - -0.667224 + - 0.58356047 + - -1.0613954 + - 0.43691406 + - -0.59987056 + - -0.30487376 + - 0.007052924 + - -0.32386264 + - -0.05361281 + - -1.5188938 + - -0.5596059 + - 0.81915635 + - 0.20065632 + - 1.0555452 + - -0.3662693 + - 0.12234342 + - 0.199139 + - 0.3997875 + - 0.4250277 + - -0.78942347 + - -0.2393659 + - -0.9290193 + - 0.6526514 + - -0.47609806 + - -0.48884037 + - 0.13586058 + - -0.015904669 + - -0.2711965 + - -0.38438207 + - -0.91647637 + - 1.4105638 + - 0.06634407 + - -0.54394734 + - 0.19188683 + - -0.91326433 + - -0.068780616 + - 0.29199 + - 0.8736239 + - -0.6656744 + - 0.3295487 + - -0.706676 + - -0.7694737 + - -0.65125453 + - -0.3077479 + - -0.9317022 + - -0.32665482 + - 0.7258643 + - -0.7503613 + - 0.0829196 + - 0.58263254 + - -1.2264873 + - 0.057144012 + - 0.46637866 + - 0.769619 + - -0.5372799 + - -0.45179525 + - 0.10078682 + - -0.07869662 + - 0.009634495 + - -0.1755017 + - 0.0046813595 + - -0.14044896 + - -0.026245113 + - 0.40057 + - -0.30025095 + - 0.22953768 + - 0.13754176 + - 0.4285193 + - 0.3973975 + - -1.0153534 + - 0.14609443 + - -0.24231611 + - 0.8443168 + - 0.69656533 + - 0.048352692 + - -0.15647317 + - 0.4361965 + - 0.6241878 + - 0.016243378 + - -0.78102773 + - -0.010477662 + - 0.76305896 + - -0.3819266 + - -0.19829689 + - 0.80821335 + - -0.010607002 + - 0.6590781 + - -0.3427212 + - -0.0046394584 + - -0.37249613 + - -0.5596654 + - 0.62761366 + - -0.061553407 + - -0.36661023 + - 0.085924394 + - 0.6258331 + - -0.65754455 + - 0.29365936 + - -1.168679 + - 0.3876068 + - -0.15154646 + - -0.4680092 + - 0.8223411 + - 0.22256206 + - 0.39989498 + - 0.4739727 + - 0.3206084 + - -0.1219699 + - -0.3656821 + - 0.9146646 + - 0.16137339 + - 0.72202057 + - 0.006176436 + - -0.7444069 + - -0.48411816 + - 0.19201991 + - 0.047102835 + - 1.8374213 + - 0.6271216 + - -0.101507224 + - 0.33691677 + - -0.009013176 + - 0.12567987 + - -1.340964 + - -0.35267213 + - -0.23100884 + - -0.17414291 + - -1.0097998 + - -0.78583336 + - 0.85127574 + - -0.30711684 + - 0.546091 + - -0.09928613 + - 0.4857912 + - 0.2021114 + - -0.40408263 + - 0.26641545 + - -0.56869894 + - 0.18891306 + - 0.27093604 + - -0.5806636 + - 1.0632484 + - -0.3610749 + - 1.0024663 + - 0.05458 + - 0.5792243 + - -0.92720693 + - -1.0779785 + - 0.62059104 + - -0.2721579 + - 0.11107445 + - 0.0011022602 + - -0.10702686 + - 0.19780959 + - 0.072178856 + - -0.19438703 + - 0.69316137 + - 0.6943337 + - 0.57687074 + - -1.1014832 + - 0.42551953 + - -0.58607066 + - -0.51217395 + - 0.18606445 + - -0.17768076 + - 0.35027462 + - -0.44148698 + - -0.20854151 + - -0.14539146 + - 0.089815244 + - 0.17589141 + - 0.5609763 + - 0.24660146 + - 0.4190693 + - -0.09754421 + - 0.7212799 + - 0.2904666 + - -0.25766483 + - 0.4682985 + - -0.604066 + - 0.32014424 + - -1.0344561 + - -0.7218795 + - 0.037176847 + - -0.13975301 + - 0.057623085 + - -0.4861414 + - 0.84045273 + - 0.29917225 + - -0.123255536 + - -0.0783974 + - 0.9007332 + - 0.27021745 + - 0.3405612 + - -0.74217784 + - 0.980236 + - -0.16341434 + - -0.20316263 + - -0.023410704 + - -0.24500643 + - 0.4445403 + - 0.62363625 + - -0.43281606 + - -0.10284763 + - 0.35027084 + - -0.0092075085 + - 0.69164866 + - 0.1329943 + - -0.5267061 + - 1.0120076 + - 0.09034811 + - 0.6417969 + - -0.068846844 + - -0.30966592 + - -0.549686 + - 0.09696341 + - 0.1896585 + - -0.40163562 + - -0.071826085 + - 0.68074477 + - -0.20377605 + - 0.19715694 + - 0.90802956 + - 0.02462401 + - -0.2899885 + - 0.13454807 + - -0.40604225 + - 0.12468822 + - -0.051036604 + - 0.16812144 + - -0.6695822 + - 0.41169128 + - 0.041757975 + - 0.29515356 + - -0.5747506 + - -0.25879142 + - 1.1121366 + - 0.10842955 + - 0.36287335 + - -0.55900866 + - -0.03730956 + - 0.052817266 + - -0.11966578 + - -0.3027204 + - 0.5752762 + - 0.6632332 + - 0.42750373 + - 0.5033944 + - 0.21663067 + - 0.46812105 + - -0.49616817 + - 0.43859005 + - -0.48220006 + - -0.0030807555 + - 0.8108529 + - 0.25699314 + - -0.19481133 + - 0.90552133 + - -0.024103032 + - 0.31340557 + - 0.5684291 + - 0.2434076 + - 0.0887018 + - -0.077120006 + - 0.051219642 + - 0.4842207 + - -0.52892375 + - 0.2558231 + - 0.23758171 + - -0.34859946 + - 0.13730937 + - -0.16480811 + - 0.5783073 + - 0.17884485 + - -1.0146425 + - -0.6668196 + - 0.601889 + - 0.48329034 + - -0.71892196 + - -0.56138873 + - -0.043210823 + - -0.027440896 + - 0.5754655 + - -0.46964046 + - -0.4366773 + - -0.72764033 + - -0.7211243 + - -0.29358405 + - 0.0015932109 + - -0.5031633 + - -0.107117176 + - -1.0039937 + - -0.13279192 + - -0.70395243 + - -0.32447544 + - -0.76735896 + - -0.33175758 + - -0.015077961 + - -0.49323383 + - 1.0507339 + - 0.5211093 + - 0.25942016 + - 0.9989556 + - -0.36023825 + - 0.2752296 + - -0.45524743 + - -0.24631281 + - -0.015422097 + - -0.75090945 + - 0.07263755 + - 0.17731108 + - -0.51749355 + - 0.022234142 + - -0.022686418 + - 0.15864582 + - 0.6128289 + - 0.09107399 + - -0.76027334 + - -0.69433504 + - 0.728763 + - 0.11569237 + - 0.10708189 + - -0.5050128 + - -0.74649155 + - -0.0812074 + - -0.13342683 + - -0.803402 + - -0.04353934 + - 0.26879126 + - -0.13799225 + - -0.5837923 + - 0.10597635 + - 0.6506162 + - 0.13967924 + - 0.22376694 + - -0.20936067 + - -0.928748 + - -1.0964314 + - 0.40670997 + - -0.052957743 + - -0.09751829 + - -0.054456435 + - -0.119991586 + - -0.36898383 + - 0.3871018 + - -0.10110208 + - 0.60962784 + - -0.69676197 + - 0.46454045 + - -0.7093356 + - 1.3312905 + - 0.15540238 + - 0.04982522 + - 1.1623503 + - -0.7759637 + - -0.032434512 + - 0.28740007 + - 0.5178268 + - -0.21571222 + - -1.4859992 + - 0.1829937 + - -0.053918153 + - -0.36006457 + - -0.4752746 + - 0.205453 + - -0.05270856 + - -0.23025362 + - 0.9785634 + - 0.4197931 + - 0.31019348 + - 0.5481998 + - -0.038456056 + - -0.6436347 + - 0.41197804 + - -0.096051134 + - -0.5797923 + - -0.05929942 + - -0.3528435 + - -0.21341619 + - -0.21503787 + - 0.4439845 + - -0.21653461 + - -1.0152838 + - 0.14308062 + - 0.5781709 + - 0.6095633 + - 0.6356165 + - -0.5208993 + - -0.27639818 + - 0.9113302 + - -0.186468 + - -0.5261108 + - -0.9746019 + - -0.18573788 + - 0.7562442 + - -0.38231394 + - -1.1720121 + - 0.5825735 + - 0.40958115 + - -0.19192345 + - 0.3751216 + - -0.40434864 + - -0.2755611 + - -0.8410682 + - -0.06365899 + - 1.4388415 + - 0.13686097 + - 0.01997731 + - 0.17997758 + - -0.7766442 + - -0.4941811 + - -0.021396607 + - 0.3970068 + - -0.01249384 + - -0.6617173 + - 0.0026280028 + - 0.42203423 + - -0.36252597 + - -0.13971612 + - 0.299872 + - -0.67692715 + - -1.2585526 + - 0.59110105 + - 0.45884624 + - 0.13478602 + - 0.5914718 + - -0.07193293 + - -0.28348863 + - -0.0611645 + - -0.7110287 + - 0.12286919 + - 1.0284764 + - -0.5590409 + - 0.60044163 + - -0.48428342 + - -0.44081593 + - 0.26646346 + - 0.27263233 + - 0.44107154 + - -0.4993699 + - 0.070135005 + - -0.8562868 + - -0.09105264 + - -0.033029974 + - -0.99967617 + - 0.47625273 + - -0.07518111 + - -0.18114437 + - 0.64506567 + - 0.3145454 + - -0.14525138 + - -0.6040987 + - 0.6250855 + - 0.7729617 + - -0.5830136 + - 0.16460012 + - -0.3941438 + - 0.49310216 + - 0.6046754 + - -0.29815525 + - 1.5054228 + - 0.24868424 + - 0.75935614 + - 0.049756367 + - -0.22769414 + - -1.0260017 + - -0.18081093 + - 0.012250213 + - -0.79507744 + - -0.20898542 + - 0.8943629 + - 0.04096524 + - -0.26393303 + - 0.24376135 + - 1.0225416 + - -0.7895724 + - 0.7053045 + - -0.79659206 + - 0.21728578 + - -0.33226946 + - 0.8294584 + - -0.04298716 + - 0.6767645 + - 0.026965244 + - -0.07045885 + - 0.3374315 + - -0.36341807 + - 0.71740633 + - 0.27053788 + - -0.24694262 + - 1.031089 + - 0.0010727005 + - 0.8515307 + - 0.6787901 + - 0.2004985 + - -0.088589594 + - -0.7608832 + - 0.16530542 + - -0.060887106 + - -0.43441907 + - 0.27762964 + - 0.3499584 + - 0.0022304782 + - 0.7714241 + - -0.9295735 + - -0.28016114 + - -0.29263496 + - -0.22947809 + - 0.5989312 + - 0.69004786 + - 0.30413532 + - 0.058415506 + - -0.75590885 + - -0.40068144 + - -0.022342889 + - -0.8754985 + - -0.9253731 + - 1.2867957 + - -0.14284058 + - 0.6293072 + - -0.254001 + - -0.27964157 + - -0.56052685 + - 0.03074463 + - 0.43876895 + - -0.7460645 + - 0.8694146 + - 0.26861903 + - 0.3245183 + - -0.4265612 diff --git a/backends/candle/tests/test_flash_nomic.rs b/backends/candle/tests/test_flash_nomic.rs index ad45438e..bfb521a2 100644 --- a/backends/candle/tests/test_flash_nomic.rs +++ b/backends/candle/tests/test_flash_nomic.rs @@ -51,3 +51,48 @@ fn test_flash_nomic_small() -> Result<()> { Ok(()) } + +#[test] +#[serial_test::serial] +#[cfg(all(feature = "cuda", feature = "flash-attn"))] +fn test_flash_nomic_moe() -> Result<()> { + let model_root = download_artifacts("nomic-ai/nomic-embed-text-v2-moe", None)?; + let tokenizer = load_tokenizer(&model_root)?; + + let backend = CandleBackend::new( + &model_root, + "float16".to_string(), + ModelType::Embedding(Pool::Mean), + )?; + + let input_batch = batch( + vec![ + tokenizer.encode("What is Deep Learning?", true).unwrap(), + tokenizer.encode("Deep Learning is...", true).unwrap(), + tokenizer.encode("What is Deep Learning?", true).unwrap(), + ], + [0, 1, 2].to_vec(), + vec![], + ); + + let matcher = cosine_matcher(); + + let (pooled_embeddings, _) = sort_embeddings(backend.embed(input_batch)?); + let embeddings_batch = SnapshotEmbeddings::from(pooled_embeddings); + insta::assert_yaml_snapshot!("nomic_moe_batch", embeddings_batch, &matcher); + + let input_single = batch( + vec![tokenizer.encode("What is Deep Learning?", true).unwrap()], + [0].to_vec(), + vec![], + ); + + let (pooled_embeddings, _) = sort_embeddings(backend.embed(input_single)?); + let embeddings_single = SnapshotEmbeddings::from(pooled_embeddings); + + insta::assert_yaml_snapshot!("nomic_moe_single", embeddings_single, &matcher); + assert_eq!(embeddings_batch[0], embeddings_single[0]); + assert_eq!(embeddings_batch[2], embeddings_single[0]); + + Ok(()) +} diff --git a/backends/candle/tests/test_nomic.rs b/backends/candle/tests/test_nomic.rs index 6c444ea7..484e2fe6 100644 --- a/backends/candle/tests/test_nomic.rs +++ b/backends/candle/tests/test_nomic.rs @@ -48,3 +48,46 @@ fn test_nomic_small() -> Result<()> { Ok(()) } + +#[test] +fn test_nomic_moe() -> Result<()> { + let model_root = download_artifacts("nomic-ai/nomic-embed-text-v2-moe", None)?; + let tokenizer = load_tokenizer(&model_root)?; + + let backend = CandleBackend::new( + &model_root, + "float32".to_string(), + ModelType::Embedding(Pool::Mean), + )?; + + let input_batch = batch( + vec![ + tokenizer.encode("What is Deep Learning?", true).unwrap(), + tokenizer.encode("Deep Learning is...", true).unwrap(), + tokenizer.encode("What is Deep Learning?", true).unwrap(), + ], + [0, 1, 2].to_vec(), + vec![], + ); + + let matcher = cosine_matcher(); + + let (pooled_embeddings, _) = sort_embeddings(backend.embed(input_batch)?); + let embeddings_batch = SnapshotEmbeddings::from(pooled_embeddings); + insta::assert_yaml_snapshot!("nomic_moe_batch", embeddings_batch, &matcher); + + let input_single = batch( + vec![tokenizer.encode("What is Deep Learning?", true).unwrap()], + [0].to_vec(), + vec![], + ); + + let (pooled_embeddings, _) = sort_embeddings(backend.embed(input_single)?); + let embeddings_single = SnapshotEmbeddings::from(pooled_embeddings); + + insta::assert_yaml_snapshot!("nomic_moe_single", embeddings_single, &matcher); + assert_eq!(embeddings_batch[0], embeddings_single[0]); + assert_eq!(embeddings_batch[2], embeddings_single[0]); + + Ok(()) +}