Skip to content

Commit b750d77

Browse files
guilloadguilload
committed
Use version 0.9 for versioning object
1 parent 6c3159e commit b750d77

File tree

30 files changed

+299
-318
lines changed

30 files changed

+299
-318
lines changed

‎docs/reference/rest-api.md

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,7 @@ curl -XPOST http://localhost:7280/api/v1/indexes --data @index_config.json -H "C
257257

258258
```json title="index_config.json
259259
{
260-
"version": "0.8",
260+
"version": "0.9",
261261
"index_id": "hdfs-logs",
262262
"doc_mapping": {
263263
"field_mappings": [
@@ -355,7 +355,7 @@ curl -XPUT http://localhost:7280/api/v1/indexes/hdfs-logs --data @updated_index_
355355

356356
```json title="updated_index_update.json
357357
{
358-
"version": "0.8",
358+
"version": "0.9",
359359
"index_id": "hdfs-logs",
360360
"doc_mapping": {
361361
"field_mappings": [
@@ -513,7 +513,7 @@ GET /api/v1/indexes/stackoverflow/splits?offset=0&limit=10
513513
"split_state": "Published",
514514
"update_timestamp": 1695642901,
515515
"publish_timestamp": 1695642901,
516-
"version": "0.7",
516+
"version": "0.9",
517517
"split_id": "01HB632HD8W6WHNM7CZFH3KG1X",
518518
"index_uid": "stackoverflow:01HB6321TDT3SP58D4EZP14KSX",
519519
"partition_id": 0,
@@ -617,7 +617,7 @@ curl -XPOST http://localhost:7280/api/v1/indexes/my-index/sources --data @source
617617

618618
```json title="source_config.json
619619
{
620-
"version": "0.8",
620+
"version": "0.9",
621621
"source_id": "kafka-source",
622622
"source_type": "kafka",
623623
"params": {
@@ -664,7 +664,7 @@ curl -XPOST http://localhost:7280/api/v1/indexes/my-index/sources --data @source
664664

665665
```json title="source_config.json
666666
{
667-
"version": "0.8",
667+
"version": "0.9",
668668
"source_id": "kafka-source",
669669
"source_type": "kafka",
670670
"params": {

‎quickwit/quickwit-cli/src/cli_doc_ext.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ quickwit run --service metastore --config=./config/quickwit.yaml
232232
# Open a new terminal and run:
233233
cat << EOF > wikipedia-kafka-source.json
234234
{
235-
"version": "0.7",
235+
"version": "0.9",
236236
"source_id": "kafka-source",
237237
"source_type": "kafka",
238238
"params": {

‎quickwit/quickwit-config/resources/tests/index_config/hdfs-logs-create-config.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version: 0.8
1+
version: 0.9
22

33
doc_mapping:
44
field_mappings:

‎quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Comments are supported.
22
{
3-
"version": "0.7",
3+
"version": "0.9",
44
"index_id": "hdfs-logs",
55
"index_uri": "s3://quickwit-indexes/hdfs-logs",
66
"doc_mapping": {

‎quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version = "0.7"
1+
version = "0.9"
22
index_id = "hdfs-logs"
33
index_uri = "s3://quickwit-indexes/hdfs-logs"
44

‎quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version: 0.8
1+
version: 0.9
22
index_id: hdfs-logs
33
index_uri: s3://quickwit-indexes/hdfs-logs
44

‎quickwit/quickwit-config/resources/tests/index_config/minimal-hdfs-logs.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version: 0.8
1+
version: 0.9
22

33
index_id: hdfs-logs
44
index_uri: s3://quickwit-indexes/hdfs-logs

‎quickwit/quickwit-config/resources/tests/index_config/partial-hdfs-logs.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version: 0.8
1+
version: 0.9
22

33
index_id: hdfs-logs
44
index_uri: s3://quickwit-indexes/hdfs-logs

‎quickwit/quickwit-config/resources/tests/node_config/quickwit.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Comments are supported.
22
{
3-
"version": "0.7",
3+
"version": "0.9",
44
"cluster_id": "quickwit-cluster",
55
"node_id": "my-unique-node-id",
66
"enabled_services": [

‎quickwit/quickwit-config/resources/tests/node_config/quickwit.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version = "0.7"
1+
version = "0.9"
22

33
cluster_id = "quickwit-cluster"
44
node_id = "my-unique-node-id"

‎quickwit/quickwit-config/resources/tests/node_config/quickwit.wrongkey.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version: 0.8
1+
version: 0.9
22
searcher:
33
fast_field_cache_capacity: 10G
44
# Typo here. It is supposed to be searches.

‎quickwit/quickwit-config/resources/tests/node_config/quickwit.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version: 0.8
1+
version: 0.9
22

33
cluster_id: quickwit-cluster
44
node_id: my-unique-node-id

‎quickwit/quickwit-config/resources/tests/source_config/ingest-api-source.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": "0.7",
2+
"version": "0.9",
33
"source_id": "_ingest-api-source",
44
"enabled": true,
55
"source_type": "ingest-api",

‎quickwit/quickwit-config/resources/tests/source_config/kafka-source.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
2-
"version": "0.7",
2+
"version": "0.9",
33
"source_id": "hdfs-logs-kafka-source",
4-
"desired_num_pipelines": 2,
4+
"num_pipelines": 2,
55
"source_type": "kafka",
66
"params": {
77
"topic": "cloudera-cluster-logs",

‎quickwit/quickwit-config/resources/tests/source_config/kinesis-source.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version: 0.8
1+
version: 0.9
22
source_id: hdfs-logs-kinesis-source
33
source_type: kinesis
44
params:

‎quickwit/quickwit-config/src/lib.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ pub use crate::node_config::{
7474
IndexerConfig, IngestApiConfig, JaegerConfig, NodeConfig, SearcherConfig, SplitCacheLimits,
7575
StorageTimeoutPolicy, DEFAULT_QW_CONFIG_PATH,
7676
};
77-
use crate::source_config::serialize::{SourceConfigV0_7, SourceConfigV0_8, VersionedSourceConfig};
77+
use crate::source_config::serialize::{SourceConfigV0_7, SourceConfigV0_9, VersionedSourceConfig};
7878
pub use crate::storage_config::{
7979
AzureStorageConfig, FileStorageConfig, GoogleCloudStorageConfig, RamStorageConfig,
8080
S3StorageConfig, StorageBackend, StorageBackendFlavor, StorageConfig, StorageConfigs,
@@ -104,7 +104,7 @@ pub fn disable_ingest_v1() -> bool {
104104
DocMapping,
105105
VersionedSourceConfig,
106106
SourceConfigV0_7,
107-
SourceConfigV0_8,
107+
SourceConfigV0_9,
108108
VersionedIndexConfig,
109109
IndexConfigV0_8,
110110
VersionedIndexTemplate,

‎quickwit/quickwit-config/src/node_config/serialize.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,16 +145,16 @@ pub async fn load_node_config_with_env(
145145
#[derive(Debug, Deserialize)]
146146
#[serde(tag = "version")]
147147
enum VersionedNodeConfig {
148-
#[serde(rename = "0.8")]
148+
#[serde(rename = "0.9")]
149149
// Retro compatibility.
150-
#[serde(alias = "0.7")]
151-
V0_8(NodeConfigBuilder),
150+
#[serde(alias = "0.7", alias = "0.8")]
151+
V0_9(NodeConfigBuilder),
152152
}
153153

154154
impl From<VersionedNodeConfig> for NodeConfigBuilder {
155155
fn from(versioned_node_config: VersionedNodeConfig) -> Self {
156156
match versioned_node_config {
157-
VersionedNodeConfig::V0_8(node_config_builder) => node_config_builder,
157+
VersionedNodeConfig::V0_9(node_config_builder) => node_config_builder,
158158
}
159159
}
160160
}

‎quickwit/quickwit-config/src/source_config/mod.rs

Lines changed: 10 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -861,41 +861,24 @@ mod tests {
861861
{
862862
let content = r#"
863863
{
864-
"version": "0.7",
864+
"version": "0.9",
865865
"source_id": "hdfs-logs-void-source",
866-
"desired_num_pipelines": 0,
867-
"max_num_pipelines_per_indexer": 1,
866+
"num_pipelines": 0,
868867
"source_type": "void",
869868
"params": {}
870869
}
871870
"#;
872871
let error = load_source_config_from_user_config(ConfigFormat::Json, content.as_bytes())
873872
.unwrap_err();
873+
println!("error: {error}");
874874
assert!(error
875875
.to_string()
876-
.contains("`desired_num_pipelines` must be"));
876+
.contains("`num_pipelines` must be"));
877877
}
878-
// {
879-
// let content = r#"
880-
// {
881-
// "version": "0.7",
882-
// "source_id": "hdfs-logs-void-source",
883-
// "desired_num_pipelines": 1,
884-
// "max_num_pipelines_per_indexer": 0,
885-
// "source_type": "void",
886-
// "params": {}
887-
// }
888-
// "#;
889-
// let error = load_source_config_from_user_config(ConfigFormat::Json,
890-
// content.as_bytes()) .unwrap_err();
891-
// assert!(error
892-
// .to_string()
893-
// .contains("`max_num_pipelines_per_indexer` must be"));
894-
// }
895878
{
896879
let content = r#"
897880
{
898-
"version": "0.8",
881+
"version": "0.9",
899882
"source_id": "hdfs-logs-void-source",
900883
"num_pipelines": 2,
901884
"source_type": "void",
@@ -909,10 +892,9 @@ mod tests {
909892
{
910893
let content = r#"
911894
{
912-
"version": "0.7",
895+
"version": "0.9",
913896
"source_id": "hdfs-logs-void-source",
914-
"desired_num_pipelines": 2,
915-
"max_num_pipelines_per_indexer": 1,
897+
"num_pipelines": 2,
916898
"source_type": "void",
917899
"params": {}
918900
}
@@ -970,7 +952,7 @@ mod tests {
970952
{
971953
let content = r#"
972954
{
973-
"version": "0.8",
955+
"version": "0.9",
974956
"source_id": "hdfs-logs-kafka-source",
975957
"num_pipelines": 3,
976958
"source_type": "kafka",
@@ -1461,10 +1443,9 @@ mod tests {
14611443
#[tokio::test]
14621444
async fn test_source_config_plain_text_input_format() {
14631445
let file_content = r#"{
1464-
"version": "0.7",
1446+
"version": "0.9",
14651447
"source_id": "logs-file-source",
1466-
"desired_num_pipelines": 1,
1467-
"max_num_pipelines_per_indexer": 1,
1448+
"num_pipelines": 1,
14681449
"source_type": "file",
14691450
"params": {
14701451
"filepath": "s3://mybucket/test_non_json_corpus.txt"

‎quickwit/quickwit-config/src/source_config/serialize.rs

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,15 @@ use crate::{
2424
SourceParams,
2525
};
2626

27-
type SourceConfigForSerialization = SourceConfigV0_8;
27+
type SourceConfigForSerialization = SourceConfigV0_9;
2828

2929
#[derive(Serialize, Deserialize, utoipa::ToSchema)]
3030
#[serde(deny_unknown_fields)]
3131
#[serde(tag = "version")]
3232
pub enum VersionedSourceConfig {
3333
#[serde(rename = "0.9")]
3434
#[serde(alias = "0.8")]
35-
V0_8(SourceConfigV0_8),
35+
V0_9(SourceConfigV0_9),
3636
// Retro compatibility.
3737
#[serde(rename = "0.7")]
3838
V0_7(SourceConfigV0_7),
@@ -42,7 +42,7 @@ impl From<VersionedSourceConfig> for SourceConfigForSerialization {
4242
fn from(versioned_source_config: VersionedSourceConfig) -> Self {
4343
match versioned_source_config {
4444
VersionedSourceConfig::V0_7(v0_7) => v0_7.into(),
45-
VersionedSourceConfig::V0_8(v0_8) => v0_8,
45+
VersionedSourceConfig::V0_9(v0_9) => v0_9,
4646
}
4747
}
4848
}
@@ -102,7 +102,7 @@ impl SourceConfigForSerialization {
102102
validate_identifier("source", &self.source_id)?;
103103
}
104104
let num_pipelines = NonZeroUsize::new(self.num_pipelines)
105-
.ok_or_else(|| anyhow::anyhow!("`desired_num_pipelines` must be strictly positive"))?;
105+
.ok_or_else(|| anyhow::anyhow!("`num_pipelines` must be strictly positive"))?;
106106
match &self.source_params {
107107
SourceParams::Stdin => {
108108
bail!(
@@ -158,9 +158,9 @@ impl SourceConfigForSerialization {
158158
}
159159
}
160160

161-
impl From<SourceConfig> for SourceConfigV0_8 {
161+
impl From<SourceConfig> for SourceConfigV0_9 {
162162
fn from(source_config: SourceConfig) -> Self {
163-
SourceConfigV0_8 {
163+
SourceConfigV0_9 {
164164
source_id: source_config.source_id,
165165
num_pipelines: source_config.num_pipelines.get(),
166166
enabled: source_config.enabled,
@@ -173,15 +173,15 @@ impl From<SourceConfig> for SourceConfigV0_8 {
173173

174174
impl From<SourceConfig> for VersionedSourceConfig {
175175
fn from(source_config: SourceConfig) -> Self {
176-
VersionedSourceConfig::V0_8(source_config.into())
176+
VersionedSourceConfig::V0_9(source_config.into())
177177
}
178178
}
179179

180180
impl TryFrom<VersionedSourceConfig> for SourceConfig {
181181
type Error = anyhow::Error;
182182

183183
fn try_from(versioned_source_config: VersionedSourceConfig) -> anyhow::Result<Self> {
184-
let v1: SourceConfigV0_8 = versioned_source_config.into();
184+
let v1: SourceConfigV0_9 = versioned_source_config.into();
185185
v1.validate_and_build()
186186
}
187187
}
@@ -230,7 +230,7 @@ pub struct SourceConfigV0_7 {
230230

231231
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, utoipa::ToSchema)]
232232
#[serde(deny_unknown_fields)]
233-
pub struct SourceConfigV0_8 {
233+
pub struct SourceConfigV0_9 {
234234
#[schema(value_type = String)]
235235
pub source_id: SourceId,
236236

@@ -252,7 +252,7 @@ pub struct SourceConfigV0_8 {
252252
pub input_format: SourceInputFormat,
253253
}
254254

255-
impl From<SourceConfigV0_7> for SourceConfigV0_8 {
255+
impl From<SourceConfigV0_7> for SourceConfigV0_9 {
256256
fn from(source_config_v0_7: SourceConfigV0_7) -> Self {
257257
let SourceConfigV0_7 {
258258
source_id,
@@ -263,7 +263,7 @@ impl From<SourceConfigV0_7> for SourceConfigV0_8 {
263263
transform,
264264
input_format,
265265
} = source_config_v0_7;
266-
SourceConfigV0_8 {
266+
SourceConfigV0_9 {
267267
source_id,
268268
num_pipelines: desired_num_pipelines,
269269
enabled,

‎quickwit/quickwit-integration-tests/src/tests/update_tests/restart_indexer_tests.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ async fn test_update_doc_mapping_restart_indexing_pipeline() {
6464
.indexes()
6565
.create(
6666
json!({
67-
"version": "0.8",
67+
"version": "0.9",
6868
"index_id": index_id,
6969
"doc_mapping": {
7070
"doc_mapping_uid": initial_mapping_uid,
@@ -141,7 +141,7 @@ async fn test_update_doc_mapping_restart_indexing_pipeline() {
141141
.update(
142142
index_id,
143143
json!({
144-
"version": "0.8",
144+
"version": "0.9",
145145
"index_id": index_id,
146146
"doc_mapping": {
147147
"doc_mapping_uid": final_mapping_uid,

0 commit comments

Comments
 (0)