Skip to content

Commit bcaef89

Browse files
authored
Expose flag truncate-ragged-lines in polars open (#13939)
# Description Introduces a new flag `--truncate-ragged-lines` for `polars open` that will truncate lines that are longer than the schema. # User-Facing Changes - Introduction of the flag `--truncate-ragged-lines` for `polars open`
1 parent 5bef81a commit bcaef89

File tree

1 file changed

+8
-9
lines changed
  • crates/nu_plugin_polars/src/dataframe/command/core

1 file changed

+8
-9
lines changed

crates/nu_plugin_polars/src/dataframe/command/core/open.rs

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ impl PluginCommand for OpenDataFrame {
9797
r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
9898
Some('s')
9999
)
100+
.switch("truncate-ragged-lines", "Truncate lines that are longer than the schema. CSV file", None)
100101
.input_output_type(Type::Any, Type::Custom("dataframe".into()))
101102
.category(Category::Custom("dataframe".into()))
102103
}
@@ -466,11 +467,11 @@ fn from_csv(
466467
.unwrap_or(DEFAULT_INFER_SCHEMA);
467468
let skip_rows: Option<usize> = call.get_flag("skip-rows")?;
468469
let columns: Option<Vec<String>> = call.get_flag("columns")?;
469-
470470
let maybe_schema = call
471471
.get_flag("schema")?
472472
.map(|schema| NuSchema::try_from(&schema))
473473
.transpose()?;
474+
let truncate_ragged_lines: bool = call.has_flag("truncate-ragged-lines")?;
474475

475476
if !call.has_flag("eager")? {
476477
let csv_reader = LazyCsvReader::new(file_path);
@@ -496,14 +497,11 @@ fn from_csv(
496497
}
497498
};
498499

499-
let csv_reader = csv_reader.with_has_header(!no_header);
500-
501-
let csv_reader = match maybe_schema {
502-
Some(schema) => csv_reader.with_schema(Some(schema.into())),
503-
None => csv_reader,
504-
};
505-
506-
let csv_reader = csv_reader.with_infer_schema_length(Some(infer_schema));
500+
let csv_reader = csv_reader
501+
.with_has_header(!no_header)
502+
.with_infer_schema_length(Some(infer_schema))
503+
.with_schema(maybe_schema.map(Into::into))
504+
.with_truncate_ragged_lines(truncate_ragged_lines);
507505

508506
let csv_reader = match skip_rows {
509507
None => csv_reader,
@@ -542,6 +540,7 @@ fn from_csv(
542540
.unwrap_or(b','),
543541
)
544542
.with_encoding(CsvEncoding::LossyUtf8)
543+
.with_truncate_ragged_lines(truncate_ragged_lines)
545544
})
546545
.try_into_reader_with_file_path(Some(file_path.to_path_buf()))
547546
.map_err(|e| ShellError::GenericError {

0 commit comments

Comments
 (0)