veld_preprocess_sample.yaml

x-veld:
  code:
    description: "takes a random sample of lines from a txt file. Randomness can be set with a seed
      too"
    topics: 
      - "NLP"
      - "preprocessing"
      - "ETL"

    inputs:
      - volume: /veld/input/
        environment: in_txt_file
        file_type: "txt"
        contents: "raw text"

    outputs:
      - volume: /veld/output/
        environment: out_txt_file 
        file_type: "txt"
        contents: "raw text"

    settings:
      - environment: out_data_description
        description: "automatic data description for generating a data veld yaml file"
        env_type: "str" 
      - environment: percentage_sample
        description: "percentage of lines to be randomly sampled"
        env_type: "int"
      - environment: sample_random_seed
        description: "seed to make randomness stable and reproducible"
        env_type: "str"
      - environment: buffer_segments
        description: "percentage of segments where processing results are persisted in between. So
          that processing could continue should it have crashed"
        env_type: "int"
        default: 100

services:
  veld_preprocess_sample:
    build: .
    volumes:
      - ./src/:/veld/code/:z
    command: python3 /veld/code/preprocess_sample.py
    environment:
      in_file: null
      out_file: null
      out_data_description: null
      percentage_sample: null
      sample_random_seed: null
      buffer_segments: 100