how to expose parameters to the return object of load_dataset

[aihao2000]

When using custom dataset scrip, how to expose parameters to the return object of load_dataset? For example, when I use load dataset streaming, I hope to obtain num samples. I defined len in the custom category, but I cannot obtain it.

class Images(datasets.GeneratorBasedBuilder):

    def __init__(self, **kwargs):
        super(Images, self).__init__(**kwargs)

    def _split_generators(self, dl_manager: datasets.DownloadManager):
        data = []
        data_dir = f"{self.config.data_dir}/data/train"
        with jsonlines.open(f"{data_dir}/meta_data.jsonl") as reader:
            for obj in reader:
                data.append((obj["img_path"], obj["prompt"]))
        self.num_examples = len(data)
        return [
            datasets.SplitGenerator(
                name=datasets.Split.TRAIN,
                gen_kwargs={"split": datasets.Split.TRAIN, "data": data},
            )
        ]

    def __len__(self):
        return self.num_examples

    def _info(self):
        return datasets.DatasetInfo(
            features=datasets.Features(
                {
                    "reference_image": datasets.features.Image(),
                    "image": datasets.features.Image(),
                    "text": datasets.Value("string"),
                }
            )
        )

    def _generate_examples(self, split, data):
        data_dir = f"{self.config.data_dir}/data/train/imgs_unzip"
        for i, (image_path, prompt) in enumerate(data):
            image_path = f"{data_dir}/{image_path}"
            image = Image.open(image_path)
            if os.path.exists(image_path):
                yield i, {
                    "reference_image": image,
                    "image": image,
                    "text": prompt,
                }