Help Needed with Dataset Preparation and Fine-Tuning Process

[mohamed2020m]

Hi everyone,

I’m trying to fine-tune the model and have been following their paper for guidance. While I understand the dataset originates from Objaverse, I still have several questions about the setup and process:

1. Where can I find logs/instant-nerf-large-train/checkpoints/last.ckpt? Is it a pretrained checkpoint provided somewhere, or do we need to train it from scratch?
2. I believe filtered_obj_name.json is generated during the preprocessing stage of the Objaverse dataset. Does anyone know how this preprocessing is done, or where the relevant script might be?
3. What exactly is rendering_random_32views? Is it a standard rendering of the dataset, or does it require a specific rendering pipeline?

For reference, here’s the configuration file from the config/configs/instant-mesh-large-train.yaml

model:
  base_learning_rate: 4.0e-05
  target: src.model_mesh.MVRecon
  params:
    init_ckpt: logs/instant-nerf-large-train/checkpoints/last.ckpt
    input_size: 320
    render_size: 512
    
    lrm_generator_config:
      target: src.models.lrm_mesh.InstantMesh
      params:
        encoder_feat_dim: 768
        encoder_freeze: false
        encoder_model_name: facebook/dino-vitb16
        transformer_dim: 1024
        transformer_layers: 16
        transformer_heads: 16
        triplane_low_res: 32
        triplane_high_res: 64
        triplane_dim: 80
        rendering_samples_per_ray: 128
        grid_res: 128
        grid_scale: 2.1


data:
  target: src.data.objaverse.DataModuleFromConfig
  params:
    batch_size: 2
    num_workers: 8
    train:
      target: src.data.objaverse.ObjaverseData
      params:
        root_dir: data/objaverse
        meta_fname: filtered_obj_name.json
        input_image_dir: rendering_random_32views
        target_image_dir: rendering_random_32views
        input_view_num: 6
        target_view_num: 4
        total_view_n: 32
        fov: 50
        camera_rotation: true
        validation: false
    validation:
      target: src.data.objaverse.ValidationData
      params:
        root_dir: data/valid_samples
        input_view_num: 6
        input_image_size: 320
        fov: 30


lightning:
  modelcheckpoint:
    params:
      every_n_train_steps: 2000
      save_top_k: -1
      save_last: true
  callbacks: {}

  trainer:
    benchmark: true
    max_epochs: -1
    val_check_interval: 1000
    num_sanity_val_steps: 0
    accumulate_grad_batches: 1
    check_val_every_n_epoch: null   # if not set this, validation does not run