fetch_models.js

import fetch from "node-fetch";
import fs from "fs";
import path from "path";
import ProgressBar from "progress";
import { fileURLToPath } from "url";
import { dirname } from "path";
// Uncomment the following line if proxy is required to fetch files in your network
// import { HttpsProxyAgent } from 'https-proxy-agent';

const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

const models = [
    {
        url: "xenova/resnet-50/resolve/main/onnx/model_fp16.onnx",
        path: "./demos/image-classification/models/xenova/resnet-50/onnx",
    },
    {
        url: "webnn/mobilenet-v2/resolve/main/onnx/model_fp16.onnx",
        path: "./demos/image-classification/models/webnn/mobilenet-v2/onnx",
    },
    {
        url: "webnn/efficientnet-lite4/resolve/main/onnx/model_fp16.onnx",
        path: "./demos/image-classification/models/webnn/efficientnet-lite4/onnx",
    },
    {
        url: "microsoft/sd-turbo-webnn/resolve/main/text_encoder/model_layernorm.onnx",
        path: "./demos/sd-turbo/models/text_encoder",
    },
    {
        url: "microsoft/sd-turbo-webnn/resolve/main/unet/model_layernorm.onnx",
        path: "./demos/sd-turbo/models/unet",
    },
    {
        url: "microsoft/sd-turbo-webnn/resolve/main/vae_decoder/model.onnx",
        path: "./demos/sd-turbo/models/vae_decoder",
    },
    {
        url: "microsoft/sd-turbo-webnn/resolve/main/safety_checker/safety_checker_int32_reduceSum.onnx",
        path: "./demos/sd-turbo/models/safety_checker",
    },
    {
        url: "microsoft/stable-diffusion-v1.5-webnn/resolve/main/text-encoder.onnx",
        path: "./demos/stable-diffusion-1.5/models",
    },
    {
        url: "microsoft/stable-diffusion-v1.5-webnn/resolve/main/sd-unet-v1.5-model-b2c4h64w64s77-float16-compute-and-inputs-layernorm.onnx",
        path: "./demos/stable-diffusion-1.5/models",
    },
    {
        url: "microsoft/stable-diffusion-v1.5-webnn/resolve/main/Stable-Diffusion-v1.5-vae-decoder-float16-fp32-instancenorm.onnx",
        path: "./demos/stable-diffusion-1.5/models",
    },
    {
        url: "microsoft/stable-diffusion-v1.5-webnn/resolve/main/safety_checker_int32_reduceSum.onnx",
        path: "./demos/stable-diffusion-1.5/models",
    },
    {
        url: "microsoft/segment-anything-model-webnn/resolve/main/sam_vit_b_01ec64.encoder-fp16.onnx",
        path: "./demos/segment-anything/models",
    },
    {
        url: "microsoft/segment-anything-model-webnn/resolve/main/sam_vit_b_01ec64.decoder-fp16.onnx",
        path: "./demos/segment-anything/models",
    },
    {
        url: "microsoft/segment-anything-model-webnn/resolve/main/sam_vit_b-encoder-int8.onnx",
        path: "./demos/segment-anything/models",
    },
    {
        url: "microsoft/segment-anything-model-webnn/resolve/main/sam_vit_b-decoder-int8.onnx",
        path: "./demos/segment-anything/models",
    },
    {
        url: "microsoft/whisper-base-webnn/resolve/main/whisper_base_decoder_static_kvcache_128_lm_fp16_layernorm_gelu_4dmask.onnx",
        path: "./demos/whisper-base/models",
        note: "Gelu and 4D mask are used for decoder",
    },
    {
        url: "microsoft/whisper-base-webnn/resolve/main/whisper_base_decoder_static_non_kvcache_lm_fp16_layernorm_gelu_4dmask.onnx",
        path: "./demos/whisper-base/models",
        note: "Gelu and 4D mask are used for decoder",
    },
    {
        url: "microsoft/whisper-base-webnn/resolve/main/whisper_base_encoder_lm_fp16_layernorm_gelu.onnx",
        path: "./demos/whisper-base/models",
        note: "Gelu is used for encoder",
    },
];

const downloadFile = async (url, outputPath, retries = 2) => {
    // Proxy configuration
    // const proxyHost = '';
    // const proxyPort = 8080;
    // const proxyUrl = `http://${proxyHost}:${proxyPort}`;
    const proxyAgent = undefined;
    // Enable this line to use the proxy:
    // const proxyAgent = new HttpsProxyAgent(proxyUrl);

    try {
        const response = await fetch(url, { agent: proxyAgent });
        if (response.ok) {
            const totalLength = response.headers.get("content-length");
            const progressBar = new ProgressBar("-> downloading [:bar] :percent :etas", {
                width: 40,
                complete: "=",
                incomplete: " ",
                renderThrottle: 1,
                total: parseInt(totalLength),
            });

            const fileStream = fs.createWriteStream(outputPath);
            response.body.on("data", chunk => progressBar.tick(chunk.length));
            response.body.pipe(fileStream);

            return new Promise((resolve, reject) => {
                fileStream.on("finish", resolve);
                fileStream.on("error", reject);
            });
        } else {
            throw new Error(`HTTP error! Status: ${response.status} ${response.statusText}`);
        }
    } catch (error) {
        if (retries === 0) {
            console.error(`Failed to download ${url} after multiple attempts`, error);
            throw error;
        } else {
            console.warn(`Retrying download for ${url} (${retries} retries left)`);
            return downloadFile(url, outputPath, retries - 1);
        }
    }
};

(async () => {
    let i = 1;
    for (const model of models) {
        const outputPath = path.join(__dirname, model.path, path.basename(model.url));
        fs.mkdirSync(path.dirname(outputPath), { recursive: true });
        console.log(`[${i}/${models.length}] Downloading https://huggingface.co/${model.url} to ${outputPath}`);
        try {
            await downloadFile(`https://huggingface.co/${model.url}`, outputPath);
            console.log(`[${i}/${models.length}] Downloaded https://huggingface.co/${model.url} to ${outputPath}`);
        } catch (error) {
            console.error(`Failed to download https://huggingface.co/${model.url}`, error);
        }
        i++;
    }
})();