updated crawler

.gitignore

@@ -7,4 +7,7 @@ crawler/target/
 *.DS_Store*
 package-lock.json
 package.json
-repositories.db
+repositories.db
+crawler/node_modules/
+crawler/zipped/*
+crawler/zipped/

.vscode/settings.json

@@ -1,5 +1,6 @@
 {
     "rust-analyzer.linkedProjects": [
         "./crawler/Cargo.toml"
-    ]
+    ],
+    "git.ignoreLimitWarning": true
 }

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "soldeer"
-version = "0.1.1"
+version = "0.1.2"
 edition = "2021"
 authors = ["m4rio"]
 license = "MIT"

crawler/.gitignore

@@ -0,0 +1 @@
+.gitattributes

crawler/Cargo.toml

@@ -13,4 +13,5 @@ serde_derive = "1.0.171"
 serde_json = "1.0.1"
 toml = "0.7.6"
 zip = "0.6.6"
-walkdir = "2.3.3"
+walkdir = "2.3.3"
+git2 = "0.17.2"

crawler/npm_repositories.toml

@@ -12,4 +12,8 @@
   "@prb/contracts",
   "@prb/proxy",
   "@prb/test",
+  "@gelatonetwork/relay-context",
+  "@axelar-network/axelar-cgp-solidity",
+  "@axelar-network/axelar-gmp-sdk-solidity",
+  "@axelar-network/interchain-token-service",
 ]

crawler/push_to_git.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+remote_url="[email protected]:mario-eth/soldeer.git"
+git_name="Soldeer CI"
+git_email="[email protected]"
+
+
+if [ -z "$1" ]; then
+    echo "Please provide a commit message as an argument."
+    exit 1
+fi
+
+if [ -z "$2" ]; then
+    echo "Please provide a key"
+    exit 1
+fi
+
+
+# Start ssh-agent and add your key
+eval $(ssh-agent)
+ssh-add "$2"
+
+cd zipped && \
+git lfs track "*.zip" && \
+git init && \
+(git remote get-url origin > /dev/null 2>&1 || git remote add origin "$remote_url") && \
+git fetch origin && \
+git config user.name "$git_name" && \
+git config user.email "$git_email" && \
+git checkout versions && \ 
+git add . && \
+git commit -m "$1" && \
+git push -u origin versions
+
+# Kill the ssh-agent after use
+ssh-agent -k

crawler/src/main.rs

@@ -8,7 +8,7 @@ use npm::LoadError;
 use db::{ get_versions_for_repo_from_db, insert_version_into_db, Version };
 use rusqlite::Error;
 use chrono::Utc;
-use manager::{ zip_version, push_to_repository };
+use manager::{ zip_version, push_to_repository, clean };
 fn main() {
     let repositories: Vec<String> = load_repositories()
         .map_err(|err: LoadError| {
@@ -28,16 +28,23 @@ fn main() {
             })
             .unwrap();
 
+        let mut index: usize = 0;
+        let versions_len: usize = versions.len();
         for version in versions {
+            index = index + 1;
             if existing_versions.contains(&version) {
                 continue;
             }
-            retrieve_version(&repository, &version)
-                .map_err(|err| {
-                    println!("{:?}", err);
-                })
-                .unwrap();
-            let version_to_insert = Version {
+            match retrieve_version(&repository, &version) {
+                Ok(_) => {}
+                Err(_) => {
+                    if versions_len == index {
+                        clean();
+                    }
+                    continue;
+                }
+            }
+            let version_to_insert: Version = Version {
                 repository: repository.clone(),
                 version: version.clone(),
                 last_updated: Utc::now(),
@@ -51,6 +58,7 @@ fn main() {
                     println!("{:?}", err);
                 })
                 .unwrap();
+            clean();
         }
     }
 }