Support sparseCheckout (#123)

* support sparseCheckout * add some workaround for fetchFromGitHub * add a sparseCheckout example * fix git test
berberman · Jun 13, 2024 · fa76099 · fa76099
1 parent 2a82432
commit fa76099
Show file tree

Hide file tree

Showing 6 changed files with 39 additions and 17 deletions.
diff --git a/app/Config/PackageFetcher.hs b/app/Config/PackageFetcher.hs
@@ -50,7 +50,8 @@ fetcherKeys =
 data GitOptions = GitOptions
   { goDeepClone :: Maybe Bool,
     goFetchSubmodules :: Maybe Bool,
-    goLeaveDotGit :: Maybe Bool
+    goLeaveDotGit :: Maybe Bool,
+    goSparseCheckout :: Maybe [Text]
   }
   deriving (Eq, Generic)
 
@@ -60,6 +61,7 @@ gitOptionsDecoder =
     <$> getFieldsOpt ["git", "deepClone"]
     <*> getFieldsOpt ["git", "fetchSubmodules"]
     <*> getFieldsOpt ["git", "leaveDotGit"]
+    <*> getFieldsOpt ["git", "sparseCheckout"]
 
 _GitOptions :: Traversal' (NixFetcher f) GitOptions
 _GitOptions f x@FetchGit {..} =
@@ -68,16 +70,18 @@ _GitOptions f x@FetchGit {..} =
         & deepClone .~ fromMaybe False goDeepClone
         & fetchSubmodules .~ fromMaybe False goFetchSubmodules
         & leaveDotGit .~ fromMaybe False goLeaveDotGit
+        & sparseCheckout .~ fromMaybe [] goSparseCheckout
   )
-    <$> f (GitOptions (Just _deepClone) (Just _fetchSubmodules) (Just _leaveDotGit))
+    <$> f (GitOptions (Just _deepClone) (Just _fetchSubmodules) (Just _leaveDotGit) (Just _sparseCheckout))
 _GitOptions f x@FetchGitHub {..} =
   ( \GitOptions {..} ->
       x
         & deepClone .~ fromMaybe False goDeepClone
         & fetchSubmodules .~ fromMaybe False goFetchSubmodules
         & leaveDotGit .~ fromMaybe False goLeaveDotGit
+        & sparseCheckout .~ fromMaybe [] goSparseCheckout
   )
-    <$> f (GitOptions (Just _deepClone) (Just _fetchSubmodules) (Just _leaveDotGit))
+    <$> f (GitOptions (Just _deepClone) (Just _fetchSubmodules) (Just _leaveDotGit) (Just _sparseCheckout))
 _GitOptions _ x = pure x
 
 --------------------------------------------------------------------------------

diff --git a/nvfetcher_example.toml b/nvfetcher_example.toml
@@ -68,3 +68,9 @@ fetch.url = "https://files.yande.re/image/3fc51f6a2fb10c96b73dd0fce6ddb9c8/yande
 src.manual = "latest"
 # Override the name of the file in the Nix store
 url.name = "wallpaper.jpg"
+
+# To demonstrate how to use `sparseCheckout` option
+[noto-fonts-cjk-sans-fix-weight]
+src.manual = "Sans2.004"
+fetch.github = "notofonts/noto-cjk"
+git.sparseCheckout = [ "Sans/OTC" ]
diff --git a/src/NvFetcher/NixExpr.hs b/src/NvFetcher/NixExpr.hs
@@ -67,6 +67,7 @@ nixFetcher = \case
       _fetchSubmodules = toNixExpr -> fetchSubmodules,
       _deepClone = toNixExpr -> deepClone,
       _leaveDotGit = toNixExpr -> leaveDotGit,
+      _sparseCheckout = toNixExpr . map quote -> sparseCheckout,
       _furl = quote -> url,
       _name = nameField -> n
     } ->
@@ -76,7 +77,8 @@ nixFetcher = \case
             rev = $rev;
             fetchSubmodules = $fetchSubmodules;
             deepClone = $deepClone;
-            leaveDotGit = $leaveDotGit;$n
+            leaveDotGit = $leaveDotGit;
+            sparseCheckout = $sparseCheckout;$n
             sha256 = $sha256;
           }
     |]
@@ -86,13 +88,14 @@ nixFetcher = \case
       _fetchSubmodules = toNixExpr -> fetchSubmodules,
       _deepClone = toNixExpr -> deepClone,
       _leaveDotGit = toNixExpr -> leaveDotGit,
+      _sparseCheckout = toNixExpr . map quote -> sparseCheckout,
       _fowner = quote -> owner,
       _frepo = quote -> repo,
       _name = nameField -> n
     } ->
-      -- TODO: fix fetchFromGitHub in Nixpkgs so that deepClone and
-      -- leaveDotGit won't get passed to fetchzip
-      if (deepClone == "true") || (leaveDotGit == "true")
+      -- TODO: fix fetchFromGitHub in Nixpkgs so that deepClone, leaveDotGit
+      -- and sparseCheckout won't get passed to fetchzip
+      if (deepClone == "true") || (leaveDotGit == "true") || (sparseCheckout /= "[ ]")
         then
           [trimming|
                fetchFromGitHub {
@@ -101,7 +104,8 @@ nixFetcher = \case
                  rev = $rev;
                  fetchSubmodules = $fetchSubmodules;
                  deepClone = $deepClone;
-                 leaveDotGit = $leaveDotGit;$n
+                 leaveDotGit = $leaveDotGit;
+                 sparseCheckout = $sparseCheckout;$n
                  sha256 = $sha256;
                }
          |]

diff --git a/src/NvFetcher/NixFetcher.hs b/src/NvFetcher/NixFetcher.hs
@@ -97,8 +97,8 @@ runNixPrefetchUrl url unpack name = do
 newtype FetchedGit = FetchedGit {sha256 :: Text}
   deriving (Show, Generic, A.FromJSON)
 
-runNixPrefetchGit :: Text -> Text -> Bool -> Bool -> Bool -> Action Checksum
-runNixPrefetchGit url rev fetchSubmodules deepClone leaveDotGit = do
+runNixPrefetchGit :: Text -> Text -> Bool -> Bool -> Bool -> [Text] -> Action Checksum
+runNixPrefetchGit url rev fetchSubmodules deepClone leaveDotGit sparseCheckout = do
   (CmdTime t, Stdout out, CmdLine c) <-
     quietly $
       command [EchoStderr False] "nix-prefetch-git" $
@@ -107,6 +107,7 @@ runNixPrefetchGit url rev fetchSubmodules deepClone leaveDotGit = do
           <> ["--fetch-submodules" | fetchSubmodules]
           <> ["--deepClone" | deepClone]
           <> ["--leave-dotGit" | leaveDotGit]
+          <> if null sparseCheckout then [] else ["--sparse-checkout", T.unpack $ T.intercalate "\n" sparseCheckout]
   putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s"
   case A.eitherDecode out of
     Right (FetchedGit x) -> sha256ToSri x
@@ -117,14 +118,14 @@ runNixPrefetchGit url rev fetchSubmodules deepClone leaveDotGit = do
 runFetcher :: NixFetcher Fresh -> Action (NixFetcher Fetched)
 runFetcher = \case
   FetchGit {..} -> do
-    result <- runNixPrefetchGit _furl (coerce _rev) _fetchSubmodules _deepClone _leaveDotGit
+    result <- runNixPrefetchGit _furl (coerce _rev) _fetchSubmodules _deepClone _leaveDotGit _sparseCheckout
     pure FetchGit {_sha256 = coerce result, ..}
   FetchGitHub {..} -> do
-    let useFetchGit = _fetchSubmodules || _leaveDotGit || _deepClone
+    let useFetchGit = _fetchSubmodules || _leaveDotGit || _deepClone || not (null _sparseCheckout)
         ver = coerce _rev
     result <-
       if useFetchGit
-        then runNixPrefetchGit [trimming|https://github.com/$_fowner/$_frepo|] (coerce _rev) _fetchSubmodules _deepClone _leaveDotGit
+        then runNixPrefetchGit [trimming|https://github.com/$_fowner/$_frepo|] (coerce _rev) _fetchSubmodules _deepClone _leaveDotGit _sparseCheckout
         else runNixPrefetchUrl [trimming|https://github.com/$_fowner/$_frepo/archive/$ver.tar.gz|] True mempty
     pure FetchGitHub {_sha256 = result, ..}
   FetchUrl {..} -> do
@@ -186,14 +187,14 @@ prefetch f force = askOracle $ RunFetch force f
 
 -- | Create a fetcher from git url
 gitFetcher :: Text -> PackageFetcher
-gitFetcher furl rev = FetchGit furl rev False True False Nothing ()
+gitFetcher furl rev = FetchGit furl rev False True False [] Nothing ()
 
 -- | Create a fetcher from github repo
 gitHubFetcher ::
   -- | owner and repo
   (Text, Text) ->
   PackageFetcher
-gitHubFetcher (owner, repo) rev = FetchGitHub owner repo rev False False False Nothing ()
+gitHubFetcher (owner, repo) rev = FetchGitHub owner repo rev False False False [] Nothing ()
 
 -- | Create a fetcher from pypi
 pypiFetcher :: Text -> PackageFetcher

diff --git a/src/NvFetcher/Types.hs b/src/NvFetcher/Types.hs
@@ -394,6 +394,7 @@ data NixFetcher (k :: FetchStatus)
         _deepClone :: Bool,
         _fetchSubmodules :: Bool,
         _leaveDotGit :: Bool,
+        _sparseCheckout :: [Text],
         _name :: Maybe Text,
         _sha256 :: FetchResult Checksum k
       }
@@ -404,6 +405,7 @@ data NixFetcher (k :: FetchStatus)
         _deepClone :: Bool,
         _fetchSubmodules :: Bool,
         _leaveDotGit :: Bool,
+        _sparseCheckout :: [Text],
         _name :: Maybe Text,
         _sha256 :: FetchResult Checksum k
       }
@@ -461,6 +463,7 @@ instance A.ToJSON (NixFetcher Fetched) where
         "deepClone" A..= _deepClone,
         "fetchSubmodules" A..= _fetchSubmodules,
         "leaveDotGit" A..= _leaveDotGit,
+        "sparseCheckout" A..= _sparseCheckout,
         "name" A..= _name,
         "sha256" A..= _sha256,
         "type" A..= A.String "git"
@@ -473,6 +476,7 @@ instance A.ToJSON (NixFetcher Fetched) where
         "deepClone" A..= _deepClone,
         "fetchSubmodules" A..= _fetchSubmodules,
         "leaveDotGit" A..= _leaveDotGit,
+        "sparseCheckout" A..= _sparseCheckout,
         "name" A..= _name,
         "sha256" A..= _sha256,
         "type" A..= A.String "github"
@@ -514,7 +518,8 @@ instance Pretty (NixFetcher k) where
               "rev" <> colon <+> pretty _rev,
               "deepClone" <> colon <+> pretty _deepClone,
               "fetchSubmodules" <> colon <+> pretty _fetchSubmodules,
-              "leaveDotGit" <> colon <+> pretty _leaveDotGit
+              "leaveDotGit" <> colon <+> pretty _leaveDotGit,
+              "sparseCheckout" <> colon <+> pretty _sparseCheckout
             ]
               <> ppField "name" _name
         )
@@ -529,7 +534,8 @@ instance Pretty (NixFetcher k) where
               "rev" <> colon <+> pretty _rev,
               "deepClone" <> colon <+> pretty _deepClone,
               "fetchSubmodules" <> colon <+> pretty _fetchSubmodules,
-              "leaveDotGit" <> colon <+> pretty _leaveDotGit
+              "leaveDotGit" <> colon <+> pretty _leaveDotGit,
+              "sparseCheckout" <> colon <+> pretty _sparseCheckout
             ]
               <> ppField "name" _name
         )

diff --git a/test/NixExprSpec.hs b/test/NixExprSpec.hs
@@ -34,6 +34,7 @@ spec = describe "toNixExpr" $ do
         fetchSubmodules = true;
         deepClone = false;
         leaveDotGit = false;
+        sparseCheckout = [ ];
         sha256 = "0000000000000000000000000000000000000000000000000000000000000000";
       }
     |]