getdeps: support GETDEPS_WGET_ARGS in wget version, support skipping …

…lfs/upload steps

Summary:
X-link: facebookincubator/katran#229

X-link: facebookexperimental/edencommon#18

X-link: facebook/sapling#878

X-link: facebook/openr#154

X-link: facebook/folly#2177

X-link: facebookincubator/zstrong#748

I found it useful to be able to set `GETDEPS_WGET_ARGS` to change some of the flags to `wget` while it's in that fetch mode :)
I also need to skip the lfs upload part in my environment

Reviewed By: mackorone

Differential Revision: D56263907

fbshipit-source-id: ae45c31ebb10123e0358544d294fe2f2979dd59a

build/fbcode_builder/getdeps.py

@@ -561,6 +561,10 @@ def run_project_cmd(self, args, loader, manifest):
         for m in projects:
             fetcher = loader.create_fetcher(m)
 
+            if args.build_skip_lfs_download and hasattr(fetcher, "skip_lfs_download"):
+                print("skipping lfs download for %s" % m.name)
+                fetcher.skip_lfs_download()
+
             if isinstance(fetcher, SystemPackageFetcher):
                 # We are guaranteed that if the fetcher is set to
                 # SystemPackageFetcher then this item is completely
@@ -652,7 +656,11 @@ def run_project_cmd(self, args, loader, manifest):
 
                     # Only populate the cache from continuous build runs, and
                     # only if we have a built_marker.
-                    if args.schedule_type == "continuous" and has_built_marker:
+                    if (
+                        not args.skip_upload
+                        and args.schedule_type == "continuous"
+                        and has_built_marker
+                    ):
                         cached_project.upload()
                 elif args.verbose:
                     print("found good %s" % built_marker)
@@ -1322,11 +1330,28 @@ def add_common_arg(*args, **kwargs):
         action="store_true",
         default=False,
     )
+    add_common_arg(
+        "-su",
+        "--skip-upload",
+        help="skip upload steps",
+        action="store_true",
+        default=False,
+    )
     add_common_arg(
         "--lfs-path",
         help="Provide a parent directory for lfs when fbsource is unavailable",
         default=None,
     )
+    add_common_arg(
+        "--build-skip-lfs-download",
+        action="store_true",
+        default=False,
+        help=(
+            "Download from the URL, rather than LFS. This is useful "
+            "in cases where the upstream project has uploaded a new "
+            "version of the archive with a different hash"
+        ),
+    )
 
     ap = argparse.ArgumentParser(
         description="Get and build dependencies and projects", parents=[common_args]

build/fbcode_builder/getdeps/fetcher.py

@@ -683,15 +683,18 @@ def progress_pycurl(self, total, amount, _uploadtotal, _uploadamount):
     start = time.time()
     try:
         if os.environ.get("GETDEPS_USE_WGET") is not None:
-            subprocess.run(
+            procargs = (
                 [
                     "wget",
+                ]
+                + os.environ.get("GETDEPS_WGET_ARGS", "").split()
+                + [
                     "-O",
                     file_name,
                     url,
                 ]
             )
-
+            subprocess.run(procargs, capture_output=True)
             headers = None
 
         elif os.environ.get("GETDEPS_USE_LIBCURL") is not None: