diff --git a/CHANGELOG.md b/CHANGELOG.md index 2be6f2ee94..df6f77ce19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +## [v0.2.2](https://github.com/ScoopInstaller/Scoop/compare/v0.2.1...v0.2.2) - 2022-06-21 + +### Features + +- **core:** Add `Get-Encoding` function to fix missing webclient encoding ([#4956](https://github.com/ScoopInstaller/Scoop/issues/4956)) +- **scoop-(un)hold:** Add `-g`/`--global` flag ([#4991](https://github.com/ScoopInstaller/Scoop/issues/4991)) +- **scoop-update:** Support `scoop update scoop` ([#4992](https://github.com/ScoopInstaller/Scoop/issues/4992)) +- **scoop-virustotal:** Migrate to VirusTotal API v3 ([#4983](https://github.com/ScoopInstaller/Scoop/issues/4983)) + +### Bug Fixes + +- **manifest:** Fix bugs in 'Get-Manifest()' ([#4986](https://github.com/ScoopInstaller/Scoop/issues/4986)) + ## [v0.2.1](https://github.com/ScoopInstaller/Scoop/compare/v0.2.0...v0.2.1) - 2022-06-10 ### Features diff --git a/bin/checkver.ps1 b/bin/checkver.ps1 index 7e089da94e..b477afc2d6 100644 --- a/bin/checkver.ps1 +++ b/bin/checkver.ps1 @@ -113,7 +113,7 @@ $Queue | ForEach-Object { } else { $wc.Headers.Add('User-Agent', (Get-UserAgent)) } - Register-ObjectEvent $wc downloadstringcompleted -ErrorAction Stop | Out-Null + Register-ObjectEvent $wc downloadDataCompleted -ErrorAction Stop | Out-Null $githubRegex = '\/releases\/tag\/(?:v|V)?([\d.]+)' @@ -190,7 +190,7 @@ $Queue | ForEach-Object { } $wc.Headers.Add('Referer', (strip_filename $url)) - $wc.DownloadStringAsync($url, $state) + $wc.DownloadDataAsync($url, $state) } function next($er) { @@ -218,7 +218,7 @@ while ($in_progress -gt 0) { $ver = $Version if (!$ver) { - $page = $ev.SourceEventArgs.Result + $page = (Get-Encoding($wc)).GetString($ev.SourceEventArgs.Result) $err = $ev.SourceEventArgs.Error if ($json.checkver.script) { $page = Invoke-Command ([scriptblock]::Create($json.checkver.script -join "`r`n")) diff --git a/bin/describe.ps1 b/bin/describe.ps1 index e2e8a81318..ae573ba484 100644 --- a/bin/describe.ps1 +++ b/bin/describe.ps1 @@ -44,7 +44,8 @@ $Queue | ForEach-Object { try { $wc = New-Object Net.Webclient $wc.Headers.Add('User-Agent', (Get-UserAgent)) - $home_html = $wc.DownloadString($manifest.homepage) + $homepage = $wc.DownloadData($manifest.homepage) + $home_html = (Get-Encoding($wc)).GetString($homepage) } catch { Write-Host "`n$($_.Exception.Message)" -ForegroundColor Red return diff --git a/lib/autoupdate.ps1 b/lib/autoupdate.ps1 index f6c991aa5b..7cf4f62053 100644 --- a/lib/autoupdate.ps1 +++ b/lib/autoupdate.ps1 @@ -1,12 +1,13 @@ # Must included with 'json.ps1' function find_hash_in_rdf([String] $url, [String] $basename) { - $data = $null + $xml = $null try { # Download and parse RDF XML file $wc = New-Object Net.Webclient $wc.Headers.Add('Referer', (strip_filename $url)) $wc.Headers.Add('User-Agent', (Get-UserAgent)) - [xml]$data = $wc.downloadstring($url) + $data = $wc.DownloadData($url) + [xml]$xml = (Get-Encoding($wc)).GetString($data) } catch [system.net.webexception] { write-host -f darkred $_ write-host -f darkred "URL $url is not valid" @@ -14,7 +15,7 @@ function find_hash_in_rdf([String] $url, [String] $basename) { } # Find file content - $digest = $data.RDF.Content | Where-Object { [String]$_.about -eq $basename } + $digest = $xml.RDF.Content | Where-Object { [String]$_.about -eq $basename } return format_hash $digest.sha256 } @@ -35,7 +36,8 @@ function find_hash_in_textfile([String] $url, [Hashtable] $substitutions, [Strin $wc = New-Object Net.Webclient $wc.Headers.Add('Referer', (strip_filename $url)) $wc.Headers.Add('User-Agent', (Get-UserAgent)) - $hashfile = $wc.downloadstring($url) + $data = $wc.DownloadData($url) + $hashfile = (Get-Encoding($wc)).GetString($data) } catch [system.net.webexception] { write-host -f darkred $_ write-host -f darkred "URL $url is not valid" @@ -88,7 +90,8 @@ function find_hash_in_json([String] $url, [Hashtable] $substitutions, [String] $ $wc = New-Object Net.Webclient $wc.Headers.Add('Referer', (strip_filename $url)) $wc.Headers.Add('User-Agent', (Get-UserAgent)) - $json = $wc.downloadstring($url) + $data = $wc.DownloadData($url) + $json = (Get-Encoding($wc)).GetString($data) } catch [system.net.webexception] { write-host -f darkred $_ write-host -f darkred "URL $url is not valid" @@ -108,7 +111,8 @@ function find_hash_in_xml([String] $url, [Hashtable] $substitutions, [String] $x $wc = New-Object Net.Webclient $wc.Headers.Add('Referer', (strip_filename $url)) $wc.Headers.Add('User-Agent', (Get-UserAgent)) - $xml = [xml]$wc.downloadstring($url) + $data = $wc.DownloadData($url) + $xml = [xml]((Get-Encoding($wc)).GetString($data)) } catch [system.net.webexception] { write-host -f darkred $_ write-host -f darkred "URL $url is not valid" diff --git a/lib/core.ps1 b/lib/core.ps1 index d5917a49b3..0a947fa0c4 100644 --- a/lib/core.ps1 +++ b/lib/core.ps1 @@ -15,6 +15,14 @@ function Optimize-SecurityProtocol { } } +function Get-Encoding($wc) { + if ($null -ne $wc.ResponseHeaders -and $wc.ResponseHeaders['Content-Type'] -match 'charset=([^;]*)') { + return [System.Text.Encoding]::GetEncoding($Matches[1]) + } else { + return [System.Text.Encoding]::GetEncoding('utf-8') + } +} + function Get-UserAgent() { return "Scoop/1.0 (+http://scoop.sh/) PowerShell/$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor) (Windows NT $([System.Environment]::OSVersion.Version.Major).$([System.Environment]::OSVersion.Version.Minor); $(if($env:PROCESSOR_ARCHITECTURE -eq 'AMD64'){'Win64; x64; '})$(if($env:PROCESSOR_ARCHITEW6432 -eq 'AMD64'){'WOW64; '})$PSEdition)" } @@ -847,7 +855,7 @@ function Confirm-InstallationStatus { $Global ) $Installed = @() - $Apps | Select-Object -Unique | Where-Object { $_.Name -ne 'scoop' } | ForEach-Object { + $Apps | Select-Object -Unique | Where-Object { $_ -ne 'scoop' } | ForEach-Object { $App, $null, $null = parse_app $_ if ($Global) { if (Test-Path (appdir $App $true)) { diff --git a/lib/depends.ps1 b/lib/depends.ps1 index 5f010f0119..bd2b26fb39 100644 --- a/lib/depends.ps1 +++ b/lib/depends.ps1 @@ -32,7 +32,7 @@ function Get-Dependency { $Unresolved = @() ) process { - $AppName, $manifest, $bucket, $null = Get-Manifest $AppName + $AppName, $manifest, $bucket, $url = Get-Manifest $AppName $Unresolved += $AppName if (!$manifest) { @@ -57,7 +57,11 @@ function Get-Dependency { if ($bucket) { $Resolved += "$bucket/$AppName" } else { - $Resolved += $AppName + if ($url) { + $Resolved += $url + } else { + $Resolved += $AppName + } } if ($Unresolved.Length -eq 0) { return $Resolved diff --git a/lib/description.ps1 b/lib/description.ps1 index c6c0e2067d..ef1da13589 100644 --- a/lib/description.ps1 +++ b/lib/description.ps1 @@ -18,7 +18,8 @@ function find_description($url, $html, $redir = $false) { if($refresh -and !$redir) { $wc = New-Object Net.Webclient $wc.Headers.Add('User-Agent', (Get-UserAgent)) - $html = $wc.downloadstring($refresh) + $data = $wc.DownloadData($refresh) + $html = (Get-Encoding($wc)).GetString($data) return find_description $refresh $html $true } diff --git a/lib/manifest.ps1 b/lib/manifest.ps1 index 7626c1348c..ffa76c7504 100644 --- a/lib/manifest.ps1 +++ b/lib/manifest.ps1 @@ -12,7 +12,8 @@ function url_manifest($url) { try { $wc = New-Object Net.Webclient $wc.Headers.Add('User-Agent', (Get-UserAgent)) - $str = $wc.downloadstring($url) + $data = $wc.DownloadData($url) + $str = (Get-Encoding($wc)).GetString($data) } catch [system.management.automation.methodinvocationexception] { warn "error: $($_.exception.innerexception.message)" } catch { @@ -24,6 +25,7 @@ function url_manifest($url) { function Get-Manifest($app) { $bucket, $manifest, $url = $null + $app = $app.TrimStart('/') # check if app is a URL or UNC path if ($app -match '^(ht|f)tps?://|\\\\') { $url = $app @@ -44,6 +46,7 @@ function Get-Manifest($app) { if (!$manifest) { # couldn't find app in buckets: check if it's a local path $appPath = $app + $bucket = $null if (!$appPath.EndsWith('.json')) { $appPath += '.json' } @@ -66,7 +69,8 @@ function save_installed_manifest($app, $bucket, $dir, $url) { if ($url) { $wc = New-Object Net.Webclient $wc.Headers.Add('User-Agent', (Get-UserAgent)) - $wc.downloadstring($url) > "$dir\manifest.json" + $data = $wc.DownloadData($url) + (Get-Encoding($wc)).GetString($data) | Out-UTF8File "$dir\manifest.json" } else { Copy-Item (manifest_path $app $bucket) "$dir\manifest.json" } diff --git a/libexec/scoop-download.ps1 b/libexec/scoop-download.ps1 index ff99995575..846750eb97 100644 --- a/libexec/scoop-download.ps1 +++ b/libexec/scoop-download.ps1 @@ -51,7 +51,7 @@ foreach ($curr_app in $apps) { $bucket = $version = $app = $manifest = $url = $null $app, $bucket, $version = parse_app $curr_app - $app, $manifest, $bucket, $url = Get-Manifest $curr_app + $app, $manifest, $bucket, $url = Get-Manifest "$bucket/$app" info "Starting download for $app..." diff --git a/libexec/scoop-hold.ps1 b/libexec/scoop-hold.ps1 index 3eb3c2e70e..ec4f8ce070 100644 --- a/libexec/scoop-hold.ps1 +++ b/libexec/scoop-hold.ps1 @@ -1,23 +1,43 @@ # Usage: scoop hold # Summary: Hold an app to disable updates +# Help: To hold a user-scoped app: +# scoop hold +# +# To hold a global app: +# scoop hold -g +# +# Options: +# -g, --global Hold globally installed apps +. "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\json.ps1" # 'save_install_info' (indirectly) . "$PSScriptRoot\..\lib\manifest.ps1" # 'install_info' 'Select-CurrentVersion' (indirectly) . "$PSScriptRoot\..\lib\versions.ps1" # 'Select-CurrentVersion' -$apps = $args +$opt, $apps, $err = getopt $args 'g' 'global' +if ($err) { "scoop hold: $err"; exit 1 } -if(!$apps) { +$global = $opt.g -or $opt.global + +if (!$apps) { my_usage exit 1 } +if ($global -and !(is_admin)) { + error 'You need admin rights to hold a global app.' + exit 1 +} + $apps | ForEach-Object { $app = $_ - $global = installed $app $true - if (!(installed $app)) { - error "'$app' is not installed." + if (!(installed $app $global)) { + if ($global) { + error "'$app' is not installed globally." + } else { + error "'$app' is not installed." + } return } @@ -29,7 +49,7 @@ $apps | ForEach-Object { $dir = versiondir $app $version $global $json = install_info $app $version $global $install = @{} - $json | Get-Member -MemberType Properties | ForEach-Object { $install.Add($_.Name, $json.($_.Name))} + $json | Get-Member -MemberType Properties | ForEach-Object { $install.Add($_.Name, $json.($_.Name)) } $install.hold = $true save_install_info $install $dir success "$app is now held and can not be updated anymore." diff --git a/libexec/scoop-unhold.ps1 b/libexec/scoop-unhold.ps1 index c749cadfee..2aa93d2df8 100644 --- a/libexec/scoop-unhold.ps1 +++ b/libexec/scoop-unhold.ps1 @@ -1,23 +1,43 @@ # Usage: scoop unhold # Summary: Unhold an app to enable updates +# Help: To unhold a user-scoped app: +# scoop unhold +# +# To unhold a global app: +# scoop unhold -g +# +# Options: +# -g, --global Unhold globally installed apps +. "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\json.ps1" # 'save_install_info' (indirectly) . "$PSScriptRoot\..\lib\manifest.ps1" # 'install_info' 'Select-CurrentVersion' (indirectly) . "$PSScriptRoot\..\lib\versions.ps1" # 'Select-CurrentVersion' -$apps = $args +$opt, $apps, $err = getopt $args 'g' 'global' +if ($err) { "scoop unhold: $err"; exit 1 } -if(!$apps) { +$global = $opt.g -or $opt.global + +if (!$apps) { my_usage exit 1 } +if ($global -and !(is_admin)) { + error 'You need admin rights to unhold a global app.' + exit 1 +} + $apps | ForEach-Object { $app = $_ - $global = installed $app $true - if (!(installed $app)) { - error "'$app' is not installed." + if (!(installed $app $global)) { + if ($global) { + error "'$app' is not installed globally." + } else { + error "'$app' is not installed." + } return } @@ -29,7 +49,7 @@ $apps | ForEach-Object { $dir = versiondir $app $version $global $json = install_info $app $version $global $install = @{} - $json | Get-Member -MemberType Properties | ForEach-Object { $install.Add($_.Name, $json.($_.Name))} + $json | Get-Member -MemberType Properties | ForEach-Object { $install.Add($_.Name, $json.($_.Name)) } $install.hold = $null save_install_info $install $dir success "$app is no longer held and can be updated again." diff --git a/libexec/scoop-update.ps1 b/libexec/scoop-update.ps1 index d06bc5d669..844127f82f 100644 --- a/libexec/scoop-update.ps1 +++ b/libexec/scoop-update.ps1 @@ -306,19 +306,24 @@ if (-not ($apps -or $all)) { 'ERROR: You need admin rights to update global apps.'; exit 1 } - if (is_scoop_outdated) { - update_scoop - } $outdated = @() + $updateScoop = $null -ne ($apps | Where-Object { $_ -eq 'scoop' }) -or (is_scoop_outdated) + $apps = $apps | Where-Object { $_ -ne 'scoop' } $apps_param = $apps + if ($updateScoop) { + update_scoop + } + if ($apps_param -eq '*' -or $all) { $apps = applist (installed_apps $false) $false if ($global) { $apps += applist (installed_apps $true) $true } } else { - $apps = Confirm-InstallationStatus $apps_param -Global:$global + if ($apps_param) { + $apps = Confirm-InstallationStatus $apps_param -Global:$global + } } if ($apps) { $apps | ForEach-Object { diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index 5be7a10f56..37da80ce52 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -1,28 +1,21 @@ # Usage: scoop virustotal [* | app1 app2 ...] [options] -# Summary: Look for app's hash on virustotal.com -# Help: Look for app's hash (MD5, SHA1 or SHA256) on virustotal.com +# Summary: Look for app's hash or url on virustotal.com +# Help: Look for app's hash or url on virustotal.com # # Use a single '*' for app to check all installed apps. # -# The download's hash is also a key to access VirusTotal's scan results. -# This allows to check the safety of the files without even downloading -# them in many cases. If the hash is unknown to VirusTotal, the -# download link is printed to submit it to VirusTotal. -# -# If you have signed up to VirusTotal's community, you have an API key -# that this script can use to submit unknown packages for inspection -# if you use the `--scan' flag. Tell scoop about your API key with: +# To use this command, you have to sign up to VirusTotal's community, +# and get an API key. Then, tell scoop about your API key with: # # scoop config virustotal_api_key # # Exit codes: -# 0 -> success -# 1 -> problem parsing arguments -# 2 -> at least one package was marked unsafe by VirusTotal -# 4 -> at least one exception was raised while looking for info -# 8 -> at least one package couldn't be queried because its hash type -# isn't supported by VirusTotal, the manifest couldn't be found -# or didn't contain a hash +# 0 -> success +# 1 -> problem parsing arguments +# 2 -> at least one package was marked unsafe by VirusTotal +# 4 -> at least one exception was raised while looking for info +# 8 -> at least one package couldn't be queried because the manifest couldn't be found +# 16 -> VirusTotal API key is not configured # Note: the exit codes (2, 4 & 8) may be combined, e.g. 6 -> exit codes # 2 & 4 combined # @@ -33,6 +26,7 @@ # your virustotal_api_key. # -n, --no-depends By default, all dependencies are checked too. This flag avoids it. # -u, --no-update-scoop Don't update Scoop before checking if it's outdated +# -p, --passthru Return reports as objects . "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' @@ -40,7 +34,7 @@ . "$PSScriptRoot\..\lib\install.ps1" # 'hash_for_url' . "$PSScriptRoot\..\lib\depends.ps1" # 'Get-Dependency' -$opt, $apps, $err = getopt $args 'a:snu' @('arch=', 'scan', 'no-depends', 'no-update-scoop') +$opt, $apps, $err = getopt $args 'a:snup' @('arch=', 'scan', 'no-depends', 'no-update-scoop', 'passthru') if ($err) { "scoop virustotal: $err"; exit 1 } if (!$apps) { my_usage; exit 1 } $architecture = ensure_architecture ($opt.a + $opt.arch) @@ -67,11 +61,17 @@ if (!$opt.n -and !$opt.'no-depends') { $_ERR_UNSAFE = 2 $_ERR_EXCEPTION = 4 $_ERR_NO_INFO = 8 +$_ERR_NO_API_KEY = 16 $exit_code = 0 -# Global flag to warn only once about missing API key: -$warned_no_api_key = $False +# Global API key: +$api_key = get_config virustotal_api_key +if (!$api_key) { + abort ("VirusTotal API key is not configured`n" + + " You could get one from https://www.virustotal.com/gui/my-apikey and set with`n" + + " scoop config virustotal_api_key ") $_ERR_NO_API_KEY +} # Global flag to explain only once about sleep between requests $explained_rate_limit_sleeping = $False @@ -80,65 +80,132 @@ $explained_rate_limit_sleeping = $False # script execution progresses $requests = 0 -Function Get-VirusTotalResult($hash, $app) { +Function ConvertTo-VirusTotalUrlId ($url) { + $url_id = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($url)) + $url_id = $url_id -replace '\+', '-' + $url_id = $url_id -replace '/', '_' + $url_id = $url_id -replace '=', '' + $url_id +} + +Function Get-RemoteFileSize ($url) { + $response = Invoke-WebRequest -Uri $url -Method HEAD -UseBasicParsing + $response.Headers.'Content-Length' | ForEach-Object { [System.Convert]::ToInt32($_) } +} + +Function Get-VirusTotalResultByHash ($hash, $url, $app) { $hash = $hash.ToLower() - $url = "https://www.virustotal.com/ui/files/$hash" - $wc = New-Object Net.Webclient - $wc.Headers.Add('User-Agent', (Get-UserAgent)) - $result = $wc.downloadstring($url) + $api_url = "https://www.virustotal.com/api/v3/files/$hash" + $headers = @{} + $headers.Add('Accept', 'application/json') + $headers.Add('x-apikey', $api_key) + $response = Invoke-WebRequest -Uri $api_url -Method GET -Headers $headers -UseBasicParsing + $result = $response.Content $stats = json_path $result '$.data.attributes.last_analysis_stats' - $malicious = json_path $stats '$.malicious' - $suspicious = json_path $stats '$.suspicious' - $undetected = json_path $stats '$.undetected' - $unsafe = [int]$malicious + [int]$suspicious - $see_url = "see https://www.virustotal.com/#/file/$hash/detection" - switch ($unsafe) { - 0 { if ($undetected -eq 0) { $fg = 'Yellow' } else { $fg = 'DarkGreen' } } - 1 { $fg = 'DarkYellow' } - 2 { $fg = 'Yellow' } - default { $fg = 'Red' } + [int]$malicious = json_path $stats '$.malicious' + [int]$suspicious = json_path $stats '$.suspicious' + [int]$timeout = json_path $stats '$.timeout' + [int]$undetected = json_path $stats '$.undetected' + [int]$unsafe = $malicious + $suspicious + [int]$total = $unsafe + $undetected + [int]$fileSize = json_path $result '$.data.attributes.size' + $report_hash = json_path $result '$.data.attributes.sha256' + $report_url = "https://www.virustotal.com/gui/file/$report_hash" + if ($total -eq 0) { + info "$app`: Analysis in progress." + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + 'App.Hash' = $hash + 'App.HashType' = $null + 'App.Size' = filesize $fileSize + 'FileReport.Url' = $report_url + 'FileReport.Hash' = $report_hash + 'UrlReport.Url' = $null + } + } else { + $vendorResults = (ConvertFrom-Json((json_path $result '$.data.attributes.last_analysis_results'))).PSObject.Properties.Value + switch ($unsafe) { + 0 { + success "$app`: $unsafe/$total, see $report_url" + } + 1 { + warn "$app`: $unsafe/$total, see $report_url" + } + 2 { + warn "$app`: $unsafe/$total, see $report_url" + } + Default { + warn "`e[31m$app`: $unsafe/$total, see $report_url`e[0m" + } + } + $maliciousResults = $vendorResults | + Where-Object -Property category -EQ 'malicious' | + Select-Object -ExpandProperty engine_name + $suspiciousResults = $vendorResults | + Where-Object -Property category -EQ 'suspicious' | + Select-Object -ExpandProperty engine_name + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + 'App.Hash' = $hash + 'App.HashType' = $null + 'App.Size' = filesize $fileSize + 'FileReport.Url' = $report_url + 'FileReport.Hash' = $report_hash + 'FileReport.Malicious' = if ($maliciousResults) { $maliciousResults } else { 0 } + 'FileReport.Suspicious' = if ($suspiciousResults) { $suspiciousResults } else { 0 } + 'FileReport.Timeout' = $timeout + 'FileReport.Undetected' = $undetected + 'UrlReport.Url' = $null + } } - Write-Host -f $fg "$app`: $unsafe/$undetected, $see_url" if ($unsafe -gt 0) { - return $_ERR_UNSAFE + $Script:exit_code = $exit_code -bor $_ERR_UNSAFE } - return 0 } -Function Search-VirusTotal ($hash, $app) { - if ($hash -match '(?[^:]+):(?.*)') { - $hash = $matches['hash'] - if ($matches['algo'] -match '(md5|sha1|sha256)') { - return Get-VirusTotalResult $hash $app +Function Get-VirusTotalResultByUrl ($url, $app) { + $id = ConvertTo-VirusTotalUrlId $url + $api_url = "https://www.virustotal.com/api/v3/urls/$id" + $headers = @{} + $headers.Add('Accept', 'application/json') + $headers.Add('x-apikey', $api_key) + $response = Invoke-WebRequest -Uri $api_url -Method GET -Headers $headers -UseBasicParsing + $result = $response.Content + $id = json_path $result '$.data.id' + $hash = json_path $result '$.data.attributes.last_http_response_content_sha256' 6>$null + $last_analysis_date = json_path $result '$.data.attributes.last_analysis_date' 6>$null + $url_report_url = "https://www.virustotal.com/gui/url/$id" + info "$app`: Url report found." + if (!$hash) { + if (!$last_analysis_date) { + info "$app`: Analysis in progress." } else { - warn "$app`: Unsupported hash $($matches['algo']). VirusTotal needs md5, sha1 or sha256." - return $_ERR_NO_INFO + info "$app`: Related file report not found." + warn "$app`: Manual file upload is required (instead of url submission)." + } + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + 'App.Hash' = $null + 'App.HashType' = $null + 'FileReport.Url' = $null + 'UrlReport.Url' = $url_report_url + 'UrlReport.Hash' = $null } - } - - return Get-VirusTotalResult $hash $app -} - -Function Submit-RedirectedUrl { - # Follow up to one level of HTTP redirection - # - # Copied from http://www.powershellmagazine.com/2013/01/29/pstip-retrieve-a-redirected-url/ - # Adapted according to Roy's response (January 23, 2014 at 11:59 am) - # Adapted to always return an URL - Param ( - [Parameter(Mandatory = $true)] - [String]$URL - ) - $request = [System.Net.WebRequest]::Create($url) - $request.AllowAutoRedirect = $false - $response = $request.GetResponse() - if (([int]$response.StatusCode -ge 300) -and ([int]$response.StatusCode -lt 400)) { - $redir = $response.GetResponseHeader('Location') } else { - $redir = $URL + info "$app`: Related file report found." + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + 'App.Hash' = $null + 'App.HashType' = $null + 'FileReport.Url' = $null + 'UrlReport.Url' = $url_report_url + 'UrlReport.Hash' = $hash + } } - $response.Close() - return $redir } # Submit-ToVirusTotal @@ -152,32 +219,38 @@ Function Submit-RedirectedUrl { # exceeded, without risking an infinite loop (as stack # overflow) if the submission keeps failing. Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { - $api_key = get_config virustotal_api_key - if ($do_scan -and !$api_key -and !$warned_no_api_key) { - $warned_no_api_key = $true - info 'Submitting unknown apps needs a VirusTotal API key. ' + - "Set it up with`n`tscoop config virustotal_api_key " - - } - if (!$do_scan -or !$api_key) { - warn "$app`: not found`: manually submit $url" + if (!$do_scan) { + warn "$app`: not found`: you can manually submit $url" return } try { - # Follow redirections (for e.g. sourceforge URLs) because - # VirusTotal analyzes only "direct" download links - $url = $url.Split('#').GetValue(0) - $new_redir = $url - do { - $orig_redir = $new_redir - $new_redir = Submit-RedirectedUrl $orig_redir - } while ($orig_redir -ne $new_redir) $requests += 1 - $result = Invoke-WebRequest -Uri 'https://www.virustotal.com/vtapi/v2/url/scan' -Body @{apikey = $api_key; url = $new_redir } -Method Post -UseBasicParsing - $submitted = $result.StatusCode -eq 200 - if ($submitted) { - warn "$app`: not found`: submitted $url" + + $encoded_url = [System.Web.HttpUtility]::UrlEncode($url) + $api_url = 'https://www.virustotal.com/api/v3/urls' + $content_type = 'application/x-www-form-urlencoded' + $headers = @{} + $headers.Add('Accept', 'application/json') + $headers.Add('x-apikey', $api_key) + $headers.Add('Content-Type', $content_type) + $body = "url=$encoded_url" + $result = Invoke-WebRequest -Uri $api_url -Method POST -Headers $headers -ContentType $content_type -Body $body -UseBasicParsing + if ($result.StatusCode -eq 200) { + $id = ((json_path $result '$.data.id') -split '-')[1] + $url_report_url = "https://www.virustotal.com/gui/url/$id" + $fileSize = Get-RemoteFileSize $url + if ($fileSize -gt 80000000) { + info "$app`: Remote file size: $(filesize $fileSize). Large files might require manual file upload instead of url submission." + } + info "$app`: Analysis in progress." + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + 'App.Size' = filesize $fileSize + 'FileReport.Url' = $null + 'UrlReport.Url' = $url_report_url + } return } @@ -188,7 +261,7 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { info "Sleeping 60+ seconds between requests due to VirusTotal's 4/min limit" } Start-Sleep -s (60 + $requests) - Submit-ToVirusTotal $new_redir $app $do_scan $True + Submit-ToVirusTotal $url $app $do_scan $True } else { warn "$app`: VirusTotal submission of $url failed`:`n" + "`tAPI returned $($result.StatusCode) after retrying" @@ -199,38 +272,120 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { } } -foreach ($app in $apps) { - # write-host $app +$reports = $apps | ForEach-Object { + $app = $_ $null, $manifest, $bucket, $null = Get-Manifest $app if (!$manifest) { $exit_code = $exit_code -bor $_ERR_NO_INFO warn "$app`: manifest not found" - continue + return } + [int]$index = 0 $urls = script:url $manifest $architecture $urls | ForEach-Object { $url = $_ + $index++ + if ($urls.GetType().IsArray) { + info "$app`: url $index" + } $hash = hash_for_url $manifest $url $architecture try { + $isHashUnsupported = $false + if ($hash -match '(?[^:]+):(?.*)') { + $algo = $matches.algo + $hash = $matches.hash + if ($matches.algo -inotin 'md5', 'sha1', 'sha256') { + $hash = $null + $isHashUnsupported = $true + warn "$app`: Unsupported hash $($matches.algo). Will search by url instead." + } + } elseif ($hash) { + $algo = 'sha256' + } if ($hash) { - $exit_code = $exit_code -bor (Search-VirusTotal $hash $app) + $file_report = Get-VirusTotalResultByHash $hash $url $app + $file_report.'App.HashType' = $algo + $file_report + return + } elseif (!$isHashUnsupported) { + warn "$app`: Hash not found. Will search by url instead." + } + } catch [Exception] { + $exit_code = $exit_code -bor $_ERR_EXCEPTION + if ($_.Exception.Response.StatusCode -eq 404) { + $file_report_not_found = $true + warn "$app`: File report not found. Will search by url instead." } else { - warn "$app`: Can't find hash for $url" + if ($_.Exception.Response.StatusCode -in 204, 429) { + abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code + } + warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" + return + } + } + + try { + $url_report = Get-VirusTotalResultByUrl $url $app + $url_report.'App.Hash' = $hash + $url_report.'App.HashType' = $algo + if ($url_report.'UrlReport.Hash' -and ($file_report_not_found -eq $true) -and $hash) { + if ($algo -eq 'sha256') { + if ($url_report.'UrlReport.Hash' -eq $hash) { + warn "$app`: Manual file upload is required (instead of url submission) for $url" + } else { + error "$app`: Hash not matched for $url" + } + } else { + error "$app`: Hash not matched or manual file upload is required (instead of url submission) for $url" + } + $url_report + return + } + if (!$url_report.'UrlReport.Hash') { + $url_report + return } } catch [Exception] { $exit_code = $exit_code -bor $_ERR_EXCEPTION - if ($_.Exception.Message -like '*(404)*') { + if ($_.Exception.Response.StatusCode -eq 404) { + warn "$app`: Url report not found. Will submit $url" Submit-ToVirusTotal $url $app ($opt.scan -or $opt.s) + return + } else { + if ($_.Exception.Response.StatusCode -in 204, 429) { + abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code + } + warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" + return + } + } + + try { + $file_report = Get-VirusTotalResultByHash $url_report.'UrlReport.Hash' $url $app + $file_report.'App.Hash' = $hash + $file_report.'App.HashType' = $algo + $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' + $file_report + warn "$app`: Unable to check hash match for $url" + } catch [Exception] { + $exit_code = $exit_code -bor $_ERR_EXCEPTION + if ($_.Exception.Response.StatusCode -eq 404) { + warn "$app`: File report not found for unknown reason. Manual file upload is required (instead of url submission)." + $url_report } else { - if ($_.Exception.Message -match '\(204|429\)') { - abort "$app`: VirusTotal request failed`: $($_.Exception.Message)", $exit_code + if ($_.Exception.Response.StatusCode -in 204, 429) { + abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code } warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" + return } } } } +if ($opt.p -or $opt.'passthru') { + $reports +} exit $exit_code