OSV integration and updates to safety (#305)

* OSV integration and updates to safety

* Updates for new APIs

* Annotate 2022 support

* Upgrade to the official mockito-kotlin package

* API update

.github/actions/Dockerfile

@@ -2,7 +2,7 @@ FROM anthonypjshaw/pycharm-security:latest
 COPY action.sh /action.sh
 COPY parse.py /code/parse.py
 COPY project.iml /code/project.iml
-COPY jdk.table.xml /root/.config/JetBrains/PyCharm2021.2/jdk.table.xml
+COPY jdk.table.xml /root/.config/JetBrains/PyCharm2021.3/jdk.table.xml
 RUN apt-get -y update && apt-get -y install python3 python3-pip python3-venv && python3 -m pip install setuptools
 RUN ["chmod", "+x", "/action.sh"]
 ENTRYPOINT ["/action.sh"]

.github/actions/jdk.table.xml

@@ -12,7 +12,7 @@
                         <root url="file:///usr/lib/python3.6/lib-dynload" type="simple" />
                         <root url="file:///usr/local/lib/python3.6/dist-packages" type="simple" />
                         <root url="file:///usr/lib/python3/dist-packages" type="simple" />
-                        <root url="file://$USER_HOME$/Library/Caches/JetBrains/PyCharm2021.2/python_stubs/-1506223722" type="simple" />
+                        <root url="file://$USER_HOME$/Library/Caches/JetBrains/PyCharm2021.3/python_stubs/-1506223722" type="simple" />
                         <root url="file://$APPLICATION_HOME_DIR$/plugins/python/helpers/python-skeletons" type="simple" />
                         <root url="file://$APPLICATION_HOME_DIR$/plugins/python/helpers/typeshed/stdlib/3.6" type="simple" />
                         <root url="file://$APPLICATION_HOME_DIR$/plugins/python/helpers/typeshed/stdlib/3" type="simple" />

.github/workflows/build.yml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        pycharm-version: ['2021.2']
+        pycharm-version: ['2021.3']
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v1
@@ -27,7 +27,7 @@ jobs:
         java-version: 11
     - uses: eskatos/gradle-command-action@v1
       with:
-        arguments: jacocoTestReport -PintellijPublishToken=FAKE_TOKEN -PintellijVersion=2021.2
+        arguments: jacocoTestReport -PintellijPublishToken=FAKE_TOKEN -PintellijVersion=2021.3
     - name: Codecov
       uses: codecov/[email protected]
       with:

Dockerfile

@@ -1,4 +1,4 @@
-ARG PYCHARM_VERSION=2021.2
+ARG PYCHARM_VERSION=2021.3
 FROM ubuntu:18.04
 ARG PYCHARM_VERSION
 RUN echo "Building PyCharm $PYCHARM_VERSION with python-security"

HISTORY.md

@@ -1,5 +1,11 @@
 # Release History
 
+## 1.25.0
+
+* Added a PyPi API option for package security checking that uses the new PyPI vulnerability API
+* Support for 2021.3
+* Update safety DB to december 
+
 ## 1.24.3
 
 * Update safety db to october 2021

README.md

@@ -42,6 +42,10 @@ This plugin will check the installed packages in your Python projects against th
 
 ![](doc/_static/safetydb-screenshot.png)
 
+## PyPi vulnerability API
+
+This plugin will check the installed packages in your Python projects against the OSV database in PyPi and raise a warning for any vulnerabilities.
+
 ## Current checks
 
 See [Supported Checks](https://pycharm-security.readthedocs.io/en/latest/checks/index.html) for a current list.

build.gradle

@@ -7,7 +7,7 @@ plugins {
 }
 
 group 'org.tonybaloney.security'
-version '1.24.3'
+version '1.25.0'
 
 def ktor_version = "1.6.6"
 def kotlin_version = "1.6.0"
@@ -21,9 +21,9 @@ repositories {
 
 dependencies {
     testCompile group: 'org.junit.jupiter', name: 'junit-jupiter', version: '5.8.2'
-    testImplementation "com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0"
-    testImplementation "net.bytebuddy:byte-buddy:1.12.2"
-    testImplementation "net.bytebuddy:byte-buddy-agent:1.12.2"
+    testImplementation "org.mockito.kotlin:mockito-kotlin:4.0.0"
+    testImplementation 'net.bytebuddy:byte-buddy:1.12.2'
+    testImplementation 'net.bytebuddy:byte-buddy-agent:1.12.2'
     implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
     compile "io.ktor:ktor-client-core:$ktor_version"
@@ -39,7 +39,7 @@ test {
 
 // See https://github.com/JetBrains/gradle-intellij-plugin/
 // Make the intellij version overridable on the command line to support multiple build versions..
-def intellijversion = project.hasProperty('intellijVersion') ? project.getProperty('intellijVersion') : '2021.2'
+def intellijversion = project.hasProperty('intellijVersion') ? project.getProperty('intellijVersion') : '2021.3'
 
 intellij {
     type = 'PC'
@@ -50,9 +50,11 @@ intellij {
 
 patchPluginXml {
     changeNotes = """
-      <h2>1.24.3</h2>
+      <h2>1.25.0</h2>
       <ul>
-        <li>Safety db update</li>
+        <li>Added a PyPi API option for package security checking that uses the new PyPI vulnerability API</li>
+        <li>Support for 2021.3</li>
+        <li>Update safety DB to december </li>
       </ul>
       """
 }

src/main/java/security/fixes/FixUtil.kt

@@ -8,19 +8,19 @@ import com.jetbrains.python.psi.*
 
 
 fun getPyExpressionAtCaret(file: PsiFile, editor: Editor): PyExpression? {
-    return PsiTreeUtil.getParentOfType(file.findElementAt(editor.caretModel.offset), PyExpression::class.java) ?: return null
+    return PsiTreeUtil.getParentOfType(file.findElementAt(editor.caretModel.offset), PyExpression::class.java)
 }
 
 fun getPyCallExpressionAtCaret(file: PsiFile, editor: Editor): PyCallExpression? {
-    return PsiTreeUtil.getTopmostParentOfType(file.findElementAt(editor.caretModel.offset), PyCallExpression::class.java) ?: return null
+    return PsiTreeUtil.getTopmostParentOfType(file.findElementAt(editor.caretModel.offset), PyCallExpression::class.java)
 }
 
 fun getListLiteralExpressionAtCaret(file: PsiFile, editor: Editor): PyListLiteralExpression? {
-    return PsiTreeUtil.getParentOfType(file.findElementAt(editor.caretModel.offset), PyListLiteralExpression::class.java) ?: return null
+    return PsiTreeUtil.getParentOfType(file.findElementAt(editor.caretModel.offset), PyListLiteralExpression::class.java)
 }
 
 fun getBinaryExpressionElementAtCaret(file: PsiFile, editor: Editor): PyBinaryExpression? {
-    return PsiTreeUtil.getParentOfType(file.findElementAt(editor.caretModel.offset), PyBinaryExpression::class.java) ?: return null
+    return PsiTreeUtil.getParentOfType(file.findElementAt(editor.caretModel.offset), PyBinaryExpression::class.java)
 }
 
 fun getNewCallExpressiontAtCaret(file: PsiFile, editor: Editor, project: Project, old: String, new: String): PyCallExpression ? {

src/main/java/security/fixes/ShellEscapeFixer.kt

@@ -11,6 +11,7 @@ import com.intellij.psi.PsiElement
 import com.intellij.psi.PsiFile
 import com.intellij.util.IncorrectOperationException
 import com.jetbrains.python.psi.*
+import com.jetbrains.python.psi.types.TypeEvalContext
 import security.helpers.QualifiedNameHelpers.getQualifiedName
 
 
@@ -56,11 +57,11 @@ class ShellEscapeFixer : LocalQuickFix, IntentionAction, HighPriorityAction {
         val list = elementGenerator.createListLiteral()
         for (item in oldElement.elements){
             if (item is PyReferenceExpression){
-                list.add(getNewEscapedExpression(file, project, item) as @org.jetbrains.annotations.NotNull PsiElement)
+                list.add(getNewEscapedExpression(file, project, item) as PsiElement)
                 continue
             } else if (item is PyCallExpression) {
-                if (getQualifiedName(item) != "shlex.quote") {
-                    list.add(getNewEscapedExpression(file, project, item) as @org.jetbrains.annotations.NotNull PsiElement)
+                if (getQualifiedName(item, TypeEvalContext.codeAnalysis(project, file)) != "shlex.quote") {
+                    list.add(getNewEscapedExpression(file, project, item) as PsiElement)
                     continue
                 }
             }

src/main/java/security/helpers/InspectionVisitorExtensions.kt

@@ -3,10 +3,17 @@ package security.helpers
 import com.intellij.codeInspection.LocalInspectionToolSession
 import com.intellij.codeInspection.ProblemsHolder
 import com.jetbrains.python.inspections.PyInspectionVisitor
+import com.jetbrains.python.psi.types.TypeEvalContext
+import security.helpers.TypeEvalContextHelper.getTypeEvalContext
 
 
-open class SecurityVisitor(holder: ProblemsHolder, session: LocalInspectionToolSession) : PyInspectionVisitor(holder, session) {
+open class SecurityVisitor(holder: ProblemsHolder, val session: LocalInspectionToolSession) : PyInspectionVisitor(holder, getTypeEvalContext(session)) {
     override fun getHolder(): ProblemsHolder {
         return super.getHolder()!!
     }
+
+    val typeEvalContext: TypeEvalContext
+        get() {
+            return getTypeEvalContext(session)
+        }
 }

src/main/java/security/helpers/QualifiedNameHelpers.kt

@@ -4,12 +4,14 @@ import com.intellij.psi.PsiInvalidElementAccessException
 import com.jetbrains.python.psi.PyCallExpression
 import com.jetbrains.python.psi.PyReferenceExpression
 import com.jetbrains.python.psi.resolve.PyResolveContext
+import com.jetbrains.python.psi.types.TypeEvalContext
 
 object QualifiedNameHelpers {
-    var resolveContext: PyResolveContext = PyResolveContext.defaultContext()
 
-    fun getQualifiedName(callExpression: PyCallExpression): String? {
+
+    fun getQualifiedName(callExpression: PyCallExpression, typeContext: TypeEvalContext): String? {
         try {
+            val resolveContext: PyResolveContext = PyResolveContext.defaultContext(typeContext)
             val resolved = callExpression.multiResolveCallee(resolveContext)
             if (resolved.isEmpty()) {
                 val firstChild = callExpression.firstChild ?: return null
@@ -24,22 +26,22 @@ object QualifiedNameHelpers {
     }
 }
 
-fun qualifiedNameMatches(node: PyCallExpression, potential: Array<String>) : Boolean {
-    val qualifiedName = QualifiedNameHelpers.getQualifiedName(node) ?: return false
+fun qualifiedNameMatches(node: PyCallExpression, potential: Array<String>, typeContext: TypeEvalContext) : Boolean {
+    val qualifiedName = QualifiedNameHelpers.getQualifiedName(node, typeContext) ?: return false
     return listOf(*potential).contains(qualifiedName)
 }
 
-fun qualifiedNameMatches(node: PyCallExpression, potential: String) : Boolean {
-    val qualifiedName = QualifiedNameHelpers.getQualifiedName(node) ?: return false
-    return (qualifiedName.equals(potential))
+fun qualifiedNameMatches(node: PyCallExpression, potential: String, typeContext: TypeEvalContext) : Boolean {
+    val qualifiedName = QualifiedNameHelpers.getQualifiedName(node, typeContext) ?: return false
+    return (qualifiedName == potential)
 }
 
-fun qualifiedNameStartsWith(node: PyCallExpression, potential: String) : Boolean {
-    val qualifiedName = QualifiedNameHelpers.getQualifiedName(node) ?: return false
+fun qualifiedNameStartsWith(node: PyCallExpression, potential: String, typeContext: TypeEvalContext) : Boolean {
+    val qualifiedName = QualifiedNameHelpers.getQualifiedName(node, typeContext) ?: return false
     return (qualifiedName.startsWith(potential))
 }
 
-fun qualifiedNameStartsWith(node: PyCallExpression, potential: Array<String>) : Boolean {
-    val qualifiedName = QualifiedNameHelpers.getQualifiedName(node) ?: return false
+fun qualifiedNameStartsWith(node: PyCallExpression, potential: Array<String>, typeContext: TypeEvalContext) : Boolean {
+    val qualifiedName = QualifiedNameHelpers.getQualifiedName(node, typeContext) ?: return false
     return potential.any { qualifiedName.startsWith(it) }
 }

src/main/java/security/helpers/TypeEvalContextHelper.kt

@@ -0,0 +1,10 @@
+package security.helpers
+
+import com.intellij.codeInspection.LocalInspectionToolSession
+import com.jetbrains.python.psi.types.TypeEvalContext
+
+object TypeEvalContextHelper {
+    fun getTypeEvalContext(session: LocalInspectionToolSession): TypeEvalContext {
+        return TypeEvalContext.codeAnalysis(session.file.project, session.file)
+    }
+}

src/main/java/security/packaging/PyPackageSecurityScan.kt

@@ -31,6 +31,7 @@ object PyPackageSecurityScan {
                 SecuritySettings.SafetyDbType.Api -> checkPackagesInSdks(pythonSdks, project, SafetyDbChecker(SecuritySettings.instance.pyupApiKey, SecuritySettings.instance.pyupApiUrl))
                 SecuritySettings.SafetyDbType.Custom -> checkPackagesInSdks(pythonSdks, project, SafetyDbChecker("", SecuritySettings.instance.pyupCustomUrl))
                 SecuritySettings.SafetyDbType.Snyk -> checkPackagesInSdks(pythonSdks, project, SnykChecker(SecuritySettings.instance.snykApiKey, SecuritySettings.instance.snykOrgId))
+                SecuritySettings.SafetyDbType.Pypi -> checkPackagesInSdks(pythonSdks, project, PypiChecker())
             }
             return true
         } catch (ex: PackageCheckerLoadException){

src/main/java/security/packaging/PypiChecker.kt

@@ -0,0 +1,87 @@
+package security.packaging
+
+import com.jetbrains.python.packaging.PyPackage
+import io.ktor.client.*
+import io.ktor.client.engine.apache.*
+import io.ktor.client.features.*
+import io.ktor.client.features.json.*
+import io.ktor.client.request.*
+import io.ktor.http.*
+import kotlinx.coroutines.TimeoutCancellationException
+import java.net.SocketTimeoutException
+
+
+class PypiChecker : BasePackageChecker() {
+    var baseUrl = "https://pypi.org"
+
+    class PyPiIssue (val record: VulnerabilityRecord, pyPackage: PyPackage): PackageIssue(pyPackage = pyPackage) {
+        override fun getMessage(): String {
+            return "${record.id} found in ${pyPackage.name} impacting version ${pyPackage.version}. <br/>See <a href='${record.link}'>${record.link}</a> for details"
+        }
+    }
+
+    data class VulnerabilityRecord (
+            val id: String,
+            val aliases: List<String>?,
+            val details: String,
+            val fixed_in: List<String>?,
+            val link: String,
+            val source: String
+    )
+
+    data class PyPiPackageApiResponse(
+            val info: Any,
+            val last_serial: Int,
+            val releases: Any,
+            val urls: Any,
+            val vulnerabilities: List<VulnerabilityRecord>?
+    )
+
+    private suspend fun load(packageName: String, packageVersion: String): PyPiPackageApiResponse? {
+        val client = HttpClient(Apache) {
+            install(JsonFeature) {
+                serializer = GsonSerializer{
+                    serializeNulls()
+                    disableHtmlEscaping()
+                }
+            }
+            defaultRequest {
+                headers {
+                    header("Content-Type", "application/json; charset=utf-8")
+                }
+            }
+            engine {
+                connectTimeout = 60_000
+                connectionRequestTimeout = 60_000
+                socketTimeout = 60_000
+            }
+        }
+
+        try {
+            return client.get<PyPiPackageApiResponse>(Url("$baseUrl/pypi/$packageName/$packageVersion/json"))
+        } catch (t: TimeoutCancellationException){
+            throw PackageCheckerLoadException("Timeout connecting to PyPi API.")
+        } catch (t: SocketTimeoutException){
+            throw PackageCheckerLoadException("Timeout on socket.")
+        } catch (t: ServerResponseException){
+            throw PackageCheckerLoadException("Server error on PyPi API.")
+        }
+    }
+
+    override fun hasMatch(pythonPackage: PyPackage?): Boolean {
+        return true // Hardcode to prevent it being called twice
+    }
+
+    override suspend fun getMatches (pythonPackage: PyPackage?): List<PyPiIssue> {
+        if (pythonPackage==null) return listOf()
+        val records: ArrayList<PyPiIssue> = ArrayList()
+        val data = load(pythonPackage.name.lowercase(), pythonPackage.version) ?: return records
+        if (data.vulnerabilities == null) return records
+
+        data.vulnerabilities.forEach { issue ->
+            records.add(PyPiIssue(issue, pythonPackage))
+        }
+
+        return records
+    }
+}

src/main/java/security/settings/SecuritySettings.kt

@@ -1,7 +1,7 @@
 package security.settings
 
+import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.components.PersistentStateComponent
-import com.intellij.openapi.components.ServiceManager
 import com.intellij.openapi.components.State
 import com.intellij.openapi.components.Storage
 import com.intellij.util.xmlb.XmlSerializerUtil
@@ -75,12 +75,13 @@ class SecuritySettings : PersistentStateComponent<SecuritySettings.State> {
         Bundled,
         Api,
         Custom,
-        Snyk
+        Snyk,
+        Pypi
     }
 
     companion object {
         @JvmStatic
         val instance: SecuritySettings
-            get() = ServiceManager.getService(SecuritySettings::class.java)
+            get() = ApplicationManager.getApplication().getService(SecuritySettings::class.java)
     }
 }