feat(python): Update source and sink summary queries

Author: GeekMasher

python/ql/lib/modeling/Util.qll

@@ -11,7 +11,7 @@ private import semmle.python.filters.Tests
  */
 class TestFile extends File {
   TestFile() {
-    this.getRelativePath().regexpMatch(".*(test|spec|examples).+") and
+    this.getRelativePath().regexpMatch(".*(test|spec|examples|__main__).+") and
     not this.getAbsolutePath().matches("%/ql/test/%") // allows our test cases to work
   }
 }

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll

@@ -332,7 +332,7 @@ predicate sourceModel(string type, string path, string kind, string model) {
 }
 
 /** Holds if a sink model exists for the given parameters. */
-private predicate sinkModel(string type, string path, string kind, string model) {
+predicate sinkModel(string type, string path, string kind, string model) {
   any(DeprecationAdapter a).sinkModel(type, path, kind) and
   model = "SinkModelCsv"
   or

python/ql/src/utils/modelgenerator/internal/CaptureModels.qll

@@ -14,8 +14,15 @@ private import semmle.python.dataflow.new.internal.TaintTrackingImplSpecific
 private import semmle.python.frameworks.data.internal.ApiGraphModels as ExternalFlow
 private import semmle.python.dataflow.new.internal.DataFlowImplCommon as DataFlowImplCommon
 private import semmle.python.dataflow.new.internal.DataFlowPrivate as DataFlowPrivate
+private import semmle.python.dataflow.new.TaintTracking
 private import codeql.mad.modelgenerator.internal.ModelGeneratorImpl
 private import modeling.ModelEditor
+private import modeling.Util as ModelEditorUtil
+// Concepts
+private import semmle.python.Concepts
+private import semmle.python.security.dataflow.CodeInjectionCustomizations
+private import semmle.python.security.dataflow.ServerSideRequestForgeryCustomizations
+private import semmle.python.security.dataflow.UnsafeDeserializationCustomizations
 
 module ModelGeneratorInput implements ModelGeneratorInputSig<P::Location, PythonDataFlow> {
   class Type = Unit;
@@ -36,19 +43,41 @@ module ModelGeneratorInput implements ModelGeneratorInputSig<P::Location, Python
     Parameter asParameter() { result = this }
   }
 
-  private predicate relevant(Callable api) { any() }
+  private predicate relevant(Callable api) {
+    api.(DataFlowCallable).getScope() instanceof ModelEditorUtil::RelevantScope
+  }
 
   predicate isUninterestingForDataFlowModels(Callable api) { none() }
 
   predicate isUninterestingForHeuristicDataFlowModels(Callable api) { none() }
 
+  private predicate hasManualSourceModel(Callable api) {
+    exists(Endpoint endpoint |
+      endpoint = api.(DataFlowCallable).getScope() and
+      ExternalFlow::sourceModel(endpoint.getNamespace(), _, endpoint.getKind(), _)
+    )
+    or
+    api.(DataFlowCallable).getScope() = any(ActiveThreatModelSource ats).getScope()
+  }
+
+  private predicate hasManualSinkModel(Callable api) {
+    exists(Endpoint endpoint |
+      endpoint = api.(DataFlowCallable).getScope() and
+      ExternalFlow::sinkModel(endpoint.getNamespace(), _, endpoint.getKind(), _)
+    )
+  }
+
   class SourceOrSinkTargetApi extends Callable {
     SourceOrSinkTargetApi() { relevant(this) }
   }
 
-  class SinkTargetApi extends SourceOrSinkTargetApi { }
+  class SinkTargetApi extends SourceOrSinkTargetApi {
+    SinkTargetApi() { not hasManualSinkModel(this) }
+  }
 
-  class SourceTargetApi extends SourceOrSinkTargetApi { }
+  class SourceTargetApi extends SourceOrSinkTargetApi {
+    SourceTargetApi() { not hasManualSourceModel(this) }
+  }
 
   class SummaryTargetApi extends Callable {
     private Callable lift;
@@ -67,15 +96,20 @@ module ModelGeneratorInput implements ModelGeneratorInputSig<P::Location, Python
 
   Type getUnderlyingContentType(DataFlow::ContentSet c) { result = any(Type t) and exists(c) }
 
-  string qualifierString() { result = "Argument[this]" }
+  string qualifierString() { result = "Argument[self]" }
 
-  string parameterAccess(Parameter p) {
-    result = "Argument[" + p.getParameter().getName() + "]"
-    or
-    not exists(p.getParameter().getName()) and
-    result = "Argument[" + p.getParameter().getPosition().toString() + "]"
+  private string parameterMad(Parameter p) {
+    exists(P::Parameter param |
+      param = p.getParameter() and
+      (
+        not param.isSelf() and
+        result = "Argument[" + param.getPosition().toString() + "," + param.getName() + ":]"
+      )
+    )
   }
 
+  string parameterAccess(Parameter p) { result = parameterMad(p) }
+
   string parameterContentAccess(Parameter p) { result = "Argument[]" }
 
   class InstanceParameterNode extends DataFlow::ParameterNode {
@@ -101,13 +135,22 @@ module ModelGeneratorInput implements ModelGeneratorInputSig<P::Location, Python
 
   predicate isOwnInstanceAccessNode(DataFlowPrivate::ReturnNode node) { none() }
 
-  predicate sinkModelSanitizer(DataFlow::Node node) { none() }
+  predicate sinkModelSanitizer(DataFlow::Node node) {
+    // Any Sanitizer
+    node instanceof Escaping
+  }
 
-  predicate apiSource(DataFlow::Node source) { none() }
+  predicate apiSource(DataFlow::Node source) {
+    // TODO: Non-Function Parameter support
+    source instanceof DataFlow::ParameterNode
+  }
 
   predicate irrelevantSourceSinkApi(Callable source, SourceTargetApi api) { none() }
 
-  string getInputArgument(DataFlow::Node source) { result = "getInputArgument(" + source + ")" }
+  string getInputArgument(DataFlow::Node source) {
+    source instanceof DataFlow::ParameterNode and
+    result = parameterMad(source)
+  }
 
   bindingset[kind]
   predicate isRelevantSinkKind(string kind) {
@@ -134,21 +177,80 @@ module ModelGeneratorInput implements ModelGeneratorInputSig<P::Location, Python
 
   string printContent(DataFlow::ContentSet c) { result = c.toString() }
 
-  string partialModelRow(Callable api, int i) {
-    exists(Endpoint e | e = api.(DataFlowFunction).getScope() |
-      i = 0 and result = e.getNamespace() + "." + e.getClass()
+  private string modelEndpoint(Endpoint endpoint) {
+    endpoint.getKind() = ["Function", "StaticMethod"] and
+    (
+      endpoint.getClass() != "" and
+    result =
+      "Member[" + endpoint.getClass().replaceAll(".", "].Member[") + "].Member[" +
+        endpoint.getFunctionName() + "]"
       or
-      i = 1 and result = "Member[" + e.getFunctionName() + "]"
+      endpoint.getClass() = "" and
+      result = "Member[" + endpoint.getFunctionName() + "]"
+    )
+    or
+    endpoint.getKind() = ["InstanceMethod", "ClassMethod", "InitMethod"] and
+    result =
+      "Member[" + endpoint.getClass().replaceAll(".", "].Member[") + "].Instance.Member[" +
+        endpoint.getFunctionName() + "]"
+  }
+
+  string partialModelRow(Callable api, int i) {
+    exists(Endpoint e |
+      e = api.(DataFlowFunction).getScope() and
+      (
+        i = 0 and result = e.getNamespace()
+        or
+        i = 1 and
+        result = modelEndpoint(e)
+      )
     )
   }
 
   string partialNeutralModelRow(Callable api, int i) { result = partialModelRow(api, i) }
 
-  // TODO: Implement this when we want to generate sources.
-  predicate sourceNode(DataFlow::Node node, string kind) { none() }
+  /**
+   *  Holds if the given node is a source node of the given kind.
+   */
+  predicate sourceNode(DataFlow::Node node, string kind) {
+    exists(ThreatModelSource tms |
+      node.getScope() = tms.getScope() and
+      kind = tms.getThreatModel()
+    )
+  }
 
-  // TODO: Implement this when we want to generate sinks.
-  predicate sinkNode(DataFlow::Node node, string kind) { none() }
+  /**
+   *  Holds if the given node is a sink node of the given kind.
+   */
+  predicate sinkNode(DataFlow::Node node, string kind) {
+    // Command Injection
+    node = any(SystemCommandExecution sce).getCommand() and
+    kind = "command-injection"
+    or
+    // Code Injection
+    node = any(CodeInjection::Sink ci) and
+    kind = "code-injection"
+    or
+    // Unsafe Deserialization
+    node = any(UnsafeDeserialization::Sink ud) and
+    kind = "unsafe-deserialization"
+    or
+    // SQL Injection
+    node = any(SqlExecution sql).getSql() and
+    kind = "sql-injection"
+    or
+    // File
+    node = any(FileSystemAccess fcs).getAPathArgument() and
+    kind = "path-injection"
+    or
+    // Template Injection
+    node = any(TemplateConstruction tc).getSourceArg() and
+    kind = "template-injection"
+    or
+    // Server Side Request Forgery
+    node = any(ServerSideRequestForgery::Sink ssrf).getRequest() and
+    kind = "request-forgery"
+  }
 }
 
 import MakeModelGenerator<P::Location, PythonDataFlow, PythonTaintTracking, ModelGeneratorInput>