Merge pull request #13819 from yoff/python/relax-module-resolution

Python: Relax module resolution

Author: RasmusWL

python/ql/lib/change-notes/2023-08-08-relax-module-resolution.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Support analyzing packages (folders with python code) that do not have `__init__.py` files, although this is technically required, we see real world projects that don't have this.

python/ql/lib/semmle/python/Module.qll

@@ -195,7 +195,22 @@ private predicate isPotentialPackage(Folder f) {
 }
 
 private string moduleNameFromBase(Container file) {
-  isPotentialPackage(file) and result = file.getBaseName()
+  // We used to also require `isPotentialPackage(f)` to hold in this case,
+  // but we saw modules not getting resolved because their folder did not
+  // contain an `__init__.py` file.
+  //
+  // This makes the folder not be a package but a namespace package instead.
+  // In most cases this is a mistake :| See following links for more details
+  // - https://dev.to/methane/don-t-omit-init-py-3hga
+  // - https://packaging.python.org/en/latest/guides/packaging-namespace-packages/
+  // - https://discuss.python.org/t/init-py-pep-420-and-iter-modules-confusion/9642
+  //
+  // It is possible that we can keep the original requirement on
+  // `isPotentialPackage(f)` here, but relax `isPotentialPackage` itself to allow
+  // for this behavior of missing `__init__.py` files. However, doing so involves
+  // cascading changes (for example to `moduleNameFromFile`), and was a more involved
+  // task than we wanted to take on.
+  result = file.getBaseName()
   or
   file instanceof File and result = file.getStem()
 }

python/ql/test/experimental/library-tests/CallGraph-implicit-init/example.py

@@ -17,6 +17,6 @@
 from foo.bar.a import afunc
 from foo_explicit.bar.a import explicit_afunc
 
-afunc() # $ MISSING: pt,tt=afunc
+afunc() # $ pt,tt="foo/bar/a.py:afunc"
 
 explicit_afunc() # $ pt,tt="foo_explicit/bar/a.py:explicit_afunc"