Swift: make codegen a bit more language-agnostic

Author: redsun82

swift/codegen/codegen.py

@@ -24,31 +24,44 @@ def _parse_args() -> argparse.Namespace:
                         "and cpp")
     p.add_argument("--verbose", "-v", action="store_true", help="print more information")
     p.add_argument("--quiet", "-q", action="store_true", help="only print errors")
-    p.add_argument("--swift-dir", type=_abspath, default=paths.swift_dir,
-                   help="the directory that should be regarded as the root of the swift codebase. Used to compute QL "
-                        "imports and in some comments (default %(default)s)")
-    p.add_argument("--schema", type=_abspath, default=paths.swift_dir / "schema.py",
-                   help="input schema file (default %(default)s)")
-    p.add_argument("--dbscheme", type=_abspath, default=paths.swift_dir / "ql/lib/swift.dbscheme",
-                   help="output file for dbscheme generation, input file for trap generation (default %(default)s)")
-    p.add_argument("--ql-output", type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/generated",
-                   help="output directory for generated QL files (default %(default)s)")
-    p.add_argument("--ql-stub-output", type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/elements",
-                   help="output directory for QL stub/customization files (default %(default)s). Defines also the "
-                        "generated qll file importing every class file")
-    p.add_argument("--ql-test-output", type=_abspath, default=paths.swift_dir / "ql/test/extractor-tests/generated",
-                   help="output directory for QL generated extractor test files (default %(default)s)")
+    p.add_argument("--root-dir", type=_abspath, default=paths.root_dir,
+                   help="the directory that should be regarded as the root of the language pack codebase. Used to"
+                        "compute QL imports and in some comments and as root for relative paths provided as options "
+                        "(default %(default)s)")
+    p.add_argument("--language", default=paths.root_dir.name,
+                   help="string that should replace {language} in other provided options")
+    path_arguments = [
+        p.add_argument("--schema", default="schema.py",
+                       help="input schema file (default %(default)s)"),
+        p.add_argument("--dbscheme", default="ql/lib/{language}.dbscheme",
+                       help="output file for dbscheme generation, input file for trap generation (default "
+                            "%(default)s)"),
+        p.add_argument("--ql-output", default="ql/lib/codeql/{language}/generated",
+                       help="output directory for generated QL files (default %(default)s)"),
+        p.add_argument("--ql-stub-output", default="ql/lib/codeql/{language}/elements",
+                       help="output directory for QL stub/customization files (default %(default)s). Defines also the "
+                            "generated qll file importing every class file"),
+        p.add_argument("--ql-test-output", default="ql/test/extractor-tests/generated",
+                       help="output directory for QL generated extractor test files (default %(default)s)"),
+        p.add_argument("--cpp-output",
+                       help="output directory for generated C++ files, required if trap or cpp is provided to "
+                            "--generate"),
+        p.add_argument("--generated-registry", default="ql/.generated.list",
+                       help="registry file containing information about checked-in generated code"),
+    ]
     p.add_argument("--ql-format", action="store_true", default=True,
                    help="use codeql to autoformat QL files (which is the default)")
     p.add_argument("--no-ql-format", action="store_false", dest="ql_format", help="do not format QL files")
     p.add_argument("--codeql-binary", default="codeql", help="command to use for QL formatting (default %(default)s)")
-    p.add_argument("--cpp-output", type=_abspath,
-                   help="output directory for generated C++ files, required if trap or cpp is provided to --generate")
-    p.add_argument("--generated-registry", type=_abspath, default=paths.swift_dir / "ql/.generated.list",
-                   help="registry file containing information about checked-in generated code")
     p.add_argument("--force", "-f", action="store_true",
-                   help="generate all files without skipping unchanged files and overwriting modified ones")
-    return p.parse_args()
+                   help="generate all files without skipping unchanged files and overwriting modified ones"),
+    opts = p.parse_args()
+    # absolutize all paths relative to --root-dir
+    for arg in path_arguments:
+        path = getattr(opts, arg.dest)
+        if path is not None:
+            setattr(opts, arg.dest, opts.root_dir / path.format(language=opts.language))
+    return opts
 
 
 def _abspath(x: str) -> typing.Optional[pathlib.Path]:
@@ -65,7 +78,7 @@ def run():
         log_level = logging.INFO
     logging.basicConfig(format="{levelname} {message}", style='{', level=log_level)
     for target in opts.generate:
-        generate(target, opts, render.Renderer(opts.swift_dir))
+        generate(target, opts, render.Renderer(opts.root_dir))
 
 
 if __name__ == "__main__":

swift/codegen/generators/dbschemegen.py

@@ -110,12 +110,12 @@ def get_declarations(data: schema.Schema):
     return declarations
 
 
-def get_includes(data: schema.Schema, include_dir: pathlib.Path, swift_dir: pathlib.Path):
+def get_includes(data: schema.Schema, include_dir: pathlib.Path, root_dir: pathlib.Path):
     includes = []
     for inc in data.includes:
         inc = include_dir / inc
         with open(inc) as inclusion:
-            includes.append(SchemeInclude(src=inc.relative_to(swift_dir), data=inclusion.read()))
+            includes.append(SchemeInclude(src=inc.relative_to(root_dir), data=inclusion.read()))
     return includes
 
 
@@ -125,8 +125,8 @@ def generate(opts, renderer):
 
     data = schemaloader.load_file(input)
 
-    dbscheme = Scheme(src=input.relative_to(opts.swift_dir),
-                      includes=get_includes(data, include_dir=input.parent, swift_dir=opts.swift_dir),
+    dbscheme = Scheme(src=input.relative_to(opts.root_dir),
+                      includes=get_includes(data, include_dir=input.parent, root_dir=opts.root_dir),
                       declarations=get_declarations(data))
 
     renderer.render(dbscheme, out)

swift/codegen/generators/qlgen.py

@@ -198,8 +198,8 @@ def get_ql_ipa_class(cls: schema.Class):
     return get_ql_ipa_class_db(cls.name)
 
 
-def get_import(file: pathlib.Path, swift_dir: pathlib.Path):
-    stem = file.relative_to(swift_dir / "ql/lib").with_suffix("")
+def get_import(file: pathlib.Path, root_dir: pathlib.Path):
+    stem = file.relative_to(root_dir / "ql/lib").with_suffix("")
     return str(stem).replace("/", ".")
 
 
@@ -344,7 +344,7 @@ def generate(opts, renderer):
 
         classes_by_dir_and_name = sorted(classes.values(), key=lambda cls: (cls.dir, cls.name))
         for c in classes_by_dir_and_name:
-            imports[c.name] = get_import(stub_out / c.path, opts.swift_dir)
+            imports[c.name] = get_import(stub_out / c.path, opts.root_dir)
 
         for c in classes.values():
             qll = out / c.path.with_suffix(".qll")
@@ -355,7 +355,7 @@ def generate(opts, renderer):
             path = _get_path(c)
             stub_file = stub_out / path
             if not renderer.is_customized_stub(stub_file):
-                base_import = get_import(out / path, opts.swift_dir)
+                base_import = get_import(out / path, opts.root_dir)
                 renderer.render(_get_stub(c, base_import), stub_file)
 
         # for example path/to/elements -> path/to/elements.qll
@@ -404,7 +404,7 @@ def generate(opts, renderer):
                     if not renderer.is_customized_stub(stub_file):
                         # stub rendering must be postponed as we might not have yet all subtracted ipa types in `ipa_type`
                         stubs[stub_file] = ql.Synth.ConstructorStub(ipa_type)
-                    constructor_import = get_import(stub_file, opts.swift_dir)
+                    constructor_import = get_import(stub_file, opts.root_dir)
                     constructor_imports.append(constructor_import)
                     if ipa_type.is_ipa:
                         ipa_constructor_imports.append(constructor_import)

swift/codegen/lib/paths.py

@@ -6,13 +6,13 @@
 
 try:
     workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']).resolve()  # <- means we are using bazel run
-    swift_dir = workspace_dir / 'swift'
+    root_dir = workspace_dir / 'swift'
 except KeyError:
     _this_file = pathlib.Path(__file__).resolve()
-    swift_dir = _this_file.parents[2]
-    workspace_dir = swift_dir.parent
+    root_dir = _this_file.parents[2]
+    workspace_dir = root_dir.parent
 
-lib_dir = swift_dir / 'codegen' / 'lib'
-templates_dir = swift_dir / 'codegen' / 'templates'
+lib_dir = root_dir / 'codegen' / 'lib'
+templates_dir = root_dir / 'codegen' / 'templates'
 
 exe_file = pathlib.Path(sys.argv[0]).resolve()

swift/codegen/lib/render.py

@@ -25,13 +25,16 @@ class Error(Exception):
 class Renderer:
     """ Template renderer using mustache templates in the `templates` directory """
 
-    def __init__(self, swift_dir: pathlib.Path):
+    def __init__(self, root_dir: pathlib.Path):
         self._r = pystache.Renderer(search_dirs=str(paths.templates_dir), escape=lambda u: u)
-        self._swift_dir = swift_dir
-        self._generator = self._get_path(paths.exe_file)
+        self._root_dir = root_dir
+        try:
+            self._generator = self._get_path(paths.exe_file)
+        except ValueError:
+            self._generator = paths.exe_file.name
 
     def _get_path(self, file: pathlib.Path):
-        return file.relative_to(self._swift_dir)
+        return file.relative_to(self._root_dir)
 
     def render(self, data: object, output: pathlib.Path):
         """ Render `data` to `output`.
@@ -60,7 +63,7 @@ def _do_write(self, mnemonic: str, contents: str, output: pathlib.Path):
 
     def manage(self, generated: typing.Iterable[pathlib.Path], stubs: typing.Iterable[pathlib.Path],
                registry: pathlib.Path, force: bool = False) -> "RenderManager":
-        return RenderManager(self._swift_dir, generated, stubs, registry, force)
+        return RenderManager(self._root_dir, generated, stubs, registry, force)
 
 
 class RenderManager(Renderer):
@@ -85,10 +88,10 @@ class Hashes:
         pre: str
         post: typing.Optional[str] = None
 
-    def __init__(self, swift_dir: pathlib.Path, generated: typing.Iterable[pathlib.Path],
+    def __init__(self, root_dir: pathlib.Path, generated: typing.Iterable[pathlib.Path],
                  stubs: typing.Iterable[pathlib.Path],
                  registry: pathlib.Path, force: bool = False):
-        super().__init__(swift_dir)
+        super().__init__(root_dir)
         self._registry_path = registry
         self._force = force
         self._hashes = {}
@@ -117,7 +120,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
                 self._hashes.pop(self._get_path(f), None)
         # clean up the registry from files that do not exist any more
         for f in list(self._hashes):
-            if not (self._swift_dir / f).exists():
+            if not (self._root_dir / f).exists():
                 self._hashes.pop(f)
         self._dump_registry()
 

swift/codegen/test/test_qlgen.py

@@ -17,16 +17,16 @@ def run_mock():
 
 
 # these are lambdas so that they will use patched paths when called
-def stub_path(): return paths.swift_dir / "ql/lib/stub/path"
+def stub_path(): return paths.root_dir / "ql/lib/stub/path"
 
 
-def ql_output_path(): return paths.swift_dir / "ql/lib/other/path"
+def ql_output_path(): return paths.root_dir / "ql/lib/other/path"
 
 
-def ql_test_output_path(): return paths.swift_dir / "ql/test/path"
+def ql_test_output_path(): return paths.root_dir / "ql/test/path"
 
 
-def generated_registry_path(): return paths.swift_dir / "registry.list"
+def generated_registry_path(): return paths.root_dir / "registry.list"
 
 
 def import_file(): return stub_path().with_suffix(".qll")
@@ -47,7 +47,7 @@ def qlgen_opts(opts):
     opts.ql_test_output = ql_test_output_path()
     opts.generated_registry = generated_registry_path()
     opts.ql_format = True
-    opts.swift_dir = paths.swift_dir
+    opts.root_dir = paths.root_dir
     opts.force = False
     return opts