QL: switch to shared YAML extractor

Author: aibaars

ql/Cargo.lock

@@ -10,7 +10,6 @@ edition = "2018"
 tree-sitter = ">= 0.20, < 0.21"
 tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "d08db734f8dc52f6bc04db53a966603122bc6985"}
 tree-sitter-ql-dbscheme = { git = "https://github.com/erik-krogh/tree-sitter-ql-dbscheme.git", rev = "63e1344353f63931e88bfbc2faa2e78e1421b213"}
-tree-sitter-ql-yaml = {git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "cf704bf3671e1ae148e173464fb65a4d2bbf5f99"}
 tree-sitter-blame = {path = "../buramu/tree-sitter-blame"}
 tree-sitter-json = {git = "https://github.com/tausbn/tree-sitter-json.git", rev = "745663ee997f1576fe1e7187e6347e0db36ec7a9"}
 clap = { version = "4.2", features = ["derive"] }

ql/extractor/Cargo.toml

@@ -15,7 +15,7 @@ pub fn run(_: Options) -> std::io::Result<()> {
 
     autobuilder::Autobuilder::new("ql", PathBuf::from(database))
         .include_extensions(&[".ql", ".qll", ".dbscheme", ".json", ".jsonc", ".jsonl"])
-        .include_globs(&["**/qlpack.yml", "deprecated.blame"])
+        .include_globs(&["deprecated.blame"])
         .size_limit("10m")
         .run()
 }

ql/extractor/src/autobuilder.rs

@@ -42,12 +42,6 @@ pub fn run(options: Options) -> std::io::Result<()> {
                 node_types: tree_sitter_ql_dbscheme::NODE_TYPES,
                 file_extensions: vec!["dbscheme".into()],
             },
-            simple::LanguageSpec {
-                prefix: "yaml",
-                ts_language: tree_sitter_ql_yaml::language(),
-                node_types: tree_sitter_ql_yaml::NODE_TYPES,
-                file_extensions: vec!["yml".into()],
-            },
             simple::LanguageSpec {
                 prefix: "json",
                 ts_language: tree_sitter_json::language(),

ql/extractor/src/extractor.rs

@@ -31,10 +31,6 @@ pub fn run(options: Options) -> std::io::Result<()> {
             name: "Dbscheme".to_owned(),
             node_types: tree_sitter_ql_dbscheme::NODE_TYPES,
         },
-        Language {
-            name: "Yaml".to_owned(),
-            node_types: tree_sitter_ql_yaml::NODE_TYPES,
-        },
         Language {
             name: "Blame".to_owned(),
             node_types: tree_sitter_blame::NODE_TYPES,

ql/extractor/src/generator.rs

@@ -38,8 +38,6 @@ class AstNode extends TAstNode {
       result = node.getLocation()
     )
     or
-    result = toGenerateYaml(this).getLocation()
-    or
     result = toDbscheme(this).getLocation()
   }
 
@@ -2573,126 +2571,49 @@ class BindingSet extends Annotation {
  * Classes modeling YAML AST nodes.
  */
 module YAML {
-  /** A node in a YAML file */
-  class YamlNode extends TYamlNode, AstNode {
-    /** Holds if the predicate is a root node (has no parent) */
-    predicate isRoot() { not exists(this.getParent()) }
-
-    override AstNode getParent() { toGenerateYaml(result) = toGenerateYaml(this).getParent() }
-  }
+  private import codeql.yaml.Yaml as LibYaml
 
-  /** DEPRECATED: Alias for YamlNode */
-  deprecated class YAMLNode = YamlNode;
+  private module YamlSig implements LibYaml::InputSig {
+    import codeql.Locations
 
-  /** A YAML comment. */
-  class YamlComment extends TYamlComment, YamlNode {
-    Yaml::Comment yamlcomment;
+    class LocatableBase extends @yaml_locatable {
+      Location getLocation() { yaml_locations(this, result) }
 
-    YamlComment() { this = TYamlComment(yamlcomment) }
-
-    override string getAPrimaryQlClass() { result = "YamlComment" }
-  }
+      string toString() { none() }
+    }
 
-  /** DEPRECATED: Alias for YamlComment */
-  deprecated class YAMLComment = YamlComment;
+    class NodeBase extends LocatableBase, @yaml_node {
+      NodeBase getChildNode(int i) { yaml(result, _, this, i, _, _) }
 
-  /** A YAML entry. */
-  class YamlEntry extends TYamlEntry, YamlNode {
-    Yaml::Entry yamle;
+      string getTag() { yaml(this, _, _, _, result, _) }
 
-    YamlEntry() { this = TYamlEntry(yamle) }
+      string getAnchor() { yaml_anchors(this, result) }
 
-    /** Gets the key of this YAML entry. */
-    YamlKey getKey() {
-      exists(Yaml::Keyvaluepair pair |
-        pair.getParent() = yamle and
-        result = TYamlKey(pair.getKey())
-      )
+      override string toString() { yaml(this, _, _, _, _, result) }
     }
 
-    YamlListItem getListItem() { toGenerateYaml(result).getParent() = yamle }
+    class ScalarNodeBase extends NodeBase, @yaml_scalar_node {
+      int getStyle() { yaml_scalars(this, result, _) }
 
-    /** Gets the value of this YAML entry. */
-    YamlValue getValue() {
-      exists(Yaml::Keyvaluepair pair |
-        pair.getParent() = yamle and
-        result = TYamlValue(pair.getValue())
-      )
+      string getValue() { yaml_scalars(this, _, result) }
     }
 
-    override string getAPrimaryQlClass() { result = "YamlEntry" }
-  }
-
-  /** DEPRECATED: Alias for YamlEntry */
-  deprecated class YAMLEntry = YamlEntry;
-
-  /** A YAML key. */
-  class YamlKey extends TYamlKey, YamlNode {
-    Yaml::Key yamlkey;
+    class CollectionNodeBase extends NodeBase, @yaml_collection_node { }
 
-    YamlKey() { this = TYamlKey(yamlkey) }
+    class MappingNodeBase extends CollectionNodeBase, @yaml_mapping_node { }
 
-    /**
-     * Gets the value of this YAML key.
-     */
-    YamlValue getValue() {
-      exists(Yaml::Keyvaluepair pair |
-        pair.getKey() = yamlkey and result = TYamlValue(pair.getValue())
-      )
-    }
-
-    override string getAPrimaryQlClass() { result = "YamlKey" }
+    class SequenceNodeBase extends CollectionNodeBase, @yaml_sequence_node { }
 
-    /** Gets the value of this YAML value. */
-    string getNamePart(int i) {
-      i = 0 and result = yamlkey.getChild(0).(Yaml::SimpleId).getValue()
-      or
-      exists(YamlKey child |
-        child = TYamlKey(yamlkey.getChild(1)) and
-        result = child.getNamePart(i - 1)
-      )
+    class AliasNodeBase extends NodeBase, @yaml_alias_node {
+      string getTarget() { yaml_aliases(this, result) }
     }
 
-    /**
-     * Gets all the name parts of this YAML key concatenated with `/`.
-     * Dashes are replaced with `/` (because we don't have that information in the generated AST).
-     */
-    string getQualifiedName() {
-      result = concat(string part, int i | part = this.getNamePart(i) | part, "/" order by i)
+    class ParseErrorBase extends LocatableBase, @yaml_error {
+      string getMessage() { yaml_errors(this, result) }
     }
   }
 
-  /** DEPRECATED: Alias for YamlKey */
-  deprecated class YAMLKey = YamlKey;
-
-  /** A YAML list item. */
-  class YamlListItem extends TYamlListitem, YamlNode {
-    Yaml::Listitem yamllistitem;
-
-    YamlListItem() { this = TYamlListitem(yamllistitem) }
-
-    /**
-     * Gets the value of this YAML list item.
-     */
-    YamlValue getValue() { result = TYamlValue(yamllistitem.getChild()) }
-
-    override string getAPrimaryQlClass() { result = "YamlListItem" }
-  }
-
-  /** DEPRECATED: Alias for YamlListItem */
-  deprecated class YAMLListItem = YamlListItem;
-
-  /** A YAML value. */
-  class YamlValue extends TYamlValue, YamlNode {
-    Yaml::Value yamlvalue;
-
-    YamlValue() { this = TYamlValue(yamlvalue) }
-
-    override string getAPrimaryQlClass() { result = "YamlValue" }
-
-    /** Gets the value of this YAML value. */
-    string getValue() { result = yamlvalue.getValue() }
-  }
+  import LibYaml::Make<YamlSig>
 
   // to not expose the entire `File` API on `QlPack`.
   private newtype TQLPack = MKQlPack(File file) { file.getBaseName() = "qlpack.yml" }
@@ -2705,15 +2626,16 @@ module YAML {
 
     QLPack() { this = MKQlPack(file) }
 
-    private string getProperty(string name) {
-      exists(YamlEntry entry |
-        entry.isRoot() and
-        entry.getKey().getQualifiedName() = name and
-        result = entry.getValue().getValue().trim() and
-        entry.getLocation().getFile() = file
+    private YamlValue get(string name) {
+      exists(YamlMapping m |
+        m instanceof YamlDocument and
+        m.getFile() = file and
+        result = m.lookup(name)
       )
     }
 
+    private string getProperty(string name) { result = this.get(name).(YamlScalar).getValue() }
+
     /** Gets the name of this qlpack */
     string getName() { result = this.getProperty("name") }
 
@@ -2728,32 +2650,12 @@ module YAML {
     /** Gets the file that this `QLPack` represents. */
     File getFile() { result = file }
 
-    private predicate isADependency(YamlEntry entry) {
-      exists(YamlEntry deps |
-        deps.getLocation().getFile() = file and entry.getLocation().getFile() = file
-      |
-        deps.isRoot() and
-        deps.getKey().getQualifiedName() = ["dependencies", "libraryPathDependencies"] and
-        entry.getLocation().getStartLine() = 1 + deps.getLocation().getStartLine() and
-        entry.getLocation().getStartColumn() > deps.getLocation().getStartColumn()
-      )
-      or
-      exists(YamlEntry prev | this.isADependency(prev) |
-        prev.getLocation().getFile() = file and
-        entry.getLocation().getFile() = file and
-        entry.getLocation().getStartLine() = 1 + prev.getLocation().getStartLine() and
-        entry.getLocation().getStartColumn() = prev.getLocation().getStartColumn()
-      )
-    }
-
     predicate hasDependency(string name, string version) {
-      exists(YamlEntry entry | this.isADependency(entry) |
-        entry.getKey().getQualifiedName().trim() = name and
-        entry.getValue().getValue() = version
-        or
-        name = entry.getListItem().getValue().getValue().trim() and
-        version = "\"*\""
-      )
+      version = this.get("dependencies").(YamlMapping).lookup(name).(YamlScalar).getValue()
+      or
+      name =
+        this.get("libraryPathDependencies").(YamlCollection).getAChild().(YamlScalar).getValue() and
+      version = "\"*\""
       or
       name = this.getProperty("libraryPathDependencies") and
       version = "\"*\""

ql/ql/src/codeql_ql/ast/Ast.qll

@@ -61,11 +61,6 @@ newtype TAstNode =
   TPredicateExpr(QL::PredicateExpr pe) or
   TAnnotation(QL::Annotation annot) or
   TAnnotationArg(QL::AnnotArg arg) or
-  TYamlComment(Yaml::Comment yc) or
-  TYamlEntry(Yaml::Entry ye) or
-  TYamlKey(Yaml::Key yk) or
-  TYamlListitem(Yaml::Listitem yli) or
-  TYamlValue(Yaml::Value yv) or
   TBuiltinClassless(string ret, string name, string args) { isBuiltinClassless(ret, name, args) } or
   TBuiltinMember(string qual, string ret, string name, string args) {
     isBuiltinMember(qual, ret, name, args)
@@ -87,15 +82,10 @@ class TCall = TPredicateCall or TMemberCall or TNoneCall or TAnyCall;
 
 class TTypeRef = TImport or TModuleExpr or TType;
 
-class TYamlNode = TYamlComment or TYamlEntry or TYamlKey or TYamlListitem or TYamlValue;
-
 class TSignatureExpr = TPredicateExpr or TType or TModuleExpr;
 
 class TComment = TQLDoc or TBlockComment or TLineComment;
 
-/** DEPRECATED: Alias for TYamlNode */
-deprecated class TYAMLNode = TYamlNode;
-
 private QL::AstNode toQLFormula(AST::AstNode n) {
   n = TConjunction(result) or
   n = TDisjunction(result) or
@@ -125,14 +115,6 @@ private QL::AstNode toQLExpr(AST::AstNode n) {
   n = TDontCare(result)
 }
 
-Yaml::AstNode toGenerateYaml(AST::AstNode n) {
-  n = TYamlComment(result) or
-  n = TYamlEntry(result) or
-  n = TYamlKey(result) or
-  n = TYamlListitem(result) or
-  n = TYamlValue(result)
-}
-
 Dbscheme::AstNode toDbscheme(AST::AstNode n) { n = TDBRelation(result) }
 
 /**