Sync upstream ruby

Author: ropwareJB

ruby/downgrades/dc51d416301df12df5b70fbc4338de6cc1f82bfd/old.dbscheme

@@ -0,0 +1,3 @@
+description: Add databaseMetadata relation
+compatibility: full
+databaseMetadata.rel: delete

ruby/downgrades/dc51d416301df12df5b70fbc4338de6cc1f82bfd/ruby.dbscheme

@@ -17,5 +17,6 @@ rayon = "1.10.0"
 regex = "1.11.1"
 encoding = "0.2"
 lazy_static = "1.5.0"
+serde_json = "1.0.140"
 
 codeql-extractor = { path = "../../shared/tree-sitter-extractor" }

ruby/downgrades/dc51d416301df12df5b70fbc4338de6cc1f82bfd/upgrade.properties

@@ -1,7 +1,9 @@
 use clap::Args;
+use codeql_extractor::file_paths::PathTransformer;
 use lazy_static::lazy_static;
 use rayon::prelude::*;
 use std::borrow::Cow;
+use std::collections::HashSet;
 use std::fs;
 use std::io::BufRead;
 use std::path::{Path, PathBuf};
@@ -78,6 +80,9 @@ pub fn run(options: Options) -> std::io::Result<()> {
 
     let file_list = fs::File::open(file_paths::path_from_string(&options.file_list))?;
 
+    let overlay_changed_files: Option<HashSet<PathBuf>> = get_overlay_changed_files();
+    let path_transformer = file_paths::load_path_transformer()?;
+
     let language: Language = tree_sitter_ruby::LANGUAGE.into();
     let erb: Language = tree_sitter_embedded_template::LANGUAGE.into();
     // Look up tree-sitter kind ids now, to avoid string comparisons when scanning ERB files.
@@ -94,7 +99,14 @@ pub fn run(options: Options) -> std::io::Result<()> {
         .try_for_each(|line| {
             let mut diagnostics_writer = diagnostics.logger();
             let path = PathBuf::from(line).canonicalize()?;
-            let src_archive_file = file_paths::path_for(&src_archive_dir, &path, "");
+            match &overlay_changed_files {
+                Some(changed_files) if !changed_files.contains(&path) => {
+                    // We are extracting an overlay and this file is not in the list of changes files, so we should skip it.
+                    return Result::Ok(());
+                }
+                _ => {},
+            }
+            let src_archive_file = file_paths::path_for(&src_archive_dir, &path, "", path_transformer.as_ref());
             let mut source = std::fs::read(&path)?;
             let mut needs_conversion = false;
             let code_ranges;
@@ -107,6 +119,7 @@ pub fn run(options: Options) -> std::io::Result<()> {
                     &erb_schema,
                     &mut diagnostics_writer,
                     &mut trap_writer,
+                    path_transformer.as_ref(),
                     &path,
                     &source,
                     &[],
@@ -151,7 +164,7 @@ pub fn run(options: Options) -> std::io::Result<()> {
                                                     "character-decoding-error",
                                                     "Character decoding error",
                                                 )
-                                                .file(&file_paths::normalize_path(&path))
+                                                .file(&file_paths::normalize_and_transform_path(&path, path_transformer.as_ref()))
                                                 .message(
                                                     "Could not decode the file contents as {}: {}. The contents of the file must match the character encoding specified in the {} {}.",
                                                     &[
@@ -171,7 +184,7 @@ pub fn run(options: Options) -> std::io::Result<()> {
                             diagnostics_writer.write(
                                 diagnostics_writer
                                     .new_entry("unknown-character-encoding", "Could not process some files due to an unknown character encoding")
-                                    .file(&file_paths::normalize_path(&path))
+                                    .file(&file_paths::normalize_and_transform_path(&path, path_transformer.as_ref()))
                                     .message(
                                         "Unknown character encoding {} in {} {}.",
                                         &[
@@ -194,6 +207,7 @@ pub fn run(options: Options) -> std::io::Result<()> {
                 &schema,
                 &mut diagnostics_writer,
                 &mut trap_writer,
+                path_transformer.as_ref(),
                 &path,
                 &source,
                 &code_ranges,
@@ -204,14 +218,26 @@ pub fn run(options: Options) -> std::io::Result<()> {
             } else {
                 std::fs::copy(&path, &src_archive_file)?;
             }
-            write_trap(&trap_dir, path, &trap_writer, trap_compression)
+            write_trap(&trap_dir, path, &trap_writer, trap_compression, path_transformer.as_ref())
         })
         .expect("failed to extract files");
 
     let path = PathBuf::from("extras");
     let mut trap_writer = trap::Writer::new();
     extractor::populate_empty_location(&mut trap_writer);
-    let res = write_trap(&trap_dir, path, &trap_writer, trap_compression);
+    let res = write_trap(
+        &trap_dir,
+        path,
+        &trap_writer,
+        trap_compression,
+        path_transformer.as_ref(),
+    );
+    if let Ok(output_path) = std::env::var("CODEQL_EXTRACTOR_RUBY_OVERLAY_BASE_METADATA_OUT") {
+        // We're extracting an overlay base. For now, we don't have any metadata we need to store
+        // that would get read when extracting the overlay, but the CLI expects us to write
+        // *something*. An empty file will do.
+        std::fs::write(output_path, b"")?;
+    }
     tracing::info!("Extraction complete");
     res
 }
@@ -237,8 +263,14 @@ fn write_trap(
     path: PathBuf,
     trap_writer: &trap::Writer,
     trap_compression: trap::Compression,
+    path_transformer: Option<&PathTransformer>,
 ) -> std::io::Result<()> {
-    let trap_file = file_paths::path_for(trap_dir, &path, trap_compression.extension());
+    let trap_file = file_paths::path_for(
+        trap_dir,
+        &path,
+        trap_compression.extension(),
+        path_transformer,
+    );
     std::fs::create_dir_all(trap_file.parent().unwrap())?;
     trap_writer.write_to_file(&trap_file, trap_compression)
 }
@@ -302,6 +334,39 @@ fn skip_space(content: &[u8], index: usize) -> usize {
     }
     index
 }
+
+/**
+* If the relevant environment variable has been set by the CLI, indicating that we are extracting
+* an overlay, this function reads the JSON file at the path given by its value, and returns a set
+* of canonicalized paths of source files that have changed and should therefore be extracted.
+*
+* If the environment variable is not set (i.e. we're not extracting an overlay), or if the file
+* cannot be read, this function returns `None`. In that case, all files should be extracted.
+*/
+fn get_overlay_changed_files() -> Option<HashSet<PathBuf>> {
+    let path = std::env::var("CODEQL_EXTRACTOR_RUBY_OVERLAY_CHANGES").ok()?;
+    let file_content = fs::read_to_string(path).ok()?;
+    let json_value: serde_json::Value = serde_json::from_str(&file_content).ok()?;
+
+    // The JSON file is expected to have the following structure:
+    // {
+    //     "changes": [
+    //         "relative/path/to/changed/file1.rb",
+    //         "relative/path/to/changed/file2.rb",
+    //         ...
+    //     ]
+    // }
+    Some(
+        json_value
+            .get("changes")?
+            .as_array()?
+            .iter()
+            .filter_map(|change| change.as_str())
+            .filter_map(|s| PathBuf::from(s).canonicalize().ok())
+            .collect(),
+    )
+}
+
 fn scan_coding_comment(content: &[u8]) -> std::option::Option<Cow<str>> {
     let mut index = 0;
     // skip UTF-8 BOM marker if there is one

ruby/extractor/Cargo.toml

@@ -582,12 +582,27 @@ abstract class StringlikeLiteralImpl extends Expr, TStringlikeLiteral {
     )
   }
 
+  pragma[nomagic]
+  private StringComponentImpl getComponentImplRestricted(int n) {
+    result = this.getComponentImpl(n) and
+    strictsum(int length, int i | length = this.getComponentImpl(i).getValue().length() | length) <
+      10000
+  }
+
   // 0 components results in the empty string
-  // if all interpolations have a known string value, we will get a result
+  // if all interpolations have a known string value, we will get a result, unless the
+  // combined length exceeds 10,000 characters
   language[monotonicAggregates]
   final string getStringValue() {
+    not exists(this.getComponentImpl(_)) and
+    result = ""
+    or
     result =
-      concat(StringComponentImpl c, int i | c = this.getComponentImpl(i) | c.getValue() order by i)
+      strictconcat(StringComponentImpl c, int i |
+        c = this.getComponentImplRestricted(i)
+      |
+        c.getValue() order by i
+      )
   }
 }
 

ruby/extractor/src/extractor.rs

@@ -1426,4 +1426,4 @@ predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) {
   ppos.isAnyNamed() and apos.isKeyword(_)
   or
   apos.isAnyNamed() and ppos.isKeyword(_)
-}
+}

ruby/ql/lib/codeql/ruby/ast/internal/Literal.qll

@@ -2560,4 +2560,4 @@ module TypeInference {
   predicate hasModuleType(Node n, DataFlowType t) {
     exists(Module tp | t = TModuleDataFlowType(tp) | hasType(n, tp, _))
   }
-}
+}