Rust: extract source files of depdendencies

Author: aibaars

rust/extractor/src/main.rs

@@ -1,6 +1,6 @@
 use crate::diagnostics::{ExtractionStep, emit_extraction_diagnostics};
 use crate::rust_analyzer::path_to_file_id;
-use crate::translate::ResolvePaths;
+use crate::translate::{ResolvePaths, SourceKind};
 use crate::trap::TrapId;
 use anyhow::Context;
 use archive::Archiver;
@@ -12,6 +12,7 @@ use ra_ap_paths::{AbsPathBuf, Utf8PathBuf};
 use ra_ap_project_model::{CargoConfig, ProjectManifest};
 use ra_ap_vfs::Vfs;
 use rust_analyzer::{ParseResult, RustAnalyzer};
+use std::collections::HashSet;
 use std::time::Instant;
 use std::{
     collections::HashMap,
@@ -47,9 +48,14 @@ impl<'a> Extractor<'a> {
         }
     }
 
-    fn extract(&mut self, rust_analyzer: &RustAnalyzer, file: &Path, resolve_paths: ResolvePaths) {
+    fn extract(
+        &mut self,
+        rust_analyzer: &RustAnalyzer,
+        file: &Path,
+        resolve_paths: ResolvePaths,
+        source_kind: SourceKind,
+    ) {
         self.archiver.archive(file);
-
         let before_parse = Instant::now();
         let ParseResult {
             ast,
@@ -71,6 +77,7 @@ impl<'a> Extractor<'a> {
             line_index,
             semantics_info.as_ref().ok(),
             resolve_paths,
+            source_kind,
         );
 
         for err in errors {
@@ -110,15 +117,27 @@ impl<'a> Extractor<'a> {
         semantics: &Semantics<'_, RootDatabase>,
         vfs: &Vfs,
         resolve_paths: ResolvePaths,
+        source_kind: SourceKind,
     ) {
-        self.extract(&RustAnalyzer::new(vfs, semantics), file, resolve_paths);
+        self.extract(
+            &RustAnalyzer::new(vfs, semantics),
+            file,
+            resolve_paths,
+            source_kind,
+        );
     }
 
-    pub fn extract_without_semantics(&mut self, file: &Path, reason: &str) {
+    pub fn extract_without_semantics(
+        &mut self,
+        file: &Path,
+        source_kind: SourceKind,
+        reason: &str,
+    ) {
         self.extract(
             &RustAnalyzer::WithoutSemantics { reason },
             file,
             ResolvePaths::No,
+            source_kind,
         );
     }
 
@@ -246,7 +265,7 @@ fn main() -> anyhow::Result<()> {
                 continue 'outer;
             }
         }
-        extractor.extract_without_semantics(file, "no manifest found");
+        extractor.extract_without_semantics(file, SourceKind::Source, "no manifest found");
     }
     let cwd = cwd()?;
     let (cargo_config, load_cargo_config) = cfg.to_cargo_config(&cwd);
@@ -255,6 +274,7 @@ fn main() -> anyhow::Result<()> {
     } else {
         ResolvePaths::Yes
     };
+    let mut processed_files = HashSet::new();
     for (manifest, files) in map.values().filter(|(_, files)| !files.is_empty()) {
         if let Some((ref db, ref vfs)) =
             extractor.load_manifest(manifest, &cargo_config, &load_cargo_config)
@@ -266,16 +286,43 @@ fn main() -> anyhow::Result<()> {
                 .push(ExtractionStep::crate_graph(before_crate_graph));
             let semantics = Semantics::new(db);
             for file in files {
+                processed_files.insert((*file).to_owned());
                 match extractor.load_source(file, &semantics, vfs) {
-                    Ok(()) => {
-                        extractor.extract_with_semantics(file, &semantics, vfs, resolve_paths)
+                    Ok(()) => extractor.extract_with_semantics(
+                        file,
+                        &semantics,
+                        vfs,
+                        resolve_paths,
+                        SourceKind::Source,
+                    ),
+                    Err(reason) => {
+                        extractor.extract_without_semantics(file, SourceKind::Source, &reason)
                     }
-                    Err(reason) => extractor.extract_without_semantics(file, &reason),
                 };
             }
+            for (_, file) in vfs.iter() {
+                if let Some(file) = file.as_path().map(<_ as AsRef<Path>>::as_ref) {
+                    if file.extension().is_some_and(|ext| ext == "rs")
+                        && processed_files.insert(file.to_owned())
+                    {
+                        extractor.extract_with_semantics(
+                            file,
+                            &semantics,
+                            vfs,
+                            resolve_paths,
+                            SourceKind::Library,
+                        );
+                        extractor.archiver.archive(file);
+                    }
+                }
+            }
         } else {
             for file in files {
-                extractor.extract_without_semantics(file, "unable to load manifest");
+                extractor.extract_without_semantics(
+                    file,
+                    SourceKind::Source,
+                    "unable to load manifest",
+                );
             }
         }
     }
@@ -286,7 +333,7 @@ fn main() -> anyhow::Result<()> {
         let entry = entry.context("failed to read builtins directory")?;
         let path = entry.path();
         if path.extension().is_some_and(|ext| ext == "rs") {
-            extractor.extract_without_semantics(&path, "");
+            extractor.extract_without_semantics(&path, SourceKind::Library, "");
         }
     }
 

rust/extractor/src/translate.rs

@@ -2,4 +2,4 @@ mod base;
 mod generated;
 mod mappings;
 
-pub use base::{ResolvePaths, Translator};
+pub use base::{ResolvePaths, SourceKind, Translator};

rust/extractor/src/translate/base.rs

@@ -16,7 +16,7 @@ use ra_ap_ide_db::RootDatabase;
 use ra_ap_ide_db::line_index::{LineCol, LineIndex};
 use ra_ap_parser::SyntaxKind;
 use ra_ap_span::TextSize;
-use ra_ap_syntax::ast::HasName;
+use ra_ap_syntax::ast::{Const, Fn, HasName, Static};
 use ra_ap_syntax::{
     AstNode, NodeOrToken, SyntaxElementChildren, SyntaxError, SyntaxNode, SyntaxToken, TextRange,
     ast,
@@ -93,6 +93,11 @@ pub enum ResolvePaths {
     Yes,
     No,
 }
+#[derive(PartialEq, Eq)]
+pub enum SourceKind {
+    Source,
+    Library,
+}
 
 pub struct Translator<'a> {
     pub trap: TrapFile,
@@ -102,6 +107,7 @@ pub struct Translator<'a> {
     file_id: Option<EditionedFileId>,
     pub semantics: Option<&'a Semantics<'a, RootDatabase>>,
     resolve_paths: ResolvePaths,
+    source_kind: SourceKind,
 }
 
 const UNKNOWN_LOCATION: (LineCol, LineCol) =
@@ -115,6 +121,7 @@ impl<'a> Translator<'a> {
         line_index: LineIndex,
         semantic_info: Option<&FileSemanticInformation<'a>>,
         resolve_paths: ResolvePaths,
+        source_kind: SourceKind,
     ) -> Translator<'a> {
         Translator {
             trap,
@@ -124,6 +131,7 @@ impl<'a> Translator<'a> {
             file_id: semantic_info.map(|i| i.file_id),
             semantics: semantic_info.map(|i| i.semantics),
             resolve_paths,
+            source_kind,
         }
     }
     fn location(&self, range: TextRange) -> Option<(LineCol, LineCol)> {
@@ -612,6 +620,31 @@ impl<'a> Translator<'a> {
     }
 
     pub(crate) fn should_be_excluded(&self, item: &impl ast::HasAttrs) -> bool {
+        if self.source_kind == SourceKind::Library {
+            let syntax = item.syntax();
+            if let Some(body) = syntax.parent().and_then(Fn::cast).and_then(|x| x.body()) {
+                if body.syntax() == syntax {
+                    tracing::debug!("Skipping Fn body");
+                    return true;
+                }
+            }
+            if let Some(body) = syntax.parent().and_then(Const::cast).and_then(|x| x.body()) {
+                if body.syntax() == syntax {
+                    tracing::debug!("Skipping Const body");
+                    return true;
+                }
+            }
+            if let Some(body) = syntax
+                .parent()
+                .and_then(Static::cast)
+                .and_then(|x| x.body())
+            {
+                if body.syntax() == syntax {
+                    tracing::debug!("Skipping Static body");
+                    return true;
+                }
+            }
+        }
         self.semantics.is_some_and(|sema| {
             item.attrs().any(|attr| {
                 attr.as_simple_call().is_some_and(|(name, tokens)| {