diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/ScipJava.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/ScipJava.scala index 92b1d111e..59191cfb3 100644 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/ScipJava.scala +++ b/scip-java/src/main/scala/com/sourcegraph/scip_java/ScipJava.scala @@ -6,7 +6,6 @@ import com.sourcegraph.scip_java.commands.IndexCommand import com.sourcegraph.scip_java.commands.IndexDependencyCommand import com.sourcegraph.scip_java.commands.IndexSemanticdbCommand import com.sourcegraph.scip_java.commands.SnapshotCommand -import com.sourcegraph.scip_java.commands.SnapshotLsifCommand import moped.cli.Application import moped.cli.CommandParser import moped.commands.HelpCommand @@ -23,8 +22,7 @@ object ScipJava { CommandParser[IndexCommand], CommandParser[IndexSemanticdbCommand], CommandParser[IndexDependencyCommand], - CommandParser[SnapshotCommand], - CommandParser[SnapshotLsifCommand] + CommandParser[SnapshotCommand] ) ) def main(args: Array[String]): Unit = { diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BuildTool.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BuildTool.scala index 778002ae7..49f86df20 100644 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BuildTool.scala +++ b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BuildTool.scala @@ -46,8 +46,7 @@ object BuildTool { def generateScipFromTargetroot( generateSemanticdbResult: CommandResult, targetroot: Path, - index: IndexCommand, - buildKind: String = "" + index: IndexCommand ): Int = { if (!Files.isDirectory(targetroot)) { generateSemanticdbResult.exitCode @@ -62,7 +61,6 @@ object BuildTool { output = index.finalOutput, targetroot = List(targetroot), packagehub = index.packagehub, - buildKind = buildKind, app = index.app ) .run() diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala index db2c0ca14..090612a5d 100644 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala +++ b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala @@ -105,8 +105,7 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) { BuildTool.generateScipFromTargetroot( generateSemanticdb(), index.finalTargetroot(defaultTargetroot), - index, - buildKind + index ) } @@ -117,7 +116,6 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) { ScipBuildTool .ConfigFileNames .map(name => index.workingDirectory.resolve(name)) - private def buildKind: String = parsedConfig.fold(_.kind, _ => "") private def generateSemanticdb(): CommandResult = { parsedConfig match { case ValueResult(value) => diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/IndexSemanticdbCommand.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/IndexSemanticdbCommand.scala index b8e09fb4f..194c1a85c 100644 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/IndexSemanticdbCommand.scala +++ b/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/IndexSemanticdbCommand.scala @@ -6,8 +6,8 @@ import java.util.concurrent.TimeUnit import scala.jdk.CollectionConverters._ +import com.sourcegraph.Scip import com.sourcegraph.io.AbsolutePath -import com.sourcegraph.lsif_protocol.LsifToolInfo import com.sourcegraph.scip_java.BuildInfo import com.sourcegraph.scip_java.buildtools.ClasspathEntry import com.sourcegraph.scip_semanticdb.ConsoleScipSemanticdbReporter @@ -47,8 +47,6 @@ final case class IndexSemanticdbCommand( @Description("Directories that contain SemanticDB files.") @PositionalArguments() targetroot: List[Path] = Nil, - @Description("The kind of this build, one of: empty string, jdk, maven") - buildKind: String = "", @Description( "If true, don't report an error when no documents have been indexed. " + "The resulting SCIP index will silently be empty instead." @@ -79,7 +77,7 @@ final case class IndexSemanticdbCommand( if (format == ScipOutputFormat.UNKNOWN) { app.error( s"unknown output format for filename '$outputFilename'. " + - s"Supported file extension are `*.scip`, `*scip '" + s"Supported file extensions are `*.scip` and `*.scip.ndjson`" ) return 1 } @@ -97,16 +95,15 @@ final case class IndexSemanticdbCommand( AbsolutePath.of(output, sourceroot), sourceroot, reporter, - LsifToolInfo + Scip + .ToolInfo .newBuilder() .setName("scip-java") .setVersion(BuildInfo.version) .build(), - "java", format, parallel, packages.map(_.toPackageInformation).asJava, - buildKind, emitInverseRelationships, allowEmptyIndex, allowExportingGlobalSymbolsFromDirectoryEntries diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/SnapshotLsifCommand.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/SnapshotLsifCommand.scala deleted file mode 100644 index 8bea91b16..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/SnapshotLsifCommand.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.sourcegraph.scip_java.commands - -import java.nio.file.Path -import java.nio.file.Paths - -import moped.annotations.CommandName -import moped.annotations.Inline -import moped.annotations.PositionalArguments -import moped.cli.Application -import moped.cli.Command -import moped.cli.CommandParser - -@CommandName("snapshot-lsif") -case class SnapshotLsifCommand( - @Inline() - app: Application = Application.default, - @PositionalArguments() - input: List[Path] = List(Paths.get("dump.lsif")) -) extends Command { - def run(): Int = { - app.error( - "this command is no longer supported. To run this command, downgrade to scip-java v0.8.14 and try again." - ) - 1 - } - -} - -object SnapshotLsifCommand { - val default = SnapshotLsifCommand() - implicit val parser = CommandParser.derive(default) -} diff --git a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/BazelBuildTool.java b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/BazelBuildTool.java index 55ffc757e..e5fb161f3 100644 --- a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/BazelBuildTool.java +++ b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/BazelBuildTool.java @@ -1,7 +1,7 @@ package com.sourcegraph.scip_semanticdb; import com.sourcegraph.scip_java.Bazelbuild; -import com.sourcegraph.lsif_protocol.LsifToolInfo; +import com.sourcegraph.Scip; import java.io.*; import java.nio.file.FileSystems; @@ -58,12 +58,10 @@ public boolean hasErrors() { options.output, options.sourceroot, reporter, - LsifToolInfo.newBuilder().setName("scip-java").setVersion("HEAD").build(), - "java", + Scip.ToolInfo.newBuilder().setName("scip-java").setVersion("HEAD").build(), ScipOutputFormat.TYPED_PROTOBUF, options.parallel, mavenPackages, - /* buildKind */ "", /* emitInverseRelationships */ true, /* allowEmptyIndex */ true, /* indexDirectoryEntries */ false // because Bazel only compiles to jar files. diff --git a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/MarkupContent.java b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/MarkupContent.java deleted file mode 100644 index 47614bc4d..000000000 --- a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/MarkupContent.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.sourcegraph.scip_semanticdb; - -import com.sourcegraph.lsif_protocol.MarkupKind; - -public class MarkupContent { - public final String value; - public final MarkupKind kind; - - public MarkupContent(MarkupKind kind, String value) { - this.value = value; - this.kind = kind; - } -} diff --git a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/PackageTable.java b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/PackageTable.java index 9f3f6b40b..ee918f177 100644 --- a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/PackageTable.java +++ b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/PackageTable.java @@ -8,23 +8,17 @@ import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; import java.util.jar.JarEntry; import java.util.jar.JarFile; -import java.util.regex.Pattern; -public class PackageTable implements Function { +public class PackageTable { private final Map byClassfile = new HashMap<>(); private final Set cachedJdkSymbols = new HashSet<>(); - private final Map scip = new ConcurrentHashMap<>(); private final JavaVersion javaVersion; - private final ScipWriter writer; private final boolean indexDirectoryEntries; private static final PathMatcher CLASS_PATTERN = @@ -32,8 +26,7 @@ public class PackageTable implements Function { private static final PathMatcher JAR_PATTERN = FileSystems.getDefault().getPathMatcher("glob:**.jar"); - public PackageTable(ScipSemanticdbOptions options, ScipWriter writer) throws IOException { - this.writer = writer; + public PackageTable(ScipSemanticdbOptions options) throws IOException { this.javaVersion = new JavaVersion(); this.indexDirectoryEntries = options.allowExportingGlobalSymbolsFromDirectoryEntries; // NOTE: it's important that we index the JDK before maven packages. Some maven packages @@ -46,11 +39,6 @@ public PackageTable(ScipSemanticdbOptions options, ScipWriter writer) throws IOE } } - public void writeMonikerPackage(int monikerId, Package pkg) { - int pkgId = scip.computeIfAbsent(pkg, this); - writer.emitPackageInformationEdge(monikerId, pkgId); - } - public Optional packageForSymbol(String symbol) { return SymbolDescriptor.toplevel(symbol) .flatMap( @@ -148,9 +136,4 @@ private void indexBootstrapClasspath() throws IOException { } } } - - @Override - public Integer apply(Package pkg) { - return writer.emitpackageinformationVertex(pkg); - } } diff --git a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ResultIds.java b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ResultIds.java deleted file mode 100644 index 6a3d6a2ff..000000000 --- a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ResultIds.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.sourcegraph.scip_semanticdb; - -/** Utility to deal with a group of IDs for a result set, definition result and reference result. */ -public class ResultIds { - public final int resultSet; - public final int definitionResult; - public final int referenceResult; - - public ResultIds(int resultSet, int definitionResult, int referenceResult) { - this.resultSet = resultSet; - this.definitionResult = definitionResult; - this.referenceResult = referenceResult; - } - - public boolean isDefinitionDefined() { - return definitionResult > 0; - } -} diff --git a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ResultSets.java b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ResultSets.java deleted file mode 100644 index 1afdfca29..000000000 --- a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ResultSets.java +++ /dev/null @@ -1,69 +0,0 @@ -package com.sourcegraph.scip_semanticdb; - -import com.sourcegraph.semanticdb_javac.SemanticdbSymbols; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; - -/** Utility to deal with the creation of result sets. */ -public class ResultSets implements Function { - - private final ScipWriter writer; - private final Map globals; - private final HashMap locals; - private final Set exportedSymbols; - private final Set localDefinitions; - private final PackageTable packages; - private final boolean isJdkRepo; - - public ResultSets( - ScipWriter writer, - Map globals, - Set exportedSymbols, - Set localDefinitions, - PackageTable packages, - ScipSemanticdbOptions options) { - this.writer = writer; - this.globals = globals; - this.exportedSymbols = exportedSymbols; - this.localDefinitions = localDefinitions; - this.packages = packages; - this.isJdkRepo = options.buildKind.equals("jdk"); - locals = new HashMap<>(); - } - - public ResultIds getOrInsertResultSet(String symbol) { - boolean isLocal = SemanticdbSymbols.isLocal(symbol); - Map cache = isLocal ? locals : globals; - return cache.computeIfAbsent(symbol, this); - } - - @Override - public ResultIds apply(String symbol) { - boolean isExportedSymbol = exportedSymbols.contains(symbol); - boolean hasDefinitionResult = isExportedSymbol || localDefinitions.contains(symbol); - int resultSet = writer.emitResultSet(); - - // Moniker - Optional pkg = packages.packageForSymbol(symbol); - if (pkg.isPresent() && pkg.get() instanceof JdkPackage && !isJdkRepo) { - // Never export monikers for the JDK repo unless we're indexing the JDK repo. - // Some Maven packages contain sources that redefine symbols like `java/lang/String#` - // even if the the jar files don't contain `java/lang/String.class`. For example, - // see the package com.google.gwt:gwt-user:2.9.0. - // Related issue: https://github.com/sourcegraph/sourcegraph/issues/21058 - isExportedSymbol = false; - } - int monikerId = writer.emitMonikerVertex(symbol, hasDefinitionResult); - writer.emitMonikerEdge(resultSet, monikerId); - if (pkg.isPresent()) { - packages.writeMonikerPackage(monikerId, pkg.get()); - } - - int definitionId = hasDefinitionResult ? writer.emitDefinitionResult(resultSet) : -1; - - return new ResultIds(resultSet, definitionId, writer.emitReferenceResult(resultSet)); - } -} diff --git a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipByteOutputStream.java b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipByteOutputStream.java deleted file mode 100644 index 60dc97c11..000000000 --- a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipByteOutputStream.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.sourcegraph.scip_semanticdb; - -import java.io.ByteArrayOutputStream; -import java.io.OutputStreamWriter; - -/** Wrapper around a ByteArrayOutputStream and OutputStreamWriter. */ -public class ScipByteOutputStream { - public final ByteArrayOutputStream output; - public final OutputStreamWriter writer; - - public ScipByteOutputStream() { - this.output = new ByteArrayOutputStream(); - this.writer = new OutputStreamWriter(output); - } -} diff --git a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipOutputFormat.java b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipOutputFormat.java index 52b7faa34..29ae00934 100644 --- a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipOutputFormat.java +++ b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipOutputFormat.java @@ -1,28 +1,16 @@ package com.sourcegraph.scip_semanticdb; -/** - * Whether to generate index.scip (JSON) or index.scip-protobuf (Protobuf). - * - *

The Protobuf format is experimental and currently only exists as a proof-of-concept. - */ +/** Whether to generate index.scip (Protobuf) or index.scip.ndjson (newline-delimited JSON). */ public enum ScipOutputFormat { - GRAPH_NDJSON, - GRAPH_PROTOBUF, TYPED_PROTOBUF, TYPED_NDJSON, UNKNOWN; - public boolean isTyped() { - return this == TYPED_NDJSON || this == TYPED_PROTOBUF; - } - public boolean isNewlineDelimitedJSON() { - return this == GRAPH_NDJSON || this == TYPED_NDJSON; + return this == TYPED_NDJSON; } public static ScipOutputFormat fromFilename(String name) { - if (name.endsWith(".lsif")) return GRAPH_NDJSON; - if (name.endsWith(".lsif-protobuf")) return GRAPH_PROTOBUF; if (name.endsWith(".scip")) return TYPED_PROTOBUF; if (name.endsWith(".scip.ndjson")) return TYPED_NDJSON; return UNKNOWN; diff --git a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipOutputStream.java b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipOutputStream.java index 3eaf6af56..26cef44b8 100644 --- a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipOutputStream.java +++ b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipOutputStream.java @@ -1,42 +1,24 @@ package com.sourcegraph.scip_semanticdb; -import com.google.protobuf.Descriptors.FieldDescriptor; -import com.google.protobuf.util.JsonFormat; -import com.google.protobuf.util.JsonFormat.Printer; -import com.sourcegraph.lsif_protocol.LsifObject; -import com.sourcegraph.lsif_protocol.LsifPosition; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; -import java.util.HashSet; -import java.util.Set; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.atomic.AtomicBoolean; -/** Low-level methods to write raw SCIP objects into the output stream. */ +/** Low-level methods to write raw SCIP bytes into the output stream. */ public class ScipOutputStream { - private final ScipSemanticdbOptions options; + private final ScipOutputFormat format; private final OutputStream out; private final ConcurrentLinkedDeque buffer; private final AtomicBoolean isFlushing; private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8); - private final ThreadLocal baos = - ThreadLocal.withInitial(ScipByteOutputStream::new); - private final Printer jsonPrinter; - public ScipOutputStream(ScipSemanticdbOptions options, OutputStream out) { - this.options = options; + public ScipOutputStream(ScipOutputFormat format, OutputStream out) { + this.format = format; this.out = out; - buffer = new ConcurrentLinkedDeque<>(); - isFlushing = new AtomicBoolean(false); - - Set fieldsToAlwaysInclude = - new HashSet<>(LsifPosition.getDescriptor().getFields()); - fieldsToAlwaysInclude.add(LsifObject.getDescriptor().findFieldByName("id")); - jsonPrinter = - JsonFormat.printer() - .includingDefaultValueFields(fieldsToAlwaysInclude) - .omittingInsignificantWhitespace(); + this.buffer = new ConcurrentLinkedDeque<>(); + this.isFlushing = new AtomicBoolean(false); } public void write(byte[] bytes) { @@ -44,32 +26,12 @@ public void write(byte[] bytes) { buffer.add(bytes); } - public void writeLsifObject(LsifObject.Builder object) { - ScipByteOutputStream b = baos.get(); - b.output.reset(); - try { - switch (options.format) { - case GRAPH_PROTOBUF: - object.buildPartial().writeTo(b.output); - break; - case GRAPH_NDJSON: - default: - jsonPrinter.appendTo(object, b.writer); - b.writer.flush(); - break; - } - } catch (IOException e) { - options.reporter.error(e); - } - write(b.output.toByteArray()); - } - public void flush() throws IOException { if (isFlushing.compareAndSet(false, true)) { byte[] bytes = buffer.poll(); while (bytes != null) { out.write(bytes); - if (options.format.isNewlineDelimitedJSON()) { + if (format.isNewlineDelimitedJSON()) { out.write(NEWLINE); } bytes = buffer.poll(); diff --git a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipProcessingException.java b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipProcessingException.java deleted file mode 100644 index 79ceb717e..000000000 --- a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipProcessingException.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.sourcegraph.scip_semanticdb; - -public class ScipProcessingException extends Throwable { - - public ScipProcessingException(ScipTextDocument doc, Throwable cause) { - super(doc.semanticdbPath.toString(), cause); - } - - @Override - public Throwable fillInStackTrace() { - return this; - } -} diff --git a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipSemanticdb.java b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipSemanticdb.java index 3bd4c67b6..f9cba6108 100644 --- a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipSemanticdb.java +++ b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipSemanticdb.java @@ -1,7 +1,6 @@ package com.sourcegraph.scip_semanticdb; import com.google.protobuf.CodedInputStream; -import com.sourcegraph.lsif_protocol.MarkupKind; import com.sourcegraph.semanticdb_javac.Semanticdb; import com.sourcegraph.semanticdb_javac.Semanticdb.SymbolInformation; import com.sourcegraph.semanticdb_javac.Semanticdb.SymbolInformation.Kind; @@ -16,7 +15,7 @@ import java.net.URI; import java.nio.file.*; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; + import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.stream.Collectors; @@ -27,12 +26,10 @@ public class ScipSemanticdb { private final ScipWriter writer; private final ScipSemanticdbOptions options; - private final Map globals; public ScipSemanticdb(ScipWriter writer, ScipSemanticdbOptions options) { this.writer = writer; this.options = options; - this.globals = new ConcurrentHashMap<>(); } public static void run(ScipSemanticdbOptions options) throws IOException { @@ -41,7 +38,7 @@ public static void run(ScipSemanticdbOptions options) throws IOException { } private void run() throws IOException { - PackageTable packages = new PackageTable(options, writer); + PackageTable packages = new PackageTable(options); List files = SemanticdbWalker.findSemanticdbFiles(options); Collections.sort(files); if (options.reporter.hasErrors()) return; @@ -56,11 +53,7 @@ private void run() throws IOException { return; } options.reporter.startProcessing(files.size()); - if (options.format.isTyped()) { - runTyped(files, packages); - } else { - runGraph(files, packages); - } + runTyped(files, packages); writer.build(); options.reporter.endProcessing(); } @@ -323,22 +316,10 @@ private Scip.Index typedMetadata() { Scip.ToolInfo.newBuilder() .setName(options.toolInfo.getName()) .setVersion(options.toolInfo.getVersion()) - .addAllArguments(options.toolInfo.getArgsList()))) + .addAllArguments(options.toolInfo.getArgumentsList()))) .build(); } - private void runGraph(List files, PackageTable packages) { - writer.emitMetaData(); - int projectId = writer.emitProject(options.language); - - Set isExportedSymbol = exportSymbols(files); - List documentIds = - filesStream(files) - .flatMap(d -> processPath(d, isExportedSymbol, packages)) - .collect(Collectors.toList()); - writer.emitContains(projectId, documentIds); - } - private Stream filesStream(List files) { return options.parallel ? files.parallelStream() : files.stream(); } @@ -387,139 +368,6 @@ private Stream referenceRelationships(ScipTextDocument docum return relationships.stream(); } - private Set exportSymbols(List files) { - return filesStream(files) - .flatMap(this::parseTextDocument) - .flatMap( - d -> - d.semanticdb.getOccurrencesList().stream() - .filter(occ -> isDefinitionRole(occ.getRole())) - .map(SymbolOccurrence::getSymbol) - .filter(SemanticdbSymbols::isGlobal)) - .collect(Collectors.toSet()); - } - - private Stream processPath( - Path path, Set isExportedSymbol, PackageTable packages) { - return parseTextDocument(path).flatMap(d -> processDocument(d, isExportedSymbol, packages)); - } - - private Stream processDocument( - ScipTextDocument doc, Set isExportedSymbol, PackageTable packages) { - try { - return Stream.of(processDocumentUnsafe(doc, isExportedSymbol, packages)); - } catch (Exception e) { - options.reporter.error(new ScipProcessingException(doc, e)); - return Stream.empty(); - } - } - - private Integer processDocumentUnsafe( - ScipTextDocument doc, Set isExportedSymbol, PackageTable packages) { - Symtab symtab = new Symtab(doc.semanticdb); - - int documentId = writer.emitDocument(doc); - Set localDefinitions = - doc.semanticdb.getOccurrencesList().stream() - .filter( - occ -> - isDefinitionRole(occ.getRole()) && SemanticdbSymbols.isLocal(occ.getSymbol())) - .map(SymbolOccurrence::getSymbol) - .collect(Collectors.toSet()); - doc.id = documentId; - ResultSets results = - new ResultSets(writer, globals, isExportedSymbol, localDefinitions, packages, options); - Set rangeIds = new LinkedHashSet<>(); - - for (SymbolOccurrence occ : doc.sortedSymbolOccurrences()) { - for (String symbol : occ.getSymbol().split(";")) { - SymbolInformation symbolInformation = - doc.symbols.getOrDefault(symbol, SymbolInformation.getDefaultInstance()); - ResultIds ids = results.getOrInsertResultSet(symbol); - int rangeId = writer.emitRange(occ.getRange()); - rangeIds.add(rangeId); - - // Range - if (occ.getRole() != Role.SYNTHETIC_DEFINITION) { - writer.emitNext(rangeId, ids.resultSet); - } - - // Reference - writer.emitItem(ids.referenceResult, rangeId, doc.id); - - // Definition - if (isDefinitionRole(occ.getRole())) { - if (ids.isDefinitionDefined()) { - writer.emitItem(ids.definitionResult, rangeId, doc.id); - } else { - options.reporter.error( - new NoSuchElementException( - String.format("no definition ID for symbol '%s'", symbol))); - } - - // Hover 1: signature - String documentation = symbolInformation.getDocumentation().getMessage(); - StringBuilder markupContent = new StringBuilder(documentation.length()); - if (symbolInformation.hasSignature()) { - String language = - doc.semanticdb.getLanguage().toString().toLowerCase(Locale.ROOT).intern(); - String signature = new SignatureFormatter(symbolInformation, symtab).formatSymbol(); - markupContent - .append("```") - .append(language) - .append('\n') - .append(signature) - .append("\n```"); - } - - // Hover 2: docstring - if (!documentation.isEmpty()) { - if (markupContent.length() != 0) markupContent.append("\n---\n"); - markupContent.append(documentation.replaceAll("\n", "\n\n")); - } - - if (markupContent.length() == 0) { - // Always emit a non-empty hover message to prevent Sourcegraph from falling - // back to - // Search-Based hover messages. - markupContent.append(symbolInformation.getDisplayName()); - } - - int hoverId = - writer.emitHoverResult( - new MarkupContent(MarkupKind.MARKDOWN, markupContent.toString())); - writer.emitHoverEdge(ids.resultSet, hoverId); - } - - // Overrides - if (symbolInformation.getOverriddenSymbolsCount() > 0 - && supportsReferenceRelationship(symbolInformation) - && occ.getRole() == Role.DEFINITION) { - List overriddenReferenceResultIds = - new ArrayList<>(symbolInformation.getOverriddenSymbolsCount()); - for (int i = 0; i < symbolInformation.getOverriddenSymbolsCount(); i++) { - String overriddenSymbol = symbolInformation.getOverriddenSymbols(i); - if (isIgnoredOverriddenSymbol(overriddenSymbol)) { - continue; - } - ResultIds overriddenIds = results.getOrInsertResultSet(overriddenSymbol); - overriddenReferenceResultIds.add(overriddenIds.referenceResult); - writer.emitReferenceResultsItemEdge( - overriddenIds.referenceResult, Collections.singletonList(rangeId), doc.id); - } - if (overriddenReferenceResultIds.size() > 0) { - writer.emitReferenceResultsItemEdge( - ids.referenceResult, overriddenReferenceResultIds, doc.id); - } - } - } - } - writer.emitContains(doc.id, new ArrayList<>(rangeIds)); - writer.flush(); - options.reporter.processedOneItem(); - return documentId; - } - private static boolean supportsReferenceRelationship(SymbolInformation info) { switch (info.getKind()) { case INTERFACE: diff --git a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipSemanticdbOptions.java b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipSemanticdbOptions.java index 515502d08..e598fe495 100644 --- a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipSemanticdbOptions.java +++ b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipSemanticdbOptions.java @@ -1,6 +1,6 @@ package com.sourcegraph.scip_semanticdb; -import com.sourcegraph.lsif_protocol.LsifToolInfo; +import com.sourcegraph.Scip; import java.nio.file.Path; import java.util.List; @@ -11,12 +11,10 @@ public class ScipSemanticdbOptions { public final Path output; public final Path sourceroot; public final ScipSemanticdbReporter reporter; - public final LsifToolInfo toolInfo; - public final String language; + public final Scip.ToolInfo toolInfo; public final ScipOutputFormat format; public final boolean parallel; public final List packages; - public final String buildKind; public final boolean emitInverseRelationships; public final boolean allowEmptyIndex; public final boolean allowExportingGlobalSymbolsFromDirectoryEntries; @@ -26,12 +24,10 @@ public ScipSemanticdbOptions( Path output, Path sourceroot, ScipSemanticdbReporter reporter, - LsifToolInfo toolInfo, - String language, + Scip.ToolInfo toolInfo, ScipOutputFormat format, boolean parallel, List packages, - String buildKind, boolean emitInverseRelationships, boolean allowEmptyIndex, boolean allowExportingGlobalSymbolsFromDirectoryEntries) { @@ -40,11 +36,9 @@ public ScipSemanticdbOptions( this.sourceroot = sourceroot; this.reporter = reporter; this.toolInfo = toolInfo; - this.language = language; this.format = format; this.parallel = parallel; this.packages = packages; - this.buildKind = buildKind; this.emitInverseRelationships = emitInverseRelationships; this.allowEmptyIndex = allowEmptyIndex; this.allowExportingGlobalSymbolsFromDirectoryEntries = diff --git a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipTextDocument.java b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipTextDocument.java index 3b53723b2..e9958383d 100644 --- a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipTextDocument.java +++ b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipTextDocument.java @@ -10,9 +10,6 @@ public class ScipTextDocument { public final Path semanticdbPath; public Semanticdb.TextDocument semanticdb; - public int id; - public final Map symbols; - public final Map localSymbols; // Map from symbols that have a definition occurrence to the list of symbols that have // `is_definition` relationships to that symbol. // This map is used to add `is_reference` relationships between all symbols in the list so that @@ -25,22 +22,13 @@ public class ScipTextDocument { public ScipTextDocument( Path semanticdbPath, Semanticdb.TextDocument semanticdb, Path sourceroot) { this.semanticdbPath = semanticdbPath; - this.symbols = new HashMap<>(); - this.localSymbols = new HashMap<>(); String uri = sourceroot.resolve(semanticdb.getUri()).toUri().toString(); setSemanticdb(Semanticdb.TextDocument.newBuilder(semanticdb).setUri(uri).build()); } @Override public String toString() { - return "ScipDocument{" - + "path=" - + semanticdbPath - + ", semanticdb=" - + semanticdb - + ", id=" - + id - + '}'; + return "ScipDocument{" + "path=" + semanticdbPath + ", semanticdb=" + semanticdb + '}'; } public List sortedSymbolOccurrences() { @@ -82,9 +70,6 @@ void visitIdTree(Semanticdb.IdTree tree) { private void setSemanticdb(Semanticdb.TextDocument semanticdb) { this.semanticdb = manifestOccurrencesForSyntheticSymbols(semanticdb); - for (Semanticdb.SymbolInformation info : semanticdb.getSymbolsList()) { - symbols.put(info.getSymbol(), info); - } } public Semanticdb.TextDocument manifestOccurrencesForSyntheticSymbols( diff --git a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipWriter.java b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipWriter.java index 2136de7b6..2b323740b 100644 --- a/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipWriter.java +++ b/scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipWriter.java @@ -1,32 +1,23 @@ package com.sourcegraph.scip_semanticdb; -import com.sourcegraph.lsif_protocol.LsifHover; -import com.sourcegraph.lsif_protocol.LsifHover.Content; -import com.sourcegraph.lsif_protocol.LsifObject; -import com.sourcegraph.lsif_protocol.LsifPosition; -import com.sourcegraph.semanticdb_javac.Semanticdb; -import com.sourcegraph.semanticdb_javac.SemanticdbSymbols; +import com.google.protobuf.util.JsonFormat; import com.sourcegraph.Scip; import java.io.BufferedOutputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.attribute.PosixFilePermissions; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -/** High-level utility methods to write SCIP vertex/edge objects into the SCIP output stream. */ +/** High-level utility methods to write SCIP index into the SCIP output stream. */ public class ScipWriter implements AutoCloseable { private final Path tmp; private final ScipOutputStream output; private final ScipSemanticdbOptions options; - private final AtomicInteger id = new AtomicInteger(); + private final JsonFormat.Printer jsonPrinter; public ScipWriter(ScipSemanticdbOptions options) throws IOException { if (OperatingSystem.isWindows()) { @@ -39,124 +30,21 @@ public ScipWriter(ScipSemanticdbOptions options) throws IOException { PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--"))); } this.output = - new ScipOutputStream(options, new BufferedOutputStream(Files.newOutputStream(tmp))); + new ScipOutputStream(options.format, new BufferedOutputStream(Files.newOutputStream(tmp))); this.options = options; + this.jsonPrinter = JsonFormat.printer().omittingInsignificantWhitespace(); } public void emitTyped(Scip.Index index) { - this.output.write(index.toByteArray()); - } - - public void emitMetaData() { - emitObject( - scipVertex("metaData") - .setVersion("0.4.3") - .setProjectRoot(options.sourceroot.toUri().toString()) - .setPositionEncoding("utf-16") - .setToolInfo(options.toolInfo)); - } - - public int emitProject(String language) { - return emitObject(scipVertex("project").setKind(language)); - } - - public int emitDocument(ScipTextDocument doc) { - return emitObject( - scipVertex("document") - .setUri(doc.semanticdb.getUri()) - .setLanguage(doc.semanticdb.getLanguage().toString().toLowerCase())); - } - - public void emitContains(int outV, List inVs) { - emitObject(scipEdge("contains").setOutV(outV).addAllInVs(inVs)); - } - - public int emitRange(Semanticdb.Range range) { - return emitObject( - scipVertex("range") - .setStart( - LsifPosition.newBuilder() - .setLine(range.getStartLine()) - .setCharacter(range.getStartCharacter())) - .setEnd( - LsifPosition.newBuilder() - .setLine(range.getEndLine()) - .setCharacter(range.getEndCharacter()))); - } - - public int emitResultSet() { - return emitObject(scipVertex("resultSet")); - } - - public void emitNext(int outV, int inV) { - emitObject(scipEdge("next").setOutV(outV).setInV(inV)); - } - - public void emitEdge(String label, int outV, int inV) { - emitObject(scipEdge(label).setOutV(outV).setInV(inV)); - } - - public int emitReferenceResult(int resultSet) { - int referenceResult = emitObject(scipVertex("referenceResult")); - emitEdge("textDocument/references", resultSet, referenceResult); - return referenceResult; - } - - public int emitDefinitionResult(int resultSet) { - int definitionResult = emitObject(scipVertex("definitionResult")); - emitEdge("textDocument/definition", resultSet, definitionResult); - return definitionResult; - } - - public int emitHoverResult(MarkupContent markupContents) { - return emitObject( - scipVertex("hoverResult") - .setResult( - LsifHover.newBuilder() - .setContents( - Content.newBuilder() - .setKind(markupContents.kind.toString().toLowerCase()) - .setValue(markupContents.value)))); - } - - public void emitHoverEdge(int outV, int inV) { - emitObject(scipEdge("textDocument/hover").setOutV(outV).setInV(inV)); - } - - public void emitMonikerEdge(int outV, int inV) { - emitObject(scipEdge("moniker").setOutV(outV).setInV(inV)); - } - - public int emitMonikerVertex(String symbol, boolean isExportedSymbol) { - String kind = - SemanticdbSymbols.isLocal(symbol) ? "local" : isExportedSymbol ? "export" : "import"; - return emitObject( - scipVertex("moniker").setKind(kind).setScheme("semanticdb").setIdentifier(symbol)); - } - - public void emitPackageInformationEdge(int monikerId, int packageId) { - emitObject(scipEdge("packageInformation").setOutV(monikerId).setInV(packageId)); - } - - public int emitpackageinformationVertex(Package pkg) { - return emitObject( - scipVertex("packageInformation") - .setName(pkg.repoName()) - .setManager("jvm-dependencies") - .setVersion(pkg.version())); - } - - public void emitItem(int outV, int inV, int document) { - emitObject(scipEdge("item").setOutV(outV).addInVs(inV).setDocument(document)); - } - - public void emitReferenceResultsItemEdge(int outV, Iterable inVs, int document) { - emitObject( - scipEdge("item") - .setOutV(outV) - .addAllInVs(inVs) - .setDocument(document) - .setProperty("referenceResults")); + try { + if (options.format.isNewlineDelimitedJSON()) { + this.output.write(jsonPrinter.print(index).getBytes(StandardCharsets.UTF_8)); + } else { + this.output.write(index.toByteArray()); + } + } catch (IOException e) { + options.reporter.error(e); + } } public void build() throws IOException { @@ -176,25 +64,4 @@ public void flush() { options.reporter.error(e); } } - - public int emitObject(LsifObject.Builder object) { - output.writeLsifObject(object); - return object.getId(); - } - - private LsifObject.Builder scipObject() { - return LsifObject.newBuilder().setId(nextId()); - } - - private LsifObject.Builder scipEdge(String label) { - return scipObject().setType("edge").setLabel(label); - } - - private LsifObject.Builder scipVertex(String label) { - return scipObject().setType("vertex").setLabel(label); - } - - private int nextId() { - return id.incrementAndGet(); - } } diff --git a/scip-semanticdb/src/main/protobuf/lsif.proto b/scip-semanticdb/src/main/protobuf/lsif.proto deleted file mode 100644 index 706542796..000000000 --- a/scip-semanticdb/src/main/protobuf/lsif.proto +++ /dev/null @@ -1,56 +0,0 @@ -syntax = "proto3"; - -option java_multiple_files = true; - -package com.sourcegraph.lsif_protocol; - -message LsifObject { - int32 id = 1; - string type = 2; - string label = 3; - string version = 4; - string projectRoot = 5; - string positionEncoding = 6; - LsifToolInfo toolInfo = 7; - string kind = 8; - string uri = 9; - string language = 10; - int32 outV = 12; - int32 inV = 13; - repeated int32 inVs = 14; - int32 document = 15; - LsifHover result = 16; - string scheme = 17; - string identifier = 18; - LsifPosition start = 19; - LsifPosition end = 20; - string name = 21; - string manager = 22; - string property = 23; -} - -message LsifToolInfo { - string name = 1; - string version = 2; - repeated string args = 3; -} - -message LsifPosition { - int32 line = 1; - int32 character = 2; -} - -message LsifHover { - Content contents = 1; - message Content { - string kind = 1; // not MarkupKind because must be lowercase - string value = 2; - } -} - -enum MarkupKind { - PLAINTEXT = 0; - MARKDOWN = 1; -} - - diff --git a/tests/snapshots/META-INF/main.kotlin_module b/tests/snapshots/META-INF/main.kotlin_module new file mode 100644 index 000000000..3ac8ae444 Binary files /dev/null and b/tests/snapshots/META-INF/main.kotlin_module differ