From de1f9c77cefa38877050ff8bbf7ab5de59f7f1c0 Mon Sep 17 00:00:00 2001 From: Michael Bien Date: Thu, 22 Jan 2026 12:52:45 +0100 Subject: [PATCH 01/13] CI: bump lower bound to JDK 21 new test and build range: JDK 21-26 few modules are still stuck on 17 and got special treatment --- .github/workflows/main.yml | 79 +++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 36 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 706c88fdb4e5..6f12a10fa972 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -137,7 +137,7 @@ jobs: timeout-minutes: 40 strategy: matrix: - java: [ '17', '25', '26-ea' ] + java: [ '21', '25', '26-ea' ] exclude: - java: ${{ github.event_name == 'pull_request' && 'nothing' || '25' }} fail-fast: false @@ -167,11 +167,11 @@ jobs: run: ant $OPTS -quiet -Dcluster.config=$CLUSTER_CONFIG build-nozip - name: Prepare Artifact - if: ${{ matrix.java == '17' }} + if: ${{ matrix.java == '21' }} run: tar -I 'zstd -9 -T0' -cf /tmp/build.tar.zst --exclude ".git" . - name: Upload Workspace - if: ${{ (matrix.java == '17') && success() }} + if: ${{ (matrix.java == '21') && success() }} uses: actions/upload-artifact@v6 with: name: build @@ -181,11 +181,11 @@ jobs: if-no-files-found: error - name: Create Dev Build - if: ${{ matrix.java == '17' && contains(github.event.pull_request.labels.*.name, 'ci:dev-build') && success() }} + if: ${{ matrix.java == '21' && contains(github.event.pull_request.labels.*.name, 'ci:dev-build') && success() }} run: ant $OPTS -quiet -Dcluster.config=$CLUSTER_CONFIG zip-cluster-config - name: Upload Dev Build - if: ${{ matrix.java == '17' && contains(github.event.pull_request.labels.*.name, 'ci:dev-build') && success() }} + if: ${{ matrix.java == '21' && contains(github.event.pull_request.labels.*.name, 'ci:dev-build') && success() }} uses: actions/upload-artifact@v6 with: name: dev-build_${{github.event.pull_request.number || github.run_id}} @@ -239,7 +239,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] - java: [ 17 ] + java: [ '21' ] include: - os: ubuntu-latest java: 26-ea @@ -273,7 +273,7 @@ jobs: run: tar --zstd -xf build.tar.zst - name: platform/masterfs - if: ${{ matrix.java == '17' }} + if: ${{ matrix.java == '21' }} run: .github/retry.sh ant $OPTS -f platform/masterfs test - name: Commit Validation tests @@ -301,7 +301,7 @@ jobs: ANT_OPTS: -Dmetabuild.jsonurl=https://raw.githubusercontent.com/apache/netbeans-jenkins-lib/master/meta/netbeansrelease.json strategy: matrix: - java: [ '17' ] + java: [ '21' ] steps: - name: Check PR labels @@ -379,7 +379,7 @@ jobs: timeout-minutes: 100 strategy: matrix: - java: [ '17' ] + java: [ '21' ] fail-fast: false steps: @@ -473,7 +473,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - java: [ '17' ] + java: [ '21' ] config: [ 'platform', 'release' ] fail-fast: false steps: @@ -519,7 +519,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - java: [ '17' ] + java: [ '21' ] fail-fast: false steps: @@ -849,9 +849,9 @@ jobs: timeout-minutes: 50 strategy: matrix: - java: [ '17', '21', '25', '26-ea' ] + java: [ '21', '25', '26-ea' ] exclude: - - java: ${{ github.event_name == 'pull_request' && 'nothing' || '21' }} + - java: ${{ github.event_name == 'pull_request' && 'nothing' || '25' }} fail-fast: false steps: @@ -935,28 +935,35 @@ jobs: run: ant $OPTS -f java/gradle.test test - name: java/gradle.java - if: ${{ matrix.java == '17' || matrix.java == '21' || matrix.java == '25' }} + if: ${{ matrix.java == '21' || matrix.java == '25' }} run: .github/retry.sh ant $OPTS -f java/gradle.java test - name: extide/gradle - if: ${{ matrix.java == '17' || matrix.java == '21' || matrix.java == '25' }} + if: ${{ matrix.java == '21' || matrix.java == '25' }} run: ant $OPTS -f extide/gradle test - name: java/gradle.dependencies - if: ${{ matrix.java == '17' || matrix.java == '21' || matrix.java == '25' }} + if: ${{ matrix.java == '21' || matrix.java == '25' }} run: ant $OPTS -f java/gradle.dependencies test # TODO next are JDK 21 or 25 incompatibe steps - name: java/java.mx.project - if: ${{ matrix.java == '17' || matrix.java == '21' }} + if: ${{ matrix.java == '21' }} run: .github/retry.sh ant $OPTS -f java/java.mx.project test + - name: Set up JDK 17 for ant + uses: actions/setup-java@v5 + if: ${{ matrix.java == '21' }} + with: + java-version: 17 + distribution: ${{ env.DEFAULT_JAVA_DISTRIBUTION }} + - name: extide/o.apache.tools.ant.module - if: ${{ matrix.java == '17' }} + if: ${{ matrix.java == '21' }} run: ant $OPTS -f extide/o.apache.tools.ant.module test - name: apisupport.ant - if: ${{ matrix.java == '17' }} + if: ${{ matrix.java == '21' }} run: ant $OPTS -f apisupport/apisupport.ant test - name: Create Test Summary @@ -975,7 +982,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - java: [ '17' ] + java: [ '21' ] fail-fast: false steps: @@ -1138,7 +1145,7 @@ jobs: timeout-minutes: 90 strategy: matrix: - java: [ '17' ] + java: [ '21' ] fail-fast: false steps: @@ -1273,7 +1280,7 @@ jobs: timeout-minutes: 100 strategy: matrix: - java: [ '17' ] + java: [ '21' ] fail-fast: false steps: @@ -1459,7 +1466,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - java: [ '17', '26-ea' ] + java: [ '21', '26-ea' ] config: [ 'batch1', 'batch2' ] exclude: - java: ${{ github.event_name == 'pull_request' && 'nothing' || '26-ea' }} @@ -1516,7 +1523,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - java: [ '17', '25', '26-ea' ] + java: [ '21', '25', '26-ea' ] exclude: - java: ${{ github.event_name == 'pull_request' && 'nothing' || '25' }} fail-fast: false @@ -1578,7 +1585,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - java: [ '17' ] + java: [ '21' ] fail-fast: false steps: @@ -1623,7 +1630,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - java: [ '17' ] + java: [ '21' ] fail-fast: false steps: @@ -1801,7 +1808,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - java: [ '17' ] + java: [ '21' ] fail-fast: false steps: @@ -1849,9 +1856,9 @@ jobs: timeout-minutes: 60 strategy: matrix: - java: [ '17', '21' ] - exclude: - - java: ${{ github.event_name == 'pull_request' && 'nothing' || '21' }} + java: [ '21' ] +# exclude: +# - java: ${{ github.event_name == 'pull_request' && 'nothing' || '21' }} fail-fast: false steps: @@ -1902,7 +1909,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - java: [ '17' ] + java: [ '21' ] fail-fast: false steps: @@ -1950,7 +1957,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - java: [ '17' ] + java: [ '21' ] fail-fast: false steps: @@ -2195,7 +2202,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - java: [ '17' ] + java: [ '21' ] fail-fast: false steps: @@ -2278,7 +2285,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - java: [ '17' ] + java: [ '21' ] fail-fast: false services: @@ -2340,7 +2347,7 @@ jobs: DISPLAY: ":99.0" strategy: matrix: - java: [ '17' ] + java: [ '21' ] os: [ 'windows-latest', 'ubuntu-latest' ] exclude: - os: ${{ github.event_name == 'pull_request' && 'nothing' || 'windows-latest' }} @@ -2496,7 +2503,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - java: [ '17' ] + java: [ '21' ] fail-fast: false steps: From 0de6cb5dfd47bd861b2c00db84bf46be40dd14a0 Mon Sep 17 00:00:00 2001 From: Michael Bien Date: Fri, 23 Jan 2026 23:52:03 +0100 Subject: [PATCH 02/13] Updated java.editor and java.source tests for JDK 21 and 25 - added unimportant classes to the ignore list so that the test data doesn't have to be updated. - refactored file read monitoring from SecurityManager to JFR --- .../editor/imports/ComputeImportsTest.java | 11 +-- .../nbproject/project.properties | 2 +- .../api/java/source/TreePathHandleTest.java | 67 ++++++++----------- 3 files changed, 36 insertions(+), 44 deletions(-) diff --git a/java/java.editor/test/unit/src/org/netbeans/modules/java/editor/imports/ComputeImportsTest.java b/java/java.editor/test/unit/src/org/netbeans/modules/java/editor/imports/ComputeImportsTest.java index a43ef3f0b499..95c431526546 100644 --- a/java/java.editor/test/unit/src/org/netbeans/modules/java/editor/imports/ComputeImportsTest.java +++ b/java/java.editor/test/unit/src/org/netbeans/modules/java/editor/imports/ComputeImportsTest.java @@ -34,7 +34,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -68,7 +67,7 @@ */ public class ComputeImportsTest extends NbTestCase { - private static final Set IGNORE_CLASSES = new HashSet(Arrays.asList(new String[] { + private static final Set IGNORE_CLASSES = Set.of( "com.sun.tools.javac.util.List", "com.sun.tools.javac.code.Attribute.RetentionPolicy", "com.sun.tools.classfile.Opcode.Set", @@ -81,6 +80,8 @@ public class ComputeImportsTest extends NbTestCase { "com.sun.xml.internal.bind.v2.runtime.output.NamespaceContextImpl.Element", "com.sun.xml.internal.bind.v2.schemagen.xmlschema.Element", "sun.text.normalizer.RangeValueIterator.Element", + "jdk.internal.org.commonmark.node.Document", + "java.lang.StringTemplate.STR", "javax.xml.bind.Element", "javax.lang.model.element.Element", "com.sun.org.apache.xalan.internal.xsltc.runtime.AttributeList", @@ -90,13 +91,13 @@ public class ComputeImportsTest extends NbTestCase { "com.sun.org.apache.xalan.internal.xsltc.compiler.util.Type.Element", "com.sun.xml.internal.ws.policy.privateutil.PolicyUtils.Collections", "com.azul.crs.client.service.JarLoadMonitor.ProcessedJarFiles.LRU.Element" - })); + ); - private static final List IGNORE_PATTERNS = Collections.unmodifiableList(Arrays.asList( + private static final List IGNORE_PATTERNS = List.of( Pattern.compile("jdk\\..*\\.internal\\..*"), Pattern.compile("org\\.graalvm\\..*"), Pattern.compile("com\\.azul\\.crs\\..*") // https://docs.azul.com/vulnerability-detection/detailed-information/configuration-options - )); + ); private FileObject testSource; private JavaSource js; diff --git a/java/java.source.base/nbproject/project.properties b/java/java.source.base/nbproject/project.properties index e74394ad6113..923ec707672b 100644 --- a/java/java.source.base/nbproject/project.properties +++ b/java/java.source.base/nbproject/project.properties @@ -206,5 +206,5 @@ test.config.vanilla-javac.excludes=\ # remove default compiler JMS flags so that we don't get "Unknown module" warnings due to --limit-modules jms-compiler.flags.jvm= -test.jms.flags=--limit-modules=java.base,java.logging,java.xml,java.prefs,java.desktop,java.management,java.instrument,jdk.zipfs,java.scripting,java.naming +test.jms.flags=--limit-modules=java.base,java.logging,java.xml,java.prefs,java.desktop,java.management,java.instrument,jdk.zipfs,java.scripting,java.naming,jdk.jfr test.bootclasspath.prepend.args=-Dno.netbeans.bootclasspath.prepend.needed=true diff --git a/java/java.source.base/test/unit/src/org/netbeans/api/java/source/TreePathHandleTest.java b/java/java.source.base/test/unit/src/org/netbeans/api/java/source/TreePathHandleTest.java index 950c9fa6afcf..6d762690fcdc 100644 --- a/java/java.source.base/test/unit/src/org/netbeans/api/java/source/TreePathHandleTest.java +++ b/java/java.source.base/test/unit/src/org/netbeans/api/java/source/TreePathHandleTest.java @@ -26,10 +26,11 @@ import com.sun.source.util.TreePath; import java.io.File; import java.io.OutputStream; -import java.security.Permission; +import java.util.concurrent.atomic.AtomicBoolean; import javax.lang.model.element.Element; import javax.lang.model.element.TypeElement; import javax.lang.model.element.VariableElement; +import jdk.jfr.consumer.RecordingStream; import org.netbeans.api.java.source.JavaSource.Phase; import org.netbeans.core.startup.Main; import org.netbeans.junit.NbTestCase; @@ -51,6 +52,7 @@ public TreePathHandleTest(String testName) { private FileObject sourceRoot; + @Override protected void setUp() throws Exception { clearWorkDir(); @@ -170,29 +172,40 @@ public void test134457() throws Exception { public void testTreePathIsNotParsing() throws Exception { FileObject file = FileUtil.createData(sourceRoot, "test/test.java"); - writeIntoFile(file, "package test; public class test {}"); - writeIntoFile(FileUtil.createData(sourceRoot, "test/test2.java"), "package test; public class test2 {}"); + + FileObject file2 = FileUtil.createData(sourceRoot, "test/test2.java"); + writeIntoFile(file2, "package test; public class test2 {}"); JavaSource js = JavaSource.forFileObject(file); SourceUtilsTestUtil.compileRecursively(sourceRoot); - js.runUserActionTask(new Task() { - public void run(CompilationController parameter) throws Exception { - parameter.toPhase(Phase.RESOLVED); - - TypeElement string = parameter.getElements().getTypeElement("test.test2"); - - SecurityManager old = System.getSecurityManager(); - - System.setSecurityManager(new SecMan()); - - TreePathHandle.create(string, parameter); - - System.setSecurityManager(old); + AtomicBoolean wasRead = new AtomicBoolean(); + + js.runUserActionTask(controller -> { + controller.toPhase(Phase.RESOLVED); + + TypeElement string = controller.getElements().getTypeElement("test.test2"); + + try (RecordingStream rs = new RecordingStream()) { + rs.setMaxSize(Long.MAX_VALUE); + rs.enable("jdk.FileRead").withoutThreshold().withStackTrace(); + rs.onEvent("jdk.FileRead", e -> { + if (e.getString("path").endsWith("test2.java")) { + wasRead.set(true); + System.err.println(e); + } + }); + rs.startAsync(); +// file2.getInputStream().read(); // check: commenting this in should fail the test + TreePathHandle.create(string, controller); + // TODO call directly post JDK 21 bump + RecordingStream.class.getMethod("stop").invoke(rs); // flushes stream } }, true); + + assertFalse("file was read", wasRead.get()); } public void testEquals() throws Exception { @@ -454,26 +467,4 @@ public void testVarInstanceMember() throws Exception { assertNotNull(handle.getElementHandle()); JavacParser.DISABLE_SOURCE_LEVEL_DOWNGRADE = false; } - - private static final class SecMan extends SecurityManager { - - @Override - public void checkRead(String file) { - assertFalse(file.endsWith("test2.java")); - } - - @Override - public void checkRead(String file, Object context) { - assertFalse(file.endsWith("test2.java")); - } - - @Override - public void checkPermission(Permission perm) { - } - - @Override - public void checkPermission(Permission perm, Object context) { - } - - } } From d165f5d39bfb3bc46f5dda74e2c0d40dc51ce9cf Mon Sep 17 00:00:00 2001 From: Michael Bien Date: Mon, 26 Jan 2026 16:55:10 +0100 Subject: [PATCH 03/13] Update enterprise/micronaut completion test data for JDK 21 --- .../allTypes.pass | 5 ++++- .../ctxTypes.pass | 5 ++++- .../intMethods.pass | 2 ++ .../intStaticMethods.pass | 2 ++ .../javaLangPackageContent.pass | 6 +++++- .../stringMethods.pass | 3 +++ 6 files changed, 20 insertions(+), 3 deletions(-) diff --git a/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/allTypes.pass b/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/allTypes.pass index 014426affcfe..feaf949b0a5f 100644 --- a/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/allTypes.pass +++ b/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/allTypes.pass @@ -20,7 +20,6 @@ ClassValue CloneNotSupportedException Cloneable Comparable -Compiler Deprecated Double Enum @@ -49,6 +48,7 @@ Iterable LayerInstantiationException LinkageError Long +MatchException Math Module ModuleLayer @@ -76,6 +76,7 @@ Runtime RuntimeException RuntimePermission SafeVarargs +ScopedValue SecurityException SecurityManager Short @@ -87,6 +88,7 @@ String StringBuffer StringBuilder StringIndexOutOfBoundsException +StringTemplate SuppressWarnings System Thread @@ -102,6 +104,7 @@ UnsupportedOperationException VerifyError VirtualMachineError Void +WrongThreadException com io jakarta diff --git a/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/ctxTypes.pass b/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/ctxTypes.pass index 291b09aee455..5e9c14a0793b 100644 --- a/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/ctxTypes.pass +++ b/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/ctxTypes.pass @@ -21,7 +21,6 @@ ClassValue CloneNotSupportedException Cloneable Comparable -Compiler Deprecated Double Enum @@ -50,6 +49,7 @@ Iterable LayerInstantiationException LinkageError Long +MatchException Math Module ModuleLayer @@ -77,6 +77,7 @@ Runtime RuntimeException RuntimePermission SafeVarargs +ScopedValue SecurityException SecurityManager Short @@ -88,6 +89,7 @@ String StringBuffer StringBuilder StringIndexOutOfBoundsException +StringTemplate SuppressWarnings System Thread @@ -103,6 +105,7 @@ UnsupportedOperationException VerifyError VirtualMachineError Void +WrongThreadException com io jakarta diff --git a/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/intMethods.pass b/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/intMethods.pass index 7eeaa3435c84..a99e559130c8 100644 --- a/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/intMethods.pass +++ b/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/intMethods.pass @@ -3,11 +3,13 @@ byteValue() : byte compare(int x, int y) : int compareTo(Integer anotherInteger) : int compareUnsigned(int x, int y) : int +compress(int i, int mask) : int decode(String nm) : Integer describeConstable() : Optional divideUnsigned(int dividend, int divisor) : int doubleValue() : double equals(Object obj) : boolean +expand(int i, int mask) : int floatValue() : float getInteger(String nm) : Integer getInteger(String nm, Integer val) : Integer diff --git a/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/intStaticMethods.pass b/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/intStaticMethods.pass index 0a92dc03ca49..1aba581617f8 100644 --- a/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/intStaticMethods.pass +++ b/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/intStaticMethods.pass @@ -1,8 +1,10 @@ bitCount(int i) : int compare(int x, int y) : int compareUnsigned(int x, int y) : int +compress(int i, int mask) : int decode(String nm) : Integer divideUnsigned(int dividend, int divisor) : int +expand(int i, int mask) : int getInteger(String nm) : Integer getInteger(String nm, Integer val) : Integer getInteger(String nm, int val) : Integer diff --git a/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/javaLangPackageContent.pass b/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/javaLangPackageContent.pass index ca6e5c869912..9c95206c2f0c 100644 --- a/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/javaLangPackageContent.pass +++ b/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/javaLangPackageContent.pass @@ -20,7 +20,6 @@ ClassValue CloneNotSupportedException Cloneable Comparable -Compiler Deprecated Double Enum @@ -49,6 +48,7 @@ Iterable LayerInstantiationException LinkageError Long +MatchException Math Module ModuleLayer @@ -76,6 +76,7 @@ Runtime RuntimeException RuntimePermission SafeVarargs +ScopedValue SecurityException SecurityManager Short @@ -87,6 +88,7 @@ String StringBuffer StringBuilder StringIndexOutOfBoundsException +StringTemplate SuppressWarnings System Thread @@ -102,8 +104,10 @@ UnsupportedOperationException VerifyError VirtualMachineError Void +WrongThreadException annotation constant +foreign instrument invoke management diff --git a/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/stringMethods.pass b/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/stringMethods.pass index 4363207a59e8..5029e946810a 100644 --- a/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/stringMethods.pass +++ b/enterprise/micronaut/test/unit/data/goldenfiles/org/netbeans/modules/micronaut/completion/MicronautExpressionLanguageConstructsCompletionTest/stringMethods.pass @@ -33,6 +33,8 @@ indexOf(String str) : int indexOf(int ch) : int indexOf(String str, int fromIndex) : int indexOf(int ch, int fromIndex) : int +indexOf(String str, int beginIndex, int endIndex) : int +indexOf(int ch, int beginIndex, int endIndex) : int intern() : String isBlank() : boolean isEmpty() : boolean @@ -56,6 +58,7 @@ replaceFirst(String regex, String replacement) : String resolveConstantDesc(Lookup lookup) : String split(String regex) : String[] split(String regex, int limit) : String[] +splitWithDelimiters(String regex, int limit) : String[] startsWith(String prefix) : boolean startsWith(String prefix, int toffset) : boolean strip() : String From a3f3ac059ec03302a80499d472a7b3cb3428190e Mon Sep 17 00:00:00 2001 From: Michael Bien Date: Mon, 26 Jan 2026 01:25:43 +0100 Subject: [PATCH 04/13] Tests: enable SM where still needed - remove no longer needed SM usage in ide/xsl test - remove NBBundleWithSecurityManager test - remove NbLifecycleManagerEDTTest which is similar to LifecycleManagerEDTTest in openide.util.ui --- .../nbproject/project.properties | 3 ++ ide/versioning/nbproject/project.properties | 3 ++ .../xsl/transform/TransformServletTest.java | 49 ----------------- .../nbproject/project.properties | 2 + .../core.startup/nbproject/project.properties | 3 ++ .../masterfs/nbproject/project.properties | 3 ++ .../netbinox/nbproject/project.properties | 3 ++ .../nbproject/project.properties | 4 ++ .../core/NbLifecycleManagerEDTTest.java | 35 ------------- .../netbeans/core/NbLifecycleManagerTest.java | 4 +- .../nbproject/project.properties | 3 ++ .../nbproject/project.properties | 3 ++ .../nbproject/project.properties | 3 ++ .../nbproject/project.properties | 3 ++ .../nbproject/project.properties | 3 ++ .../NbBundleWithSecurityManager.properties | 18 ------- .../util/NbBundleWithSecurityManagerTest.java | 52 ------------------- 17 files changed, 38 insertions(+), 156 deletions(-) delete mode 100644 platform/o.n.core/test/unit/src/org/netbeans/core/NbLifecycleManagerEDTTest.java delete mode 100644 platform/openide.util/test/unit/src/org/openide/util/NbBundleWithSecurityManager.properties delete mode 100644 platform/openide.util/test/unit/src/org/openide/util/NbBundleWithSecurityManagerTest.java diff --git a/harness/apisupport.harness/nbproject/project.properties b/harness/apisupport.harness/nbproject/project.properties index ca593b9ca344..066614f085e3 100644 --- a/harness/apisupport.harness/nbproject/project.properties +++ b/harness/apisupport.harness/nbproject/project.properties @@ -97,3 +97,6 @@ test.unit.cp.extra=${netbeans.dest.dir}/harness/jnlp/jnlp-launcher.jar javadoc.arch=${basedir}/arch.xml test.config.stableBTD.includes=**/*Test.class + +# TODO needed for tests; remove with jnlp code +test.jms.flags=-Djava.security.manager=allow diff --git a/ide/versioning/nbproject/project.properties b/ide/versioning/nbproject/project.properties index 41a90285112a..9822f827d24b 100644 --- a/ide/versioning/nbproject/project.properties +++ b/ide/versioning/nbproject/project.properties @@ -43,3 +43,6 @@ test.config.stableBTD.excludes=\ **/GetAnnotatedOwnerTest.class,\ **/VCSContextTest.class,\ **/VersioningAnnotationProviderTest.class + +# TODO remove SM usage in tests +test.jms.flags=-Djava.security.manager=allow diff --git a/ide/xsl/test/unit/src/org/netbeans/modules/xsl/transform/TransformServletTest.java b/ide/xsl/test/unit/src/org/netbeans/modules/xsl/transform/TransformServletTest.java index 802bcb3e3bca..6346ffe23f28 100644 --- a/ide/xsl/test/unit/src/org/netbeans/modules/xsl/transform/TransformServletTest.java +++ b/ide/xsl/test/unit/src/org/netbeans/modules/xsl/transform/TransformServletTest.java @@ -18,34 +18,11 @@ */ package org.netbeans.modules.xsl.transform; -//import java.io.*; import java.net.*; -import java.security.Permission; -//import java.util.*; - -//import javax.servlet.*; -//import javax.servlet.http.*; import junit.framework.*; import org.netbeans.junit.*; -/*import org.openide.util.HttpServer; -import org.openide.filesystems.FileObject; -import org.openide.util.SharedClassObject; -import org.openide.filesystems.FileUtil; -import org.openide.execution.NbfsURLConnection; - -import org.xml.sax.*; -import javax.xml.parsers.*; -import javax.xml.transform.*; -import javax.xml.transform.sax.*; -import javax.xml.transform.stream.*; - -import org.netbeans.api.xml.cookies.*; -import org.netbeans.spi.xml.cookies.*; - -import org.netbeans.modules.xsl.utils.TransformUtil; -*/ /** * * @author Libor Kramolis @@ -60,15 +37,12 @@ public static void main(java.lang.String[] args) { junit.textui.TestRunner.run(suite()); } - - public static Test suite() { TestSuite suite = new NbTestSuite(TransformServletTest.class); return suite; } - /** Test of getServletURL method, of class org.netbeans.modules.xsl.transform.TransformServlet. */ public void testGetServletURL() { // issue #196602 @@ -85,28 +59,5 @@ public void testGetServletURL() { assertTrue ("I need correct Transform Servlet URL!", (servletURL!=null & exceptionThrown!= true)); } - - @Override - protected void setUp() throws Exception { - super.setUp(); - SecurityManager sm = new SecurityManager() { - - @Override - public void checkPermission(Permission perm) { - } - - @Override - public void checkPermission(Permission perm, Object context) { - } - - }; - System.setSecurityManager(sm); - } - - @Override - protected void tearDown() throws Exception { - System.setSecurityManager(null); - super.tearDown(); - } } diff --git a/platform/core.startup.base/nbproject/project.properties b/platform/core.startup.base/nbproject/project.properties index 6e4f1711c5f7..0a484e2ef52c 100644 --- a/platform/core.startup.base/nbproject/project.properties +++ b/platform/core.startup.base/nbproject/project.properties @@ -24,3 +24,5 @@ javadoc.arch=${basedir}/arch.xml test-unit-sys-prop.xtest.data=${nb_all}/platform/o.n.bootstrap/test/unit/data test.unit.data.dir=${nb_all}/platform/o.n.bootstrap/test/unit/data +# TODO remove SM usage in test (testURLNoInetAccess) +test.jms.flags=-Djava.security.manager=allow diff --git a/platform/core.startup/nbproject/project.properties b/platform/core.startup/nbproject/project.properties index f328f174001a..d77be16e95d5 100644 --- a/platform/core.startup/nbproject/project.properties +++ b/platform/core.startup/nbproject/project.properties @@ -50,3 +50,6 @@ test.config.stableBTD.excludes=\ **/SystemFileSystemTest.class,\ **/TopLoggingNbLoggerConsoleTest.class,\ **/TopLoggingStartLogTest.class + +# TODO remove SM usage in tests +test.jms.flags=-Djava.security.manager=allow diff --git a/platform/masterfs/nbproject/project.properties b/platform/masterfs/nbproject/project.properties index b0188f8ed182..95ac0d9f4d76 100644 --- a/platform/masterfs/nbproject/project.properties +++ b/platform/masterfs/nbproject/project.properties @@ -36,3 +36,6 @@ test.config.stableBTD.excludes=\ **/SlowRefreshSuspendableTest.class,\ **/StatFilesTest.class spec.version.base=2.87.0 + +# TODO remove SM usage in tests +test.jms.flags=-Djava.security.manager=allow diff --git a/platform/netbinox/nbproject/project.properties b/platform/netbinox/nbproject/project.properties index a9205738019b..31a4902af2d3 100644 --- a/platform/netbinox/nbproject/project.properties +++ b/platform/netbinox/nbproject/project.properties @@ -28,3 +28,6 @@ test.config.stableBTD.excludes=\ **/CachingPreventsFileTouchesTest.class,\ **/LogReaderServiceTest.class,\ **/NetigsoHasSAXParserTest.class + +# TODO remove SM usage in tests +test.jms.flags=-Djava.security.manager=allow \ No newline at end of file diff --git a/platform/o.n.bootstrap/nbproject/project.properties b/platform/o.n.bootstrap/nbproject/project.properties index 429461e1f047..87855f16d1e0 100644 --- a/platform/o.n.bootstrap/nbproject/project.properties +++ b/platform/o.n.bootstrap/nbproject/project.properties @@ -31,3 +31,7 @@ javadoc.apichanges=${basedir}/apichanges.xml # When changing this list, change 'verifyexcludes' param. of 'makejnlp' task in apisupport.harness/release/jnlp.xml as well jnlp.verify.excludes=lib/nbexec,lib/nbexec.exe,lib/nbexec64.exe,lib/nbexec.dll,lib/nbexec64.dll + + +# TODO remove SM usage in test (JarClassLoaderTest, StampsExtraTest, StampsTest, StampsWithDotDotTest) +test.jms.flags=-Djava.security.manager=allow \ No newline at end of file diff --git a/platform/o.n.core/test/unit/src/org/netbeans/core/NbLifecycleManagerEDTTest.java b/platform/o.n.core/test/unit/src/org/netbeans/core/NbLifecycleManagerEDTTest.java deleted file mode 100644 index af2b167716bc..000000000000 --- a/platform/o.n.core/test/unit/src/org/netbeans/core/NbLifecycleManagerEDTTest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.netbeans.core; - -/** - * - * @author Jaroslav Tulach - */ -public class NbLifecycleManagerEDTTest extends NbLifecycleManagerTest { - - public NbLifecycleManagerEDTTest(String n) { - super(n); - } - - @Override - protected boolean runInEQ() { - return true; - } -} diff --git a/platform/o.n.core/test/unit/src/org/netbeans/core/NbLifecycleManagerTest.java b/platform/o.n.core/test/unit/src/org/netbeans/core/NbLifecycleManagerTest.java index f9ea0cc7de57..60a11054c1e8 100644 --- a/platform/o.n.core/test/unit/src/org/netbeans/core/NbLifecycleManagerTest.java +++ b/platform/o.n.core/test/unit/src/org/netbeans/core/NbLifecycleManagerTest.java @@ -18,14 +18,14 @@ */ package org.netbeans.core; +import org.netbeans.junit.NbTestCase; import org.openide.LifecycleManager; -import org.openide.LifecycleManagerTest; /** * * @author Jaroslav Tulach */ -public class NbLifecycleManagerTest extends LifecycleManagerTest { +public class NbLifecycleManagerTest extends NbTestCase { public NbLifecycleManagerTest(String n) { super(n); diff --git a/platform/openide.execution/nbproject/project.properties b/platform/openide.execution/nbproject/project.properties index 5861684c4e8a..b48ccb792f07 100644 --- a/platform/openide.execution/nbproject/project.properties +++ b/platform/openide.execution/nbproject/project.properties @@ -26,3 +26,6 @@ javadoc.apichanges=${basedir}/apichanges.xml jnlp.verify.excludes=config/ModuleAutoDeps/org-openide-execution.xml test.config.stableBTD.includes=**/*Test.class + +# TODO remove SM usage in tests +test.jms.flags=-Djava.security.manager=allow diff --git a/platform/openide.filesystems/nbproject/project.properties b/platform/openide.filesystems/nbproject/project.properties index 800dc69b22c0..00f163c75c29 100644 --- a/platform/openide.filesystems/nbproject/project.properties +++ b/platform/openide.filesystems/nbproject/project.properties @@ -42,3 +42,6 @@ test.config.stableBTD.excludes=\ **/StatFilesTest.class,\ **/URLMapperTestHidden.class,\ **/XMLFileSystemTest.class + +# TODO remove SM usage in tests +test.jms.flags=-Djava.security.manager=allow diff --git a/platform/openide.loaders/nbproject/project.properties b/platform/openide.loaders/nbproject/project.properties index e6ae801dd818..ac112f441eeb 100644 --- a/platform/openide.loaders/nbproject/project.properties +++ b/platform/openide.loaders/nbproject/project.properties @@ -21,3 +21,6 @@ javac.release=17 javadoc.main.page=org/openide/loaders/doc-files/api.html javadoc.arch=${basedir}/arch.xml javadoc.apichanges=${basedir}/apichanges.xml + +# TODO remove SM usage in tests +test.jms.flags=-Djava.security.manager=allow diff --git a/platform/openide.util.lookup/nbproject/project.properties b/platform/openide.util.lookup/nbproject/project.properties index f98abe2df894..89acefed377a 100644 --- a/platform/openide.util.lookup/nbproject/project.properties +++ b/platform/openide.util.lookup/nbproject/project.properties @@ -25,3 +25,6 @@ javadoc.apichanges=${basedir}/apichanges.xml module.javadoc.packages=org.openide.util,org.openide.util.lookup test.config.stableBTD.includes=**/*Test.class + +# TODO remove SM usage in tests +test.jms.flags=-Djava.security.manager=allow diff --git a/platform/openide.util.ui/nbproject/project.properties b/platform/openide.util.ui/nbproject/project.properties index dbb46b717842..1bf394eb24b3 100644 --- a/platform/openide.util.ui/nbproject/project.properties +++ b/platform/openide.util.ui/nbproject/project.properties @@ -33,3 +33,6 @@ test.config.stableBTD.excludes=\ **/RequestProcessorParallelTest.class,\ **/TaskTest.class,\ **/UtilitiesTest.class + +# TODO remove SM usage in tests (LifecycleManagerEDTTest, LifecycleManagerTest, XMLUtilReflectionTest) +test.jms.flags=-Djava.security.manager=allow diff --git a/platform/openide.util/test/unit/src/org/openide/util/NbBundleWithSecurityManager.properties b/platform/openide.util/test/unit/src/org/openide/util/NbBundleWithSecurityManager.properties deleted file mode 100644 index 4e88e515a17c..000000000000 --- a/platform/openide.util/test/unit/src/org/openide/util/NbBundleWithSecurityManager.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -HELLO=World diff --git a/platform/openide.util/test/unit/src/org/openide/util/NbBundleWithSecurityManagerTest.java b/platform/openide.util/test/unit/src/org/openide/util/NbBundleWithSecurityManagerTest.java deleted file mode 100644 index 4c87889f6ac4..000000000000 --- a/platform/openide.util/test/unit/src/org/openide/util/NbBundleWithSecurityManagerTest.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.openide.util; - -import java.security.Permission; -import java.util.ResourceBundle; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import org.junit.Test; - -public class NbBundleWithSecurityManagerTest { - @Test - public void testQueue() { - final Thread[] raised = { null }; - System.setSecurityManager(new SecurityManager() { - @Override - public void checkAccess(Thread t) { - if (t.getClass().getName().startsWith("java.util.logging")) { - return; - } - raised[0] = t; - throw new SecurityException(); - } - - @Override - public void checkPermission(Permission perm) { - } - }); - ResourceBundle bundle = NbBundle.getBundle("org.openide.util.NbBundleWithSecurityManager"); - assertNotNull("Bundle found", bundle); - assertEquals("World", bundle.getString("HELLO")); - - System.setSecurityManager(null); - assertNotNull("The thread has been prevented from being started", raised[0]); - } -} From 579ae1e95eca629ff7aaf0f02937abaa93f2dbec Mon Sep 17 00:00:00 2001 From: Michael Bien Date: Mon, 26 Jan 2026 04:20:38 +0100 Subject: [PATCH 05/13] disable JDK 21 incompatible tests (Thread.stop()) --- .github/workflows/main.yml | 5 +++-- .../test/unit/src/org/openide/nodes/ChildrenKeysTest.java | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6f12a10fa972..e1092081b10b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -420,8 +420,9 @@ jobs: - name: ergonomics/ide.ergonomics run: ant $OPTS -f ergonomics/ide.ergonomics test -Dtest.config=commit - - name: nb/deadlock.detector - run: ant $OPTS -f nb/deadlock.detector test + # TODO not JDK 21 compatible +# - name: nb/deadlock.detector +# run: ant $OPTS -f nb/deadlock.detector test - name: nb/ide.branding run: ant $OPTS -f nb/ide.branding test diff --git a/platform/openide.nodes/test/unit/src/org/openide/nodes/ChildrenKeysTest.java b/platform/openide.nodes/test/unit/src/org/openide/nodes/ChildrenKeysTest.java index e0487d6eec34..230bbe7739bc 100644 --- a/platform/openide.nodes/test/unit/src/org/openide/nodes/ChildrenKeysTest.java +++ b/platform/openide.nodes/test/unit/src/org/openide/nodes/ChildrenKeysTest.java @@ -137,12 +137,14 @@ public void run() { }//fail("Ok"); } - /** + /* * See #78519 * T1 has write access and gets preempted just before call to * getNodes() by another thread callig getNodes. * Other thread */ + // TODO uses Thread.stop(); rewrite or remove + /* public void testGetNodesFromWriteAccess() throws Exception { final String[] keys = { "Nenik", "Tulach" }; Keys o = new Keys (lazy(), keys); @@ -195,7 +197,8 @@ public void run() { assertTrue ("Preempted thread finished correctly", done[0]); assertTrue ("Other thread finished correctly", done[1]); } - + */ + public void testNodesCountDelegated() throws Exception { class K extends Children.Keys { int cnt; From aee6568c88430d8985e2cae376df19c20dae4591 Mon Sep 17 00:00:00 2001 From: Michael Bien Date: Mon, 26 Jan 2026 15:50:15 +0100 Subject: [PATCH 06/13] Fix htmlui tests: source/target 7 is no longer supported by javac --- .../test/unit/src/org/netbeans/api/htmlui/Compile.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform/api.htmlui/test/unit/src/org/netbeans/api/htmlui/Compile.java b/platform/api.htmlui/test/unit/src/org/netbeans/api/htmlui/Compile.java index 6b7f5faba6f3..95ca12922944 100644 --- a/platform/api.htmlui/test/unit/src/org/netbeans/api/htmlui/Compile.java +++ b/platform/api.htmlui/test/unit/src/org/netbeans/api/htmlui/Compile.java @@ -71,7 +71,7 @@ private Compile(String htmlName, String html, String code, String sl) throws IOE /** Performs compilation of given HTML page and associated Java code */ public static Compile create(String htmlName, String html, String code) throws IOException { - return create(htmlName, html, code, "1.7"); + return create(htmlName, html, code, "1.8"); } static Compile create(String htmlName, String html, String code, String sourceLevel) throws IOException { return new Compile(htmlName, html, code, sourceLevel); @@ -208,7 +208,7 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept } }; - ToolProvider.getSystemJavaCompiler().getTask(null, jfm, this, Arrays.asList("-source", sourceLevel, "-target", "1.7"), null, Arrays.asList(file)).call(); + ToolProvider.getSystemJavaCompiler().getTask(null, jfm, this, Arrays.asList("-source", sourceLevel, "-target", sourceLevel), null, Arrays.asList(file)).call(); Map result = new HashMap(); From ff4f9191b0b4788203e314c3cee83d9237f5d928 Mon Sep 17 00:00:00 2001 From: Michael Bien Date: Mon, 26 Jan 2026 16:10:11 +0100 Subject: [PATCH 07/13] Disable failing object size estimation test --- .../apisupport/project/layers/WritableXMLFileSystemTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apisupport/apisupport.project/test/unit/src/org/netbeans/modules/apisupport/project/layers/WritableXMLFileSystemTest.java b/apisupport/apisupport.project/test/unit/src/org/netbeans/modules/apisupport/project/layers/WritableXMLFileSystemTest.java index 6075ee37fd2c..e95b9cc15992 100644 --- a/apisupport/apisupport.project/test/unit/src/org/netbeans/modules/apisupport/project/layers/WritableXMLFileSystemTest.java +++ b/apisupport/apisupport.project/test/unit/src/org/netbeans/modules/apisupport/project/layers/WritableXMLFileSystemTest.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Set; import java.util.logging.Level; +import org.junit.Ignore; import org.netbeans.junit.RandomlyFails; import org.netbeans.modules.apisupport.project.TestUtil; import org.netbeans.modules.apisupport.project.layers.LayerUtils.SavableTreeEditorCookie; @@ -667,6 +668,7 @@ public void testExternalFileChangesRefired() throws Exception { fcl.changes()); } + @Ignore("fails on JDK 21+") public void testHeapUsage() throws Exception { Layer l = new Layer(""); FileSystem fs = l.read(); From d9b267bc16f5f5c8de91581ca0d4f334cb9d6461 Mon Sep 17 00:00:00 2001 From: Michael Bien Date: Mon, 26 Jan 2026 19:44:08 +0100 Subject: [PATCH 08/13] add -proc:none to test which doesn't need AP --- .../modules/netbinox/NetbinoxMultiversionJarTest.java | 4 ++-- .../test/unit/src/org/netbeans/JarClassLoaderTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/NetbinoxMultiversionJarTest.java b/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/NetbinoxMultiversionJarTest.java index f186ea40e28e..c5ec39d9a96d 100644 --- a/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/NetbinoxMultiversionJarTest.java +++ b/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/NetbinoxMultiversionJarTest.java @@ -69,7 +69,7 @@ protected void setUp() throws Exception { ToolProvider.getSystemJavaCompiler() .getTask(null, null, d -> { throw new IllegalStateException(d.toString()); - }, Arrays.asList("-d", classes.getAbsolutePath()), null, + }, Arrays.asList("-d", classes.getAbsolutePath(), "-proc:none"), null, Arrays.asList(new SourceFileObject("test/Impl.java", "package test; public class Impl { public static String get() { return \"base\"; } }"), new SourceFileObject("api/API.java", "package api; public class API { public static String run() { return test.Impl.get(); } }"))) .call(); @@ -78,7 +78,7 @@ protected void setUp() throws Exception { ToolProvider.getSystemJavaCompiler() .getTask(null, null, d -> { throw new IllegalStateException(d.toString()); - }, Arrays.asList("-d", classes9.getAbsolutePath(), "-classpath", classes.getAbsolutePath()), null, + }, Arrays.asList("-d", classes9.getAbsolutePath(), "-classpath", classes.getAbsolutePath(), "-proc:none"), null, Arrays.asList(new SourceFileObject("test/Impl.java", "package test; public class Impl { public static String get() { return \"9\"; } }"))) .call(); Map jarContent = new LinkedHashMap<>(); diff --git a/platform/o.n.bootstrap/test/unit/src/org/netbeans/JarClassLoaderTest.java b/platform/o.n.bootstrap/test/unit/src/org/netbeans/JarClassLoaderTest.java index 0a603593bba1..d7c2740bdf24 100644 --- a/platform/o.n.bootstrap/test/unit/src/org/netbeans/JarClassLoaderTest.java +++ b/platform/o.n.bootstrap/test/unit/src/org/netbeans/JarClassLoaderTest.java @@ -437,14 +437,14 @@ public void testMultiReleaseJar() throws Exception { File classes = new File(getWorkDir(), "classes"); classes.mkdirs(); ToolProvider.getSystemJavaCompiler() - .getTask(null, null, d -> { throw new IllegalStateException(d.toString()); }, Arrays.asList("-d", classes.getAbsolutePath()), null, + .getTask(null, null, d -> { throw new IllegalStateException(d.toString()); }, Arrays.asList("-d", classes.getAbsolutePath(), "-proc:none"), null, Arrays.asList(new SourceFileObject("test/Impl.java", "package test; public class Impl { public static String get() { return \"base\"; } }"), new SourceFileObject("api/API.java", "package api; public class API { public static String run() { return test.Impl.get(); } }"))) .call(); File classes9 = new File(new File(new File(classes, "META-INF"), "versions"), "9"); classes9.mkdirs(); ToolProvider.getSystemJavaCompiler() - .getTask(null, null, d -> { throw new IllegalStateException(d.toString()); }, Arrays.asList("-d", classes9.getAbsolutePath(), "-classpath", classes.getAbsolutePath()), null, + .getTask(null, null, d -> { throw new IllegalStateException(d.toString()); }, Arrays.asList("-d", classes9.getAbsolutePath(), "-classpath", classes.getAbsolutePath(), "-proc:none"), null, Arrays.asList(new SourceFileObject("test/Impl.java", "package test; public class Impl { public static String get() { return \"9\"; } }"))) .call(); Map jarContent = new LinkedHashMap<>(); From 08d23c6a15c3f3fd132a2fc91eb9f12cdd850b2d Mon Sep 17 00:00:00 2001 From: Michael Bien Date: Mon, 26 Jan 2026 19:48:59 +0100 Subject: [PATCH 09/13] disable test cases where PropertyChange event count changed --- .../org/netbeans/beaninfo/editors/FileArrayEditorTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/platform/o.n.core/test/unit/src/org/netbeans/beaninfo/editors/FileArrayEditorTest.java b/platform/o.n.core/test/unit/src/org/netbeans/beaninfo/editors/FileArrayEditorTest.java index 2d9dd126a962..d7645630854a 100644 --- a/platform/o.n.core/test/unit/src/org/netbeans/beaninfo/editors/FileArrayEditorTest.java +++ b/platform/o.n.core/test/unit/src/org/netbeans/beaninfo/editors/FileArrayEditorTest.java @@ -25,6 +25,7 @@ import java.util.List; import javax.swing.JFileChooser; +import org.junit.Ignore; import org.netbeans.junit.NbTestCase; @@ -80,6 +81,7 @@ public void testSetCombinedSingleSelection(){ myListener.size()+ " event(s)"; } + @Ignore("TODO: JDK 20+: event count is now 3") public void testSetCombinedMultiSelection(){ JFileChooser chooser = new JFileChooser(); FileArrayEditor editor = getArrayEditor(); @@ -93,6 +95,7 @@ public void testSetCombinedMultiSelection(){ myListener.size()+ " event(s)"; } + @Ignore("TODO: JDK 20+: event count is now 4") public void testSetCombinedOtherOrderSelection(){ JFileChooser chooser = new JFileChooser(); FileArrayEditor editor = getArrayEditor(); @@ -109,6 +112,7 @@ public void testSetCombinedOtherOrderSelection(){ /* * This test for fix IZ#36742 */ + @Ignore("TODO: JDK 20+: event count is now 3") public void testDoubleSelection(){ JFileChooser chooser = new JFileChooser(); FileArrayEditor editor = getArrayEditor(); From 02bafcc4c8dd36bbc785d8b8a7f340ef197c96aa Mon Sep 17 00:00:00 2001 From: Michael Bien Date: Mon, 26 Jan 2026 22:32:40 +0100 Subject: [PATCH 10/13] fix ambiguous CL problem in tests on JDK 21 by reducing clusters --- .../modules/netbinox/CachingAndExternalPathsTest.java | 5 ++++- .../org/netbeans/modules/netbinox/IntegrationTest.java | 9 ++++----- .../netbeans/modules/netbinox/LogReaderServiceTest.java | 5 ++++- .../modules/netbinox/NetigsoHasSAXParserTest.java | 5 ++++- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/CachingAndExternalPathsTest.java b/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/CachingAndExternalPathsTest.java index 36074ff0154a..1e2b8e1d8903 100644 --- a/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/CachingAndExternalPathsTest.java +++ b/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/CachingAndExternalPathsTest.java @@ -81,7 +81,10 @@ static Test create(SetExtDirProperty setter) throws IOException { NbTestSuite suite = new NbTestSuite(); Compile compile = new Compile("testCompile", setter); suite.addTest(compile); - NbModuleSuite.Configuration common = NbModuleSuite.emptyConfiguration().clusters(".*").enableClasspathModules(false) + // JDK 21+: enabling all clusters causes: + // Will not load class org.osgi.framework.launch.Framework arbitrarily from one of + // ModuleCL@1070d437[org.netbeans.libs.osgi] and ModuleCL@2bbc301d[org.netbeans.modules.websvc.restlib] + NbModuleSuite.Configuration common = NbModuleSuite.emptyConfiguration().clusters("platform|ide").enableClasspathModules(false) .gui(false).honorAutoloadEager(true); { NbModuleSuite.Configuration conf = common.reuseUserDir(false).addTest(CachingAndExternalPathsTest.class, "testInitUserDir"); diff --git a/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/IntegrationTest.java b/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/IntegrationTest.java index c2baa86f6ba8..aa1b6ec47ebe 100644 --- a/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/IntegrationTest.java +++ b/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/IntegrationTest.java @@ -19,19 +19,15 @@ package org.netbeans.modules.netbinox; import java.io.File; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.logging.Level; import junit.framework.Test; import org.netbeans.Module; import org.netbeans.ModuleManager; -import org.netbeans.NetigsoFramework; import org.netbeans.SetupHid; import org.netbeans.core.netigso.NetigsoUtil; import org.netbeans.core.startup.Main; import org.netbeans.junit.NbModuleSuite; import org.netbeans.junit.NbTestCase; -import org.openide.util.Lookup; import org.osgi.framework.Bundle; import org.osgi.framework.launch.Framework; @@ -47,11 +43,14 @@ public IntegrationTest(String name) { } public static Test suite() { + // JDK 21+: enabling all clusters causes: + // Will not load class org.osgi.framework.launch.Framework arbitrarily from one of + // ModuleCL@1070d437[org.netbeans.libs.osgi] and ModuleCL@2bbc301d[org.netbeans.modules.websvc.restlib] return NbModuleSuite.create( NbModuleSuite.emptyConfiguration().addTest( IntegrationTest.class ).honorAutoloadEager(true).clusters( - ".*" + "platform|ide" ).failOnException(Level.WARNING)/*.failOnMessage(Level.WARNING)*/ .gui(false) ); diff --git a/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/LogReaderServiceTest.java b/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/LogReaderServiceTest.java index 9fb8b2e81a61..4d1e7a6806c9 100644 --- a/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/LogReaderServiceTest.java +++ b/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/LogReaderServiceTest.java @@ -40,11 +40,14 @@ public LogReaderServiceTest(String name) { } public static Test suite() { + // JDK 21+: enabling all clusters causes: + // Will not load class org.osgi.framework.launch.Framework arbitrarily from one of + // ModuleCL@1070d437[org.netbeans.libs.osgi] and ModuleCL@2bbc301d[org.netbeans.modules.websvc.restlib] return NbModuleSuite.create( NbModuleSuite.emptyConfiguration() .addTest(LogReaderServiceTest.class) .honorAutoloadEager(true) - .clusters(".*") + .clusters("platform|ide") .failOnException(Level.WARNING)/*.failOnMessage(Level.WARNING)*/ .gui(false) ); diff --git a/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/NetigsoHasSAXParserTest.java b/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/NetigsoHasSAXParserTest.java index f5ee8145e762..cc85e7b491f9 100644 --- a/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/NetigsoHasSAXParserTest.java +++ b/platform/netbinox/test/unit/src/org/netbeans/modules/netbinox/NetigsoHasSAXParserTest.java @@ -39,11 +39,14 @@ public NetigsoHasSAXParserTest(String name) { } public static Test suite() { + // JDK 21+: enabling all clusters causes: + // Will not load class org.osgi.framework.launch.Framework arbitrarily from one of + // ModuleCL@1070d437[org.netbeans.libs.osgi] and ModuleCL@2bbc301d[org.netbeans.modules.websvc.restlib] return NbModuleSuite.create( NbModuleSuite.emptyConfiguration().addTest( NetigsoHasSAXParserTest.class ).honorAutoloadEager(true).clusters( - ".*" + "platform|ide" ).failOnException(Level.WARNING)/*.failOnMessage(Level.WARNING)*/ .gui(false) ); From b9fb370974be10eadeb668ba25e2c454f045a020 Mon Sep 17 00:00:00 2001 From: Michael Bien Date: Thu, 12 Feb 2026 18:15:03 +0100 Subject: [PATCH 11/13] Bump build and start requirements to JDK 21 (README and Main.java) --- README.md | 2 +- platform/o.n.bootstrap/src/org/netbeans/Main.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a73b134d9098..45e92d3bb0b9 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Apache NetBeans is an open source development environment, tooling platform, and * Git * Ant - * JDK 17 or above (to build and run NetBeans) + * JDK 21 or above (to build and run NetBeans) #### Notes: diff --git a/platform/o.n.bootstrap/src/org/netbeans/Main.java b/platform/o.n.bootstrap/src/org/netbeans/Main.java index 15337fc08193..fd54398db2b4 100644 --- a/platform/o.n.bootstrap/src/org/netbeans/Main.java +++ b/platform/o.n.bootstrap/src/org/netbeans/Main.java @@ -37,7 +37,7 @@ private Main() { public static void main (String args[]) throws Exception { // following code has to execute on java 8 - e.g. do not use // NbBundle or any other library - int required = 17; + int required = 21; if (Boolean.getBoolean("bootstrap.disableJDKCheck")) { System.err.println(getMessage("MSG_WarnJavaCheckDisabled")); From cd8f11351a6268b56ad2883de7d5e40cf3d5f615 Mon Sep 17 00:00:00 2001 From: Michael Bien Date: Fri, 13 Feb 2026 21:14:44 +0100 Subject: [PATCH 12/13] remove bootstrap.disableJDKCheck=true where no longer needed --- .../apisupport/project/suite/BuildZipDistributionTest.java | 2 +- .../modules/autoupdate/services/NewClustersRebootTest.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apisupport/apisupport.ant/test/unit/src/org/netbeans/modules/apisupport/project/suite/BuildZipDistributionTest.java b/apisupport/apisupport.ant/test/unit/src/org/netbeans/modules/apisupport/project/suite/BuildZipDistributionTest.java index 3e0356e34caf..db90bc075d32 100644 --- a/apisupport/apisupport.ant/test/unit/src/org/netbeans/modules/apisupport/project/suite/BuildZipDistributionTest.java +++ b/apisupport/apisupport.ant/test/unit/src/org/netbeans/modules/apisupport/project/suite/BuildZipDistributionTest.java @@ -233,7 +233,7 @@ private void run(File nbexec, String... args) throws Exception { assertTrue("file found: " + testf, testf.exists()); LinkedList allArgs = new LinkedList(Arrays.asList(args)); - allArgs.addFirst("-J-Dbootstrap.disableJDKCheck=true"); + allArgs.addFirst("-J-Dbootstrap.disableJDKCheck=true"); // TODO remove once tests can run on JDK 21 allArgs.addFirst("-J-Dnetbeans.mainclass=" + MainCallback.class.getName()); allArgs.addFirst(getWorkDirPath()); allArgs.addFirst("--userdir"); diff --git a/platform/autoupdate.services/test/unit/src/org/netbeans/modules/autoupdate/services/NewClustersRebootTest.java b/platform/autoupdate.services/test/unit/src/org/netbeans/modules/autoupdate/services/NewClustersRebootTest.java index f2e8f3f0dd1d..cda8af5e679d 100644 --- a/platform/autoupdate.services/test/unit/src/org/netbeans/modules/autoupdate/services/NewClustersRebootTest.java +++ b/platform/autoupdate.services/test/unit/src/org/netbeans/modules/autoupdate/services/NewClustersRebootTest.java @@ -93,7 +93,6 @@ private void invokeNbExecAndCreateCluster(File workDir, StringBuffer sb, String. assertTrue("file found: " + testf, testf.exists()); LinkedList allArgs = new LinkedList(Arrays.asList(args)); - allArgs.addFirst("-J-Dbootstrap.disableJDKCheck=true"); allArgs.addFirst("-J-Dnetbeans.mainclass=" + NewClustersRebootCallback.class.getName()); allArgs.addFirst(System.getProperty("java.home")); allArgs.addFirst("--jdkhome"); From 54bc32a6344cd99e4a02fba5c91ceaf9bae55ef6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Bl=C3=A4sing?= Date: Mon, 2 Feb 2026 17:35:44 +0100 Subject: [PATCH 13/13] Update to apache lucene 10.X - Additional package lucene-analysis-common required (KeywordAnalyzer, WhitespaceAnalyzer, PerFieldAnalyzerWrapper, CharTokenizer, LimitTokenCountAnalyzer) - FieldTypes were introduced to carry the field behavior (tokenization state, indexing options, storage settings) - BooleanQuery construction moved to builder - Collector interface was modified - TermEnum was replaced by TermsEnum - Terms are stored per field, so queries have to be specified with the target field - QuerySelectors were replaced by String sets - RAMDirecory was removed and is replaced by ByteBufferDirectories - Lock handling was reworked --- ide/csl.api/nbproject/project.xml | 4 +- ide/jumpto/nbproject/project.xml | 4 +- ide/libs.lucene/external/binaries-list | 3 +- ...nse.txt => lucene-core-10.3.2-license.txt} | 185 ++++++++- .../external/lucene-core-10.3.2-notice.txt | 197 ++++++++++ .../external/lucene-core-3.6.2-notice.txt | 92 ----- ide/libs.lucene/manifest.mf | 4 +- ide/libs.lucene/nbproject/project.properties | 5 +- ide/libs.lucene/nbproject/project.xml | 8 +- ide/parsing.api/nbproject/project.xml | 4 +- ide/parsing.indexing/nbproject/project.xml | 8 +- .../impl/indexing/ClusteredIndexables.java | 81 +++- .../indexing/lucene/LayeredDocumentIndex.java | 2 +- .../spi/indexing/support/QuerySupport.java | 10 +- .../impl/indexing/DocumentStoreTest.java | 2 +- ide/parsing.lucene/manifest.mf | 2 +- .../org-netbeans-modules-parsing-lucene.sig | 33 +- .../nbproject/project.properties | 4 +- ide/parsing.lucene/nbproject/project.xml | 4 +- .../parsing/lucene/AllFieldsSelector.java | 38 -- .../parsing/lucene/BitSetCollector.java | 22 +- .../modules/parsing/lucene/Convertors.java | 48 ++- .../parsing/lucene/DocumentIndexImpl.java | 3 +- .../parsing/lucene/IndexDocumentImpl.java | 62 ++- .../modules/parsing/lucene/LuceneIndex.java | 316 ++++++++------- .../modules/parsing/lucene/MemoryIndex.java | 192 +++++---- .../lucene/RecordOwnerLockFactory.java | 156 +++----- .../modules/parsing/lucene/RegexpFilter.java | 141 +++++++ .../parsing/lucene/SupportAccessor.java | 4 +- .../modules/parsing/lucene/TermCollector.java | 48 ++- .../modules/parsing/lucene/support/Index.java | 25 +- .../parsing/lucene/support/IndexManager.java | 6 +- .../parsing/lucene/support/Queries.java | 368 +----------------- .../parsing/lucene/AsyncCloseTest.java | 10 +- .../parsing/lucene/IndexTransactionTest.java | 14 +- .../parsing/lucene/LuceneIndexTest.java | 10 +- .../lucene/NativeFSLockFactoryTest.java | 14 +- .../parsing/lucene/RawIndexPerfTest.java | 33 +- .../lucene/RecordOwnerLockFactoryTest.java | 100 +++-- ...QueriesTest.java => RegexpFilterTest.java} | 15 +- java/java.hints/nbproject/project.xml | 4 +- java/java.lsp.server/nbproject/project.xml | 4 +- java/java.source.base/nbproject/project.xml | 8 +- .../java/source/usages/ClassIndexImpl.java | 3 +- .../java/source/usages/DocumentUtil.java | 152 +++++--- .../source/usages/PersistentClassIndex.java | 34 +- .../modules/java/source/usages/QueryUtil.java | 173 ++------ .../api/java/source/JavaSourceTest.java | 19 +- .../source/usages/LucenePerformanceTest.java | 5 +- java/java.source/nbproject/project.xml | 4 +- java/java.sourceui/nbproject/project.xml | 8 +- java/spi.java.hints/nbproject/project.xml | 4 +- java/whitelist/nbproject/project.xml | 4 +- nbbuild/licenses/Apache-2.0-lucene | 221 +++++++++-- 54 files changed, 1601 insertions(+), 1319 deletions(-) rename ide/libs.lucene/external/{lucene-core-3.6.2-license.txt => lucene-core-10.3.2-license.txt} (67%) create mode 100644 ide/libs.lucene/external/lucene-core-10.3.2-notice.txt delete mode 100644 ide/libs.lucene/external/lucene-core-3.6.2-notice.txt delete mode 100644 ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/AllFieldsSelector.java create mode 100644 ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/RegexpFilter.java rename ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/{support/QueriesTest.java => RegexpFilterTest.java} (78%) diff --git a/ide/csl.api/nbproject/project.xml b/ide/csl.api/nbproject/project.xml index 632dbef71b33..c8a8271063a8 100644 --- a/ide/csl.api/nbproject/project.xml +++ b/ide/csl.api/nbproject/project.xml @@ -75,8 +75,8 @@ - 3 - 3.0 + 10 + 10.0 diff --git a/ide/jumpto/nbproject/project.xml b/ide/jumpto/nbproject/project.xml index c0935153335b..1be4b76e1af3 100644 --- a/ide/jumpto/nbproject/project.xml +++ b/ide/jumpto/nbproject/project.xml @@ -106,8 +106,8 @@ - 2 - 2.31 + 3 + 3.0 diff --git a/ide/libs.lucene/external/binaries-list b/ide/libs.lucene/external/binaries-list index 72c2399b4c70..99f4a6192c43 100644 --- a/ide/libs.lucene/external/binaries-list +++ b/ide/libs.lucene/external/binaries-list @@ -14,4 +14,5 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -9EC77E2507F9CC01756964C71D91EFD8154A8C47 org.apache.lucene:lucene-core:3.6.2 +E46AFE546D01BB56A87860AD60889945F29E7E51 org.apache.lucene:lucene-core:10.3.2 +6ACEB9BC2FA78798E14E5318ED1C4251F99FCFAC org.apache.lucene:lucene-analysis-common:10.3.2 diff --git a/ide/libs.lucene/external/lucene-core-3.6.2-license.txt b/ide/libs.lucene/external/lucene-core-10.3.2-license.txt similarity index 67% rename from ide/libs.lucene/external/lucene-core-3.6.2-license.txt rename to ide/libs.lucene/external/lucene-core-10.3.2-license.txt index bceaf4264edf..f68511077bc0 100644 --- a/ide/libs.lucene/external/lucene-core-3.6.2-license.txt +++ b/ide/libs.lucene/external/lucene-core-10.3.2-license.txt @@ -1,10 +1,11 @@ Name: Apache Lucene Description: Java-based indexing and search technology -Version: 3.6.2 +Version: 10.3.2 Origin: Apache Software Foundation License: Apache-2.0-lucene URL: https://lucene.apache.org/core/ -Source: https://archive.apache.org/dist/lucene/java/3.6.2/ +Source: https://archive.apache.org/dist/lucene/java/10.3.2/ +Files: lucene-core-10.3.2.jar lucene-analysis-common-10.3.2.jar Apache License @@ -211,7 +212,7 @@ Source: https://archive.apache.org/dist/lucene/java/3.6.2/ -Some code in src/java/org/apache/lucene/util/UnicodeUtil.java was +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was derived from unicode conversion examples available at http://www.unicode.org/Public/PROGRAMS/CVTUTF. Here is the copyright from those sources: @@ -239,17 +240,80 @@ from those sources: */ -Some code in src/java/org/apache/lucene/util/ArrayUtil.java was +Some code in core/src/java/org/apache/lucene/util/ArrayUtil.java was derived from Python 2.4.2 sources available at http://www.python.org. Full license is here: http://www.python.org/download/releases/2.4.2/license/ +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from Python 3.1.2 sources available at +http://www.python.org. Full license is here: + + http://www.python.org/download/releases/3.1.2/license/ + +Some code in core/src/java/org/apache/lucene/util/automaton was +derived from Brics automaton sources available at +www.brics.dk/automaton/. Here is the copyright from those sources: -Some code in src/java/org/apache/lucene/util/UnicodeUtil.java was +/* + * Copyright (c) 2001-2009 Anders Moeller + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +The levenshtein automata tables in core/src/java/org/apache/lucene/util/automaton +were automatically generated with the moman/finenight FSA package. +Here is the copyright for those sources: + +# Copyright (c) 2010, Jean-Philippe Barrette-LaPierre, +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was derived from ICU (http://www.icu-project.org) The full license is available here: - http://source.icu-project.org/repos/icu/icu/trunk/license.html + https://github.com/unicode-org/icu/blob/main/icu4c/LICENSE /* * Copyright (C) 1999-2010, International Business Machines @@ -341,3 +405,112 @@ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The following license applies to the Morfologik project: + +Copyright (c) 2006 Dawid Weiss +Copyright (c) 2007-2011 Dawid Weiss, Marcin Miłkowski +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of Morfologik nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +The dictionary comes from Morfologik project. Morfologik uses data from +Polish ispell/myspell dictionary hosted at http://www.sjp.pl/slownik/en/ and +is licenced on the terms of (inter alia) LGPL and Creative Commons +ShareAlike. The part-of-speech tags were added in Morfologik project and +are not found in the data from sjp.pl. The tagset is similar to IPI PAN +tagset. + +--- + +The following license applies to the Morfeusz project, +used by org.apache.lucene.analysis.morfologik. + +BSD-licensed dictionary of Polish (SGJP) +http://sgjp.pl/morfeusz/ + +Copyright © 2011 Zygmunt Saloni, Włodzimierz Gruszczyński, +Marcin Woliński, Robert Wołosz + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDERS “AS IS” AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +core/src/java/org/apache/lucene/util/compress/LZ4.java is a Java +implementation of the LZ4 (https://github.com/lz4/lz4/tree/dev/lib) +compression format for Lucene's DataInput/DataOutput abstractions. + +LZ4 Library +Copyright (c) 2011-2016, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/ide/libs.lucene/external/lucene-core-10.3.2-notice.txt b/ide/libs.lucene/external/lucene-core-10.3.2-notice.txt new file mode 100644 index 000000000000..d7900a6a0da4 --- /dev/null +++ b/ide/libs.lucene/external/lucene-core-10.3.2-notice.txt @@ -0,0 +1,197 @@ +Apache Lucene +Copyright 2001-2025 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Includes software from other Apache Software Foundation projects, +including, but not limited to: + - Apache Jakarta Regexp + - Apache Commons + - Apache Xerces + +ICU4J, (under analysis/icu) is licensed under an MIT styles license +and Copyright (c) 1995-2008 International Business Machines Corporation and others + +Some data files (under analysis/icu/src/data) are derived from Unicode data such +as the Unicode Character Database. See http://unicode.org/copyright.html for more +details. + +Brics Automaton (under core/src/java/org/apache/lucene/util/automaton) is +BSD-licensed, created by Anders Møller. See http://www.brics.dk/automaton/ + +The levenshtein automata tables (under core/src/java/org/apache/lucene/util/automaton) were +automatically generated with the moman/finenight FSA library, created by +Jean-Philippe Barrette-LaPierre. This library is available under an MIT license, +see http://sites.google.com/site/rrettesite/moman and +http://bitbucket.org/jpbarrette/moman/overview/ + +The class org.apache.lucene.util.WeakIdentityMap was derived from +the Apache CXF project and is Apache License 2.0. + +The class org.apache.lucene.util.compress.LZ4 is a Java rewrite of the LZ4 +compression library (https://github.com/lz4/lz4/tree/dev/lib) that is licensed +under the 2-clause BSD license. +(https://opensource.org/licenses/bsd-license.php) + +The Google Code Prettify is Apache License 2.0. +See http://code.google.com/p/google-code-prettify/ + +This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin +g Package (jaspell): http://jaspell.sourceforge.net/ +License: The BSD License (http://www.opensource.org/licenses/bsd-license.php) + +The snowball stemmers in + analysis/common/src/java/net/sf/snowball +were developed by Martin Porter and Richard Boulton. +The snowball stopword lists in + analysis/common/src/resources/org/apache/lucene/analysis/snowball +were developed by Martin Porter and Richard Boulton. +The full snowball package is available from + https://snowballstem.org/ + +The KStem stemmer in + analysis/common/src/org/apache/lucene/analysis/en +was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst) +under the BSD-license. + +The Arabic,Persian,Romanian,Bulgarian, Hindi and Bengali analyzers (common) come with a default +stopword list that is BSD-licensed created by Jacques Savoy. These files reside in: +analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/bn/stopwords.txt +See http://members.unine.ch/jacques.savoy/clef/index.html. + +The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers +(common) are based on BSD-licensed reference implementations created by Jacques Savoy and +Ljiljana Dolamic. These files reside in: +analysis/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java + +The Stempel analyzer (stempel) includes BSD-licensed software developed +by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil, +and Edmond Nolan. + +The Polish analyzer (stempel) comes with a default +stopword list that is BSD-licensed created by the Carrot2 project. The file resides +in stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt. +See https://github.com/carrot2/carrot2. + +The SmartChineseAnalyzer source code (smartcn) was +provided by Xiaoping Gao and copyright 2009 by www.imdict.net. + +WordBreakTestUnicode_*.java (under modules/analysis/common/src/test/) +is derived from Unicode data such as the Unicode Character Database. +See http://unicode.org/copyright.html for more details. + +The Morfologik analyzer (morfologik) includes BSD-licensed software +developed by Dawid Weiss and Marcin Miłkowski +(https://github.com/morfologik/morfologik-stemming) and uses +data from the BSD-licensed dictionary of Polish (SGJP, http://sgjp.pl/morfeusz/). + +=========================================================================== +Kuromoji Japanese Morphological Analyzer - Apache Lucene Integration +=========================================================================== + +This software includes a binary and/or source version of data from + + mecab-ipadic-2.7.0-20070801 + +which can be obtained from + + http://atilika.com/releases/mecab-ipadic/mecab-ipadic-2.7.0-20070801.tar.gz + +or + + http://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz + +=========================================================================== +mecab-ipadic-2.7.0-20070801 Notice +=========================================================================== + +Nara Institute of Science and Technology (NAIST), +the copyright holders, disclaims all warranties with regard to this +software, including all implied warranties of merchantability and +fitness, in no event shall NAIST be liable for +any special, indirect or consequential damages or any damages +whatsoever resulting from loss of use, data or profits, whether in an +action of contract, negligence or other tortuous action, arising out +of or in connection with the use or performance of this software. + +A large portion of the dictionary entries +originate from ICOT Free Software. The following conditions for ICOT +Free Software applies to the current dictionary as well. + +Each User may also freely distribute the Program, whether in its +original form or modified, to any third party or parties, PROVIDED +that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear +on, or be attached to, the Program, which is distributed substantially +in the same form as set out herein and that such intended +distribution, if actually made, will neither violate or otherwise +contravene any of the laws and regulations of the countries having +jurisdiction over the User or the intended distribution itself. + +NO WARRANTY + +The program was produced on an experimental basis in the course of the +research and development conducted during the project and is provided +to users as so produced on an experimental basis. Accordingly, the +program is provided without any warranty whatsoever, whether express, +implied, statutory or otherwise. The term "warranty" used herein +includes, but is not limited to, any warranty of the quality, +performance, merchantability and fitness for a particular purpose of +the program and the nonexistence of any infringement or violation of +any right of any third party. + +Each user of the program will agree and understand, and be deemed to +have agreed and understood, that there is no warranty whatsoever for +the program and, accordingly, the entire risk arising from or +otherwise connected with the program is assumed by the user. + +Therefore, neither ICOT, the copyright holder, or any other +organization that participated in or was otherwise related to the +development of the program and their respective officials, directors, +officers and other employees shall be held liable for any and all +damages, including, without limitation, general, special, incidental +and consequential damages, arising out of or otherwise in connection +with the use or inability to use the program or any product, material +or result produced or otherwise obtained by using the program, +regardless of whether they have been advised of, or otherwise had +knowledge of, the possibility of such damages at any time during the +project or thereafter. Each user will be deemed to have agreed to the +foregoing by his or her commencement of use of the program. The term +"use" as used herein includes, but is not limited to, the use, +modification, copying and distribution of the program and the +production of secondary products from the program. + +In the case where the program, whether in its original form or +modified, was distributed or delivered to or received by a user from +any person, organization or entity other than ICOT, unless it makes or +grants independently of ICOT any specific warranty to the user in +writing, such person, organization or entity, will also be exempted +from and not be held liable to the user for any such damages as noted +above as far as the program is concerned. + +=========================================================================== +Nori Korean Morphological Analyzer - Apache Lucene Integration +=========================================================================== + +This software includes a binary and/or source version of data from + + mecab-ko-dic-2.1.1-20180720 + +which can be obtained from + + https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz diff --git a/ide/libs.lucene/external/lucene-core-3.6.2-notice.txt b/ide/libs.lucene/external/lucene-core-3.6.2-notice.txt deleted file mode 100644 index f6e4cce6d7c6..000000000000 --- a/ide/libs.lucene/external/lucene-core-3.6.2-notice.txt +++ /dev/null @@ -1,92 +0,0 @@ -Apache Lucene -Copyright 2019 The Apache Software Foundation - -The snowball stemmers in - contrib/analyzers/common/src/java/net/sf/snowball -were developed by Martin Porter and Richard Boulton. -The snowball stopword lists in - contrib/analyzers/common/src/resources/org/apache/lucene/analysis/snowball -were developed by Martin Porter and Richard Boulton. -The full snowball package is available from - http://snowball.tartarus.org/ - -The KStem stemmer in - common/src/org/apache/lucene/analysis/en -was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst) -under the BSD-license. - -The Arabic,Persian,Romanian,Bulgarian, and Hindi analyzers (contrib/analyzers) come with a default -stopword list that is BSD-licensed created by Jacques Savoy. These files reside in: -contrib/analyzers/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt, -contrib/analyzers/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt, -contrib/analyzers/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt, -contrib/analyzers/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt, -contrib/analyzers/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt -See http://members.unine.ch/jacques.savoy/clef/index.html. - -The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers -(common) are based on BSD-licensed reference implementations created by Jacques Savoy and -Ljiljana Dolamic. These files reside in: -contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java -contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java -contrib/analyzers/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java -contrib/analyzers/common/src/java/org/apache/lucene/analysis/fi/FinnishLightStemmer.java -contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchLightStemmer.java -contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchMinimalStemmer.java -contrib/analyzers/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java -contrib/analyzers/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemmer.java -contrib/analyzers/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.java -contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java -contrib/analyzers/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java - -The Stempel analyzer (contrib/analyzers) includes BSD-licensed software developed -by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil, -and Edmond Nolan. - -The Polish analyzer (contrib/analyzers) comes with a default -stopword list that is BSD-licensed created by the Carrot2 project. The file resides -in contrib/analyzers/stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt. -See http://project.carrot2.org/license.html. - -Includes lib/servlet-api-2.4.jar from Apache Tomcat -Includes lib/ant-1.7.1.jar and lib/ant-junit-1.7.1.jar from Apache Ant -Includes contrib/queries/lib/jakarta-regexp-1.4.jar from Apache Jakarta Regexp -Includes software from other Apache Software Foundation projects, -including, but not limited to: - - Commons Beanutils (contrib/benchmark/lib/commons-beanutils-1.7.0.jar) - - Commons Collections (contrib/benchmark/lib/commons-collections-3.1.jar) - - Commons Compress (contrib/benchmark/lib/commons-compress-1.0.jar) - - Commons Digester (contrib/benchmark/lib/commons-digester-1.7.jar) - - Commons Logging (contrib/benchmark/lib/commons-logging-1.0.4.jar) - - Xerces (contrib/benchmark/lib/xercesImpl-2.9.1-patched-XERCESJ-1257.jar) - -The SmartChineseAnalyzer source code (under contrib/analyzers) was -provided by Xiaoping Gao and copyright 2009 by www.imdict.net. - -ICU4J, (under contrib/icu) is licensed under an MIT styles license -(contrib/icu/lib/ICU-LICENSE.txt) and Copyright (c) 1995-2008 -International Business Machines Corporation and others - -Some files (contrib/analyzers/common/src/test/.../WordBreakTestUnicode_*.java -and data files under contrib/icu/src/data/) are derived from Unicode data such -as the Unicode Character Database. See http://unicode.org/copyright.html for more -details. - -The class org.apache.lucene.SorterTemplate was inspired by CGLIB's class with -the same name. The implementation part is mainly done using pre-existing -Lucene sorting code. In-place stable mergesort was borrowed from CGLIB, -which is Apache-licensed. - -The Google Code Prettify is Apache License 2.0. -See http://code.google.com/p/google-code-prettify/ - -JUnit (under lib/junit-4.7.jar) is licensed under the Common Public License v. 1.0 -See http://junit.sourceforge.net/cpl-v10.html - -JLine (under contrib/lucli/lib/jline.jar) is licensed under the BSD License. -See http://jline.sourceforge.net/ - -This product includes code (JaspellTernarySearchTrie) from Java Spelling Checking Package (jaspell): http://jaspell.sourceforge.net/ -License: The BSD License (http://www.opensource.org/licenses/bsd-license.php) - - diff --git a/ide/libs.lucene/manifest.mf b/ide/libs.lucene/manifest.mf index 2721dbd39724..6c8b6c20cdeb 100644 --- a/ide/libs.lucene/manifest.mf +++ b/ide/libs.lucene/manifest.mf @@ -1,4 +1,4 @@ Manifest-Version: 1.0 -OpenIDE-Module: org.netbeans.libs.lucene/3 +OpenIDE-Module: org.netbeans.libs.lucene/10 OpenIDE-Module-Localizing-Bundle: org/netbeans/libs/lucene/Bundle.properties -OpenIDE-Module-Specification-Version: 3.51 +OpenIDE-Module-Specification-Version: 10.0 diff --git a/ide/libs.lucene/nbproject/project.properties b/ide/libs.lucene/nbproject/project.properties index 833412719a8c..9a16f3e59876 100644 --- a/ide/libs.lucene/nbproject/project.properties +++ b/ide/libs.lucene/nbproject/project.properties @@ -16,9 +16,10 @@ # under the License. is.autoload=true -javac.source=1.8 +javac.release=21 javac.compilerargs=-Xlint -Xlint:-serial -release.external/lucene-core-3.6.2.jar=modules/ext/lucene-core-3.6.2.jar +release.external/lucene-core-10.3.2.jar=modules/ext/lucene-core-10.3.2.jar +release.external/lucene-analysis-common-10.3.2.jar=modules/ext/lucene-analysis-common-10.3.2.jar # This is an very old library, the complete dependencies were never explored. # Since subpackage-s are working in new sigtest version, generation of the diff --git a/ide/libs.lucene/nbproject/project.xml b/ide/libs.lucene/nbproject/project.xml index 90bbd89df36a..dfae0826407e 100644 --- a/ide/libs.lucene/nbproject/project.xml +++ b/ide/libs.lucene/nbproject/project.xml @@ -29,8 +29,12 @@ org.apache.lucene - ext/lucene-core-3.6.2.jar - external/lucene-core-3.6.2.jar + ext/lucene-core-10.3.2.jar + external/lucene-core-10.3.2.jar + + + ext/lucene-analysis-common-10.3.2.jar + external/lucene-analysis-common-10.3.2.jar diff --git a/ide/parsing.api/nbproject/project.xml b/ide/parsing.api/nbproject/project.xml index a172bcfca330..39c3cf55fb94 100644 --- a/ide/parsing.api/nbproject/project.xml +++ b/ide/parsing.api/nbproject/project.xml @@ -83,8 +83,8 @@ - 2 - 2.28 + 3 + 3.0 diff --git a/ide/parsing.indexing/nbproject/project.xml b/ide/parsing.indexing/nbproject/project.xml index 0093b40a5a3c..9473a9b85db1 100644 --- a/ide/parsing.indexing/nbproject/project.xml +++ b/ide/parsing.indexing/nbproject/project.xml @@ -57,8 +57,8 @@ - 3 - 3.16 + 10 + 10.0 @@ -101,8 +101,8 @@ - 2 - 2.31 + 3 + 3.0 diff --git a/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/ClusteredIndexables.java b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/ClusteredIndexables.java index 938caac0b010..fca065095adb 100644 --- a/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/ClusteredIndexables.java +++ b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/ClusteredIndexables.java @@ -41,7 +41,9 @@ import java.util.logging.Logger; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.Fieldable; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableFieldType; import org.netbeans.api.annotations.common.NonNull; import org.netbeans.api.annotations.common.NullAllowed; import org.netbeans.modules.parsing.impl.indexing.lucene.DocumentBasedIndexManager; @@ -64,7 +66,34 @@ public final class ClusteredIndexables { public static final String FIELD_PRIMARY_KEY = "_sn"; //NOI18N public static final String DELETE = "ci-delete-set"; //NOI18N public static final String INDEX = "ci-index-set"; //NOI18N - + + private static final IndexableFieldType STORED_SEARCHABLE; + private static final IndexableFieldType NON_STORED_SEARCHABLE; + private static final IndexableFieldType STORED_NON_SEARCHABLE; + private static final IndexableFieldType NON_STORED_NON_SEARCHABLE; + static { + STORED_SEARCHABLE = new FieldType(); + ((FieldType) STORED_SEARCHABLE).setStored(true); + ((FieldType) STORED_SEARCHABLE).setTokenized(true); + ((FieldType) STORED_SEARCHABLE).setIndexOptions(IndexOptions.DOCS_AND_FREQS); + ((FieldType) STORED_SEARCHABLE).freeze(); + NON_STORED_SEARCHABLE = new FieldType(); + ((FieldType) NON_STORED_SEARCHABLE).setStored(false); + ((FieldType) NON_STORED_SEARCHABLE).setTokenized(true); + ((FieldType) NON_STORED_SEARCHABLE).setIndexOptions(IndexOptions.DOCS_AND_FREQS); + ((FieldType) NON_STORED_SEARCHABLE).freeze(); + STORED_NON_SEARCHABLE = new FieldType(); + ((FieldType) STORED_NON_SEARCHABLE).setStored(true); + ((FieldType) STORED_NON_SEARCHABLE).setTokenized(false); + ((FieldType) STORED_NON_SEARCHABLE).setIndexOptions(IndexOptions.NONE); + ((FieldType) STORED_NON_SEARCHABLE).freeze(); + NON_STORED_NON_SEARCHABLE = new FieldType(); + ((FieldType) NON_STORED_NON_SEARCHABLE).setStored(false); + ((FieldType) NON_STORED_NON_SEARCHABLE).setTokenized(false); + ((FieldType) NON_STORED_NON_SEARCHABLE).setIndexOptions(IndexOptions.NONE); + ((FieldType) NON_STORED_NON_SEARCHABLE).freeze(); + } + // ----------------------------------------------------------------------- // Public implementation // ----------------------------------------------------------------------- @@ -122,6 +151,21 @@ public static interface AttachableDocumentIndexCache extends DocumentIndexCache. void detach(); } + private static Field createField(String key, String value, boolean searchable, boolean stored) { + IndexableFieldType ift = null; + if(stored && searchable) { + ift = STORED_SEARCHABLE; + } else if ((! stored) && searchable) { + ift = NON_STORED_SEARCHABLE; + } else if (stored && (! searchable)) { + ift = STORED_NON_SEARCHABLE; + } else if ((! stored) && (! searchable)) { + ift = NON_STORED_NON_SEARCHABLE; + } + assert ift != null; + return new Field(key, value, ift); + } + // ----------------------------------------------------------------------- // Private implementation // ----------------------------------------------------------------------- @@ -720,7 +764,7 @@ private static final class MemIndexDocument implements IndexDocument { private static final String[] EMPTY = new String[0]; - private final List fields = new ArrayList(); + private final List fields = new ArrayList(); boolean consumed; MemIndexDocument(@NonNull final String primaryKey) { @@ -728,7 +772,7 @@ private static final class MemIndexDocument implements IndexDocument { fields.add(sourceNameField(primaryKey)); } - public List getFields() { + public List getFields() { return fields; } @@ -742,15 +786,12 @@ public void addPair(String key, String value, boolean searchable, boolean stored if (consumed) { throw new IllegalStateException("Modifying Document after adding it into index."); //NOI18N } - final Field field = new Field (key, value, - stored ? Field.Store.YES : Field.Store.NO, - searchable ? Field.Index.NOT_ANALYZED_NO_NORMS : Field.Index.NO); - fields.add (field); + fields.add (createField(key, value, searchable, stored)); } @Override public String getValue(String key) { - for (Fieldable field : fields) { + for (Field field : fields) { if (field.name().equals(key)) { return field.stringValue(); } @@ -761,7 +802,7 @@ public String getValue(String key) { @Override public String[] getValues(String key) { final List result = new ArrayList(); - for (Fieldable field : fields) { + for (Field field : fields) { if (field.name().equals(key)) { result.add(field.stringValue()); } @@ -769,8 +810,8 @@ public String[] getValues(String key) { return result.toArray(result.isEmpty() ? EMPTY : new String[result.size()]); } - private Fieldable sourceNameField(@NonNull String primaryKey) { - return new Field(FIELD_PRIMARY_KEY, primaryKey, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); + private Field sourceNameField(@NonNull String primaryKey) { + return createField(FIELD_PRIMARY_KEY, primaryKey, true, true); } } // @@ -786,7 +827,7 @@ private static final class ReusableIndexDocument implements IndexDocument { ReusableIndexDocument(@NonNull final MemIndexDocument memDoc) { Parameters.notNull("memDoc", memDoc); //NOI18N - for (Fieldable field : memDoc.getFields()) { + for (Field field : memDoc.getFields()) { doc.add(field); } } @@ -809,15 +850,15 @@ public String[] getValues(String key) { @Override public void addPair(String key, String value, boolean searchable, boolean stored) { - doc.add(new Field ( + doc.add(createField( key, value, - stored ? Field.Store.YES : Field.Store.NO, - searchable ? Field.Index.NOT_ANALYZED_NO_NORMS : Field.Index.NO)); + searchable, + stored)); } void clear() { - doc.getFields().clear(); + doc.clear(); } } // @@ -873,10 +914,10 @@ private boolean addDocument( final MemIndexDocument mdoc = (MemIndexDocument)doc; final int oldDocsPointer = docsPointer; final int oldDataPointer = dataPointer; - for (Fieldable fld : mdoc.getFields()) { + for (Field fld : mdoc.getFields()) { final String fldName = fld.name(); - final boolean stored = fld.isStored(); - final boolean indexed = fld.isIndexed(); + final boolean stored = fld.fieldType().stored(); + final boolean indexed = fld.fieldType().indexOptions() != IndexOptions.NONE; final String fldValue = fld.stringValue(); int index; Integer indexBoxed = fieldNames.get(fldName); diff --git a/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/lucene/LayeredDocumentIndex.java b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/lucene/LayeredDocumentIndex.java index 3b2c4668d702..b3fcd7d14783 100644 --- a/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/lucene/LayeredDocumentIndex.java +++ b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/lucene/LayeredDocumentIndex.java @@ -25,7 +25,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Set; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.search.Query; import org.netbeans.api.annotations.common.CheckForNull; import org.netbeans.api.annotations.common.NonNull; diff --git a/ide/parsing.indexing/src/org/netbeans/modules/parsing/spi/indexing/support/QuerySupport.java b/ide/parsing.indexing/src/org/netbeans/modules/parsing/spi/indexing/support/QuerySupport.java index f98a5872f55d..bdbb0526b89d 100644 --- a/ide/parsing.indexing/src/org/netbeans/modules/parsing/spi/indexing/support/QuerySupport.java +++ b/ide/parsing.indexing/src/org/netbeans/modules/parsing/spi/indexing/support/QuerySupport.java @@ -40,13 +40,13 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; import org.netbeans.api.annotations.common.CheckForNull; import org.netbeans.api.annotations.common.NonNull; import org.netbeans.api.annotations.common.NullAllowed; import org.netbeans.api.java.classpath.ClassPath; import org.netbeans.api.java.classpath.GlobalPathRegistry; import org.netbeans.api.java.queries.BinaryForSourceQuery; -import org.netbeans.api.java.queries.SourceForBinaryQuery; import org.netbeans.api.project.FileOwnerQuery; import org.netbeans.api.project.Project; import org.netbeans.modules.parsing.impl.RunWhenScanFinishedSupport; @@ -524,13 +524,13 @@ public Query file(@NonNull final FileObject file) { @NonNull public Query and(@NonNull final Query...queries) { Parameters.notNull("queries", queries); //NOI18N - final org.apache.lucene.search.BooleanQuery bq = new org.apache.lucene.search.BooleanQuery(); + BooleanQuery.Builder bq = new BooleanQuery.Builder(); for (Query q : queries) { bq.add(new BooleanClause(q.queryImpl, org.apache.lucene.search.BooleanClause.Occur.MUST)); } return new Query( qs, - bq); + bq.build()); } /** @@ -541,13 +541,13 @@ public Query and(@NonNull final Query...queries) { @NonNull public Query or(@NonNull final Query...queries) { Parameters.notNull("queries", queries); //NOI18N - final org.apache.lucene.search.BooleanQuery bq = new org.apache.lucene.search.BooleanQuery(); + BooleanQuery.Builder bq = new BooleanQuery.Builder(); for (Query q : queries) { bq.add(new BooleanClause(q.queryImpl, org.apache.lucene.search.BooleanClause.Occur.SHOULD)); } return new Query( qs, - bq); + bq.build()); } /** diff --git a/ide/parsing.indexing/test/unit/src/org/netbeans/modules/parsing/impl/indexing/DocumentStoreTest.java b/ide/parsing.indexing/test/unit/src/org/netbeans/modules/parsing/impl/indexing/DocumentStoreTest.java index 6179dc1b2815..5cf38c7398ee 100644 --- a/ide/parsing.indexing/test/unit/src/org/netbeans/modules/parsing/impl/indexing/DocumentStoreTest.java +++ b/ide/parsing.indexing/test/unit/src/org/netbeans/modules/parsing/impl/indexing/DocumentStoreTest.java @@ -34,7 +34,7 @@ import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.Logger; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.search.Query; import org.netbeans.api.annotations.common.NonNull; diff --git a/ide/parsing.lucene/manifest.mf b/ide/parsing.lucene/manifest.mf index 892abdb13b12..b780b5970882 100644 --- a/ide/parsing.lucene/manifest.mf +++ b/ide/parsing.lucene/manifest.mf @@ -1,6 +1,6 @@ Manifest-Version: 1.0 AutoUpdate-Show-In-Client: false -OpenIDE-Module: org.netbeans.modules.parsing.lucene/2 +OpenIDE-Module: org.netbeans.modules.parsing.lucene/3 OpenIDE-Module-Implementation-Version: 1 OpenIDE-Module-Localizing-Bundle: org/netbeans/modules/parsing/lucene/Bundle.properties diff --git a/ide/parsing.lucene/nbproject/org-netbeans-modules-parsing-lucene.sig b/ide/parsing.lucene/nbproject/org-netbeans-modules-parsing-lucene.sig index cf2b66764493..779e54933536 100644 --- a/ide/parsing.lucene/nbproject/org-netbeans-modules-parsing-lucene.sig +++ b/ide/parsing.lucene/nbproject/org-netbeans-modules-parsing-lucene.sig @@ -1,5 +1,5 @@ #Signature file v4.1 -#Version 2.66.0 +#Version 2.68.0 CLSS public java.io.IOException cons public init() @@ -21,6 +21,7 @@ intf java.lang.Comparable<{java.lang.Enum%0}> intf java.lang.constant.Constable meth protected final java.lang.Object clone() throws java.lang.CloneNotSupportedException meth protected final void finalize() + anno 0 java.lang.Deprecated(boolean forRemoval=true, java.lang.String since="18") meth public final boolean equals(java.lang.Object) meth public final int compareTo({java.lang.Enum%0}) meth public final int hashCode() @@ -44,7 +45,7 @@ CLSS public java.lang.Object cons public init() meth protected java.lang.Object clone() throws java.lang.CloneNotSupportedException meth protected void finalize() throws java.lang.Throwable - anno 0 java.lang.Deprecated(boolean forRemoval=false, java.lang.String since="9") + anno 0 java.lang.Deprecated(boolean forRemoval=true, java.lang.String since="9") meth public boolean equals(java.lang.Object) meth public final java.lang.Class getClass() meth public final void notify() @@ -173,13 +174,13 @@ innr public abstract interface static Transactional innr public abstract interface static WithTermFrequencies innr public final static !enum Status innr public final static IndexClosedException -meth public abstract !varargs <%0 extends java.lang.Object, %1 extends java.lang.Object> void queryDocTerms(java.util.Map>,org.netbeans.modules.parsing.lucene.support.Convertor,org.netbeans.modules.parsing.lucene.support.Convertor,org.apache.lucene.document.FieldSelector,java.util.concurrent.atomic.AtomicBoolean,org.apache.lucene.search.Query[]) throws java.io.IOException,java.lang.InterruptedException +meth public abstract !varargs <%0 extends java.lang.Object, %1 extends java.lang.Object> void queryDocTerms(java.util.Map>,org.netbeans.modules.parsing.lucene.support.Convertor,org.netbeans.modules.parsing.lucene.support.Convertor,java.util.Set,java.util.concurrent.atomic.AtomicBoolean,org.apache.lucene.search.Query[]) throws java.io.IOException,java.lang.InterruptedException anno 2 org.netbeans.api.annotations.common.NonNull() anno 3 org.netbeans.api.annotations.common.NonNull() anno 4 org.netbeans.api.annotations.common.NullAllowed() anno 5 org.netbeans.api.annotations.common.NullAllowed() anno 6 org.netbeans.api.annotations.common.NonNull() -meth public abstract !varargs <%0 extends java.lang.Object> void query(java.util.Collection,org.netbeans.modules.parsing.lucene.support.Convertor,org.apache.lucene.document.FieldSelector,java.util.concurrent.atomic.AtomicBoolean,org.apache.lucene.search.Query[]) throws java.io.IOException,java.lang.InterruptedException +meth public abstract !varargs <%0 extends java.lang.Object> void query(java.util.Collection,org.netbeans.modules.parsing.lucene.support.Convertor,java.util.Set,java.util.concurrent.atomic.AtomicBoolean,org.apache.lucene.search.Query[]) throws java.io.IOException,java.lang.InterruptedException anno 2 org.netbeans.api.annotations.common.NonNull() anno 3 org.netbeans.api.annotations.common.NullAllowed() anno 4 org.netbeans.api.annotations.common.NullAllowed() @@ -189,11 +190,12 @@ meth public abstract <%0 extends java.lang.Object, %1 extends java.lang.Object> anno 2 org.netbeans.api.annotations.common.NonNull() anno 3 org.netbeans.api.annotations.common.NonNull() anno 4 org.netbeans.api.annotations.common.NonNull() -meth public abstract <%0 extends java.lang.Object> void queryTerms(java.util.Collection,org.apache.lucene.index.Term,org.netbeans.modules.parsing.lucene.support.StoppableConvertor,java.util.concurrent.atomic.AtomicBoolean) throws java.io.IOException,java.lang.InterruptedException +meth public abstract <%0 extends java.lang.Object> void queryTerms(java.util.Collection,java.lang.String,java.lang.String,org.netbeans.modules.parsing.lucene.support.StoppableConvertor,java.util.concurrent.atomic.AtomicBoolean) throws java.io.IOException,java.lang.InterruptedException anno 1 org.netbeans.api.annotations.common.NonNull() - anno 2 org.netbeans.api.annotations.common.NullAllowed() - anno 3 org.netbeans.api.annotations.common.NonNull() - anno 4 org.netbeans.api.annotations.common.NullAllowed() + anno 2 org.netbeans.api.annotations.common.NonNull() + anno 3 org.netbeans.api.annotations.common.NullAllowed() + anno 4 org.netbeans.api.annotations.common.NonNull() + anno 5 org.netbeans.api.annotations.common.NullAllowed() meth public abstract org.netbeans.modules.parsing.lucene.support.Index$Status getStatus(boolean) throws java.io.IOException meth public abstract void clear() throws java.io.IOException meth public abstract void close() throws java.io.IOException @@ -228,16 +230,17 @@ CLSS public abstract interface static org.netbeans.modules.parsing.lucene.suppor outer org.netbeans.modules.parsing.lucene.support.Index innr public final static TermFreq intf org.netbeans.modules.parsing.lucene.support.Index -meth public abstract <%0 extends java.lang.Object> void queryTermFrequencies(java.util.Collection,org.apache.lucene.index.Term,org.netbeans.modules.parsing.lucene.support.StoppableConvertor,java.util.concurrent.atomic.AtomicBoolean) throws java.io.IOException,java.lang.InterruptedException +meth public abstract <%0 extends java.lang.Object> void queryTermFrequencies(java.util.Collection,java.lang.String,java.lang.String,org.netbeans.modules.parsing.lucene.support.StoppableConvertor,java.util.concurrent.atomic.AtomicBoolean) throws java.io.IOException,java.lang.InterruptedException anno 1 org.netbeans.api.annotations.common.NonNull() - anno 2 org.netbeans.api.annotations.common.NullAllowed() - anno 3 org.netbeans.api.annotations.common.NonNull() - anno 4 org.netbeans.api.annotations.common.NullAllowed() + anno 2 org.netbeans.api.annotations.common.NonNull() + anno 3 org.netbeans.api.annotations.common.NullAllowed() + anno 4 org.netbeans.api.annotations.common.NonNull() + anno 5 org.netbeans.api.annotations.common.NullAllowed() CLSS public final static org.netbeans.modules.parsing.lucene.support.Index$WithTermFrequencies$TermFreq outer org.netbeans.modules.parsing.lucene.support.Index$WithTermFrequencies meth public int getFreq() -meth public org.apache.lucene.index.Term getTerm() +meth public org.apache.lucene.util.BytesRef getTerm() anno 0 org.netbeans.api.annotations.common.NonNull() supr java.lang.Object hfds freq,term @@ -336,7 +339,7 @@ CLSS public final org.netbeans.modules.parsing.lucene.support.Queries fld public final static java.lang.String OPTION_CAMEL_CASE_PART = "camelCasePart" fld public final static java.lang.String OPTION_CAMEL_CASE_SEPARATOR = "camelCaseSeparator" innr public final static !enum QueryKind -meth public !varargs static org.apache.lucene.document.FieldSelector createFieldSelector(java.lang.String[]) +meth public !varargs static java.util.Set createFieldSelector(java.lang.String[]) anno 1 org.netbeans.api.annotations.common.NonNull() meth public static boolean isCamelCase(java.lang.String,java.lang.String,java.lang.String) anno 1 org.netbeans.api.annotations.common.NonNull() @@ -375,7 +378,7 @@ meth public static org.apache.lucene.search.Query createTermCollectingQuery(java anno 5 org.netbeans.api.annotations.common.NonNull() supr java.lang.Object hfds CAMEL_CASE_FORMAT,DEFAULT_CAMEL_CASE_PART_CASE_INSENSITIVE,DEFAULT_CAMEL_CASE_PART_CASE_SENSITIVE,DEFAULT_CAMEL_CASE_PATTERN,DEFAULT_CAMEL_CASE_SEPARATOR,cache -hcls AbstractTCFilter,FieldSelectorImpl,HasFieldFilter,PrefixFilter,QueryFactory,RegexpFilter,RegexpTermEnum,StandardQueryFactory,TCBooleanQuery,TCFilter,TCFilteredQuery,TCQueryFactory,TermFilter +hcls QueryFactory,StandardQueryFactory CLSS public final static !enum org.netbeans.modules.parsing.lucene.support.Queries$QueryKind outer org.netbeans.modules.parsing.lucene.support.Queries diff --git a/ide/parsing.lucene/nbproject/project.properties b/ide/parsing.lucene/nbproject/project.properties index 92c77a84fc96..5b076eb3ba06 100644 --- a/ide/parsing.lucene/nbproject/project.properties +++ b/ide/parsing.lucene/nbproject/project.properties @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. is.autoload=true -javac.release=17 +javac.release=21 javadoc.apichanges=${basedir}/apichanges.xml javac.compilerargs=-Xlint -Xlint:-serial -spec.version.base=2.68.0 +spec.version.base=3.0.0 diff --git a/ide/parsing.lucene/nbproject/project.xml b/ide/parsing.lucene/nbproject/project.xml index 988a8457fce7..b42174a9988d 100644 --- a/ide/parsing.lucene/nbproject/project.xml +++ b/ide/parsing.lucene/nbproject/project.xml @@ -39,8 +39,8 @@ - 3 - 3.6 + 10 + 10.0 diff --git a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/AllFieldsSelector.java b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/AllFieldsSelector.java deleted file mode 100644 index 5e44190b995c..000000000000 --- a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/AllFieldsSelector.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.netbeans.modules.parsing.lucene; - -import org.apache.lucene.document.FieldSelector; -import org.apache.lucene.document.FieldSelectorResult; - -/** - * - * @author Tomas Zezula - */ -final class AllFieldsSelector implements FieldSelector { - - static final FieldSelector INSTANCE = new AllFieldsSelector(); - - private AllFieldsSelector(){} - - @Override - public FieldSelectorResult accept(final String fieldName) { - return FieldSelectorResult.LOAD; - } -} diff --git a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/BitSetCollector.java b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/BitSetCollector.java index d6ff84b5d43a..141c3b32fad4 100644 --- a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/BitSetCollector.java +++ b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/BitSetCollector.java @@ -19,16 +19,17 @@ package org.netbeans.modules.parsing.lucene; +import java.io.IOException; import java.util.BitSet; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.Scorer; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.SimpleCollector; /** * * @author Tomas Zezula */ -class BitSetCollector extends Collector { +class BitSetCollector extends SimpleCollector { private int docBase; public final BitSet bits; @@ -40,14 +41,10 @@ class BitSetCollector extends Collector { // ignore scorer @Override - public void setScorer(Scorer scorer) { + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; } - // accept docs out of order (for a BitSet it doesn't matter) - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } @Override public void collect(int doc) { @@ -55,8 +52,9 @@ public void collect(int doc) { } @Override - public void setNextReader(IndexReader reader, int docBase) { - this.docBase = docBase; + protected void doSetNextReader(LeafReaderContext context) throws IOException { + super.doSetNextReader(context); + this.docBase = context.docBase; } } \ No newline at end of file diff --git a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/Convertors.java b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/Convertors.java index 2523fd083353..e0f5aa260ef7 100644 --- a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/Convertors.java +++ b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/Convertors.java @@ -18,17 +18,19 @@ */ package org.netbeans.modules.parsing.lucene; +import java.io.IOException; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; import org.netbeans.api.annotations.common.NonNull; import org.netbeans.modules.parsing.lucene.support.Convertor; import org.netbeans.modules.parsing.lucene.support.Index; import org.netbeans.modules.parsing.lucene.support.IndexDocument; import org.netbeans.modules.parsing.lucene.support.IndexReaderInjection; import org.netbeans.modules.parsing.lucene.support.StoppableConvertor; +import org.openide.util.Exceptions; /** * @@ -52,11 +54,11 @@ static Convertor newSourceNameToQueryConvertor() { return new RemoveConvertor(); } - static StoppableConvertor newTermEnumToTermConvertor(@NonNull StoppableConvertor delegate) { + static StoppableConvertor newTermEnumToTermConvertor(@NonNull StoppableConvertor delegate) { return new TermEnumToTerm<>(delegate); } - static StoppableConvertor newTermEnumToFreqConvertor(@NonNull StoppableConvertor delegate) { + static StoppableConvertor newTermEnumToFreqConvertor(@NonNull StoppableConvertor delegate) { return new TermEnumToFreq<>(delegate); } @@ -82,21 +84,26 @@ public IndexDocumentImpl convert(Document p) { } } - private static class TermEnumToTerm implements StoppableConvertor, IndexReaderInjection { + private static class TermEnumToTerm implements StoppableConvertor, IndexReaderInjection { - private final StoppableConvertor delegate; + private final StoppableConvertor delegate; - TermEnumToTerm(@NonNull final StoppableConvertor convertor) { + TermEnumToTerm(@NonNull final StoppableConvertor convertor) { this.delegate = convertor; } @Override - public T convert(@NonNull final TermEnum terms) throws StoppableConvertor.Stop { - final Term currentTerm = terms.term(); - if (currentTerm == null) { + public T convert(@NonNull final TermsEnum terms) throws StoppableConvertor.Stop { + try { + final BytesRef currentTerm = terms.term(); + if (currentTerm == null) { + return null; + } + return delegate.convert(currentTerm); + } catch (IOException ex) { + Exceptions.printStackTrace(ex); return null; } - return delegate.convert(currentTerm); } @Override @@ -107,7 +114,7 @@ public void setIndexReader(@NonNull IndexReader indexReader) { } } - private static class TermEnumToFreq implements StoppableConvertor, IndexReaderInjection { + private static class TermEnumToFreq implements StoppableConvertor, IndexReaderInjection { private final SupportAccessor accessor = SupportAccessor.getInstance(); private final Index.WithTermFrequencies.TermFreq tf = accessor.newTermFreq(); @@ -118,16 +125,21 @@ private static class TermEnumToFreq implements StoppableConvertor Collection query( Parameters.notNull("query", query); //NOI18N Parameters.notNull("convertor", convertor); //NOI18N final Collection result = new ArrayDeque<>(); - FieldSelector selector = null; + Set selector = null; if (fieldsToLoad != null && fieldsToLoad.length > 0) { final String[] fieldsWithSource = Arrays.copyOf(fieldsToLoad, fieldsToLoad.length+1); fieldsWithSource[fieldsToLoad.length] = IndexDocumentImpl.FIELD_PRIMARY_KEY; diff --git a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/IndexDocumentImpl.java b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/IndexDocumentImpl.java index 47f194d10003..bfe7be199176 100644 --- a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/IndexDocumentImpl.java +++ b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/IndexDocumentImpl.java @@ -21,7 +21,9 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.Fieldable; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; @@ -34,7 +36,34 @@ public final class IndexDocumentImpl implements IndexDocument { static final String FIELD_PRIMARY_KEY = "_sn"; //NOI18N - + + private static final IndexableFieldType STORED_SEARCHABLE; + private static final IndexableFieldType NON_STORED_SEARCHABLE; + private static final IndexableFieldType STORED_NON_SEARCHABLE; + private static final IndexableFieldType NON_STORED_NON_SEARCHABLE; + static { + STORED_SEARCHABLE = new FieldType(); + ((FieldType) STORED_SEARCHABLE).setStored(true); + ((FieldType) STORED_SEARCHABLE).setTokenized(true); + ((FieldType) STORED_SEARCHABLE).setIndexOptions(IndexOptions.DOCS_AND_FREQS); + ((FieldType) STORED_SEARCHABLE).freeze(); + NON_STORED_SEARCHABLE = new FieldType(); + ((FieldType) NON_STORED_SEARCHABLE).setStored(false); + ((FieldType) NON_STORED_SEARCHABLE).setTokenized(true); + ((FieldType) NON_STORED_SEARCHABLE).setIndexOptions(IndexOptions.DOCS_AND_FREQS); + ((FieldType) NON_STORED_SEARCHABLE).freeze(); + STORED_NON_SEARCHABLE = new FieldType(); + ((FieldType) STORED_NON_SEARCHABLE).setStored(true); + ((FieldType) STORED_NON_SEARCHABLE).setTokenized(false); + ((FieldType) STORED_NON_SEARCHABLE).setIndexOptions(IndexOptions.NONE); + ((FieldType) STORED_NON_SEARCHABLE).freeze(); + NON_STORED_NON_SEARCHABLE = new FieldType(); + ((FieldType) NON_STORED_NON_SEARCHABLE).setStored(false); + ((FieldType) NON_STORED_NON_SEARCHABLE).setTokenized(false); + ((FieldType) NON_STORED_NON_SEARCHABLE).setIndexOptions(IndexOptions.NONE); + ((FieldType) NON_STORED_NON_SEARCHABLE).freeze(); + } + final Document doc; public IndexDocumentImpl (final String primaryKey) { @@ -50,13 +79,10 @@ public IndexDocumentImpl (final Document doc) { @Override public void addPair(String key, String value, boolean searchable, boolean stored) { - @SuppressWarnings("deprecation") //NOI18N - final Field field = new Field (key, value, - stored ? Field.Store.YES : Field.Store.NO, - searchable ? Field.Index.NOT_ANALYZED_NO_NORMS : Field.Index.NO); - doc.add (field); + Field field = createField(key, value, searchable, stored); + doc.add(field); } - + @Override public String getValue(String key) { return doc.get(key); @@ -78,8 +104,8 @@ public String toString() { } @SuppressWarnings("deprecation") //NOI18N - private static Fieldable sourceNameField(String relativePath) { - return new Field(FIELD_PRIMARY_KEY, relativePath, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); + private static Field sourceNameField(String relativePath) { + return createField(FIELD_PRIMARY_KEY, relativePath, true, true); } static Query sourceNameQuery(String relativePath) { @@ -90,5 +116,19 @@ static Term sourceNameTerm (final String relativePath) { assert relativePath != null; return new Term (FIELD_PRIMARY_KEY, relativePath); } - + + private static Field createField(String key, String value, boolean searchable, boolean stored) { + IndexableFieldType ift = null; + if(stored && searchable) { + ift = STORED_SEARCHABLE; + } else if ((! stored) && searchable) { + ift = NON_STORED_SEARCHABLE; + } else if (stored && (! searchable)) { + ift = STORED_NON_SEARCHABLE; + } else if ((! stored) && (! searchable)) { + ift = NON_STORED_NON_SEARCHABLE; + } + assert ift != null; + return new Field(key, value, ift); + } } diff --git a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/LuceneIndex.java b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/LuceneIndex.java index c45d3ccc0674..83313d59f0de 100644 --- a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/LuceneIndex.java +++ b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/LuceneIndex.java @@ -28,15 +28,16 @@ import java.lang.ref.SoftReference; import java.net.URI; import java.nio.channels.ClosedByInterruptException; -import java.nio.file.Path; import java.util.ArrayList; import java.util.BitSet; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.ListIterator; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -50,17 +51,18 @@ import java.util.logging.Logger; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldSelector; import org.apache.lucene.index.*; import org.apache.lucene.search.*; +import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.LockFactory; import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.store.NIOFSDirectory; -import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.store.SimpleFSDirectory; -import org.apache.lucene.util.Version; +import org.apache.lucene.util.BytesRef; import org.netbeans.api.annotations.common.CheckForNull; import org.netbeans.api.annotations.common.NonNull; import org.netbeans.api.annotations.common.NullAllowed; @@ -131,7 +133,7 @@ private LuceneIndex (final File refCacheRoot, final Analyzer analyzer) throws IO public void query ( final @NonNull Collection result, final @NonNull Convertor convertor, - @NullAllowed FieldSelector selector, + @NullAllowed Set selector, final @NullAllowed AtomicBoolean cancel, final @NonNull Query... queries ) throws IOException, InterruptedException { @@ -139,9 +141,6 @@ public void query ( Parameters.notNull("convertor", convertor); //NOI18N Parameters.notNull("result", result); //NOI18N - if (selector == null) { - selector = AllFieldsSelector.INSTANCE; - } IndexReader in = null; try { in = dirCache.acquireReader(); @@ -149,15 +148,14 @@ public void query ( LOGGER.log(Level.FINE, "{0} is invalid!", this); return; } + IndexSearcher searcher = new IndexSearcher(in); final BitSet bs = new BitSet(in.maxDoc()); final Collector c = new BitSetCollector(bs); - try (IndexSearcher searcher = new IndexSearcher(in)) { - for (Query q : queries) { - if (cancel != null && cancel.get()) { - throw new InterruptedException (); - } - searcher.search(q, c); + for (Query q : queries) { + if (cancel != null && cancel.get()) { + throw new InterruptedException (); } + searcher.search(q, c); } changeIndexReader(convertor, in); try { @@ -165,7 +163,7 @@ public void query ( if (cancel != null && cancel.get()) { throw new InterruptedException (); } - final Document doc = in.document(docNum, selector); + final Document doc = in.storedFields().document(docNum, selector); final T value = convertor.convert(doc); if (value != null) { result.add (value); @@ -188,28 +186,39 @@ private static void changeIndexReader(Object convertor, IndexReader in) { @Override public void queryTerms( final @NonNull Collection result, - final @NullAllowed Term seekTo, - final @NonNull StoppableConvertor filter, + final @NonNull String field, + final @NullAllowed String seekTo, + final @NonNull StoppableConvertor filter, final @NullAllowed AtomicBoolean cancel) throws IOException, InterruptedException { - queryTermsImpl(result, seekTo, Convertors.newTermEnumToTermConvertor(filter), cancel); + queryTermsImpl(result, field, seekTo, Convertors.newTermEnumToTermConvertor(filter), cancel); } @Override public void queryTermFrequencies( final @NonNull Collection result, - final @NullAllowed Term seekTo, + final @NonNull String field, + final @NullAllowed String seekTo, final @NonNull StoppableConvertor filter, final @NullAllowed AtomicBoolean cancel) throws IOException, InterruptedException { - queryTermsImpl(result, seekTo, Convertors.newTermEnumToFreqConvertor(filter), cancel); + queryTermsImpl(result, field, seekTo, Convertors.newTermEnumToFreqConvertor(filter), cancel); } //where private void queryTermsImpl( final @NonNull Collection result, - final @NullAllowed Term seekTo, - final @NonNull StoppableConvertor adapter, + final @NonNull String field, + @NullAllowed String startValue, + final @NonNull StoppableConvertor adapter, final @NullAllowed AtomicBoolean cancel) throws IOException, InterruptedException { - + + BytesRef startBytesRef; + + if (startValue == null) { + startBytesRef = new BytesRef(""); + } else { + startBytesRef = new BytesRef(startValue); + } + IndexReader in = null; try { in = dirCache.acquireReader(); @@ -218,23 +227,26 @@ private void queryTermsImpl( return; } - try (TermEnum terms = seekTo == null ? in.terms() : in.terms(seekTo)) { - changeIndexReader(adapter, in); - try { - do { - if (cancel != null && cancel.get()) { - throw new InterruptedException (); - } - final T vote = adapter.convert(terms); - if (vote != null) { - result.add(vote); - } - } while (terms.next()); - } catch (StoppableConvertor.Stop stop) { - //Stop iteration of TermEnum finally { - } finally { - changeIndexReader(adapter, null); + changeIndexReader(adapter, in); + try { + for(LeafReaderContext lrc: in.leaves()) { + TermsEnum te = lrc.reader().terms(field).iterator(); + if (te.seekCeil(startBytesRef) != TermsEnum.SeekStatus.END) { + do { + if (cancel != null && cancel.get()) { + throw new InterruptedException(); + } + final T vote = adapter.convert(te); + if (vote != null) { + result.add(vote); + } + } while (te.next() != null); + } } + } catch (StoppableConvertor.Stop stop) { + //Stop iteration of TermEnum + } finally { + changeIndexReader(adapter, null); } } finally { dirCache.releaseReader(in); @@ -245,8 +257,8 @@ private void queryTermsImpl( public void queryDocTerms( final @NonNull Map> result, final @NonNull Convertor convertor, - final @NonNull Convertor termConvertor, - @NullAllowed FieldSelector selector, + final @NonNull Convertor termConvertor, + @NullAllowed Set selector, final @NullAllowed AtomicBoolean cancel, final @NonNull Query... queries) throws IOException, InterruptedException { Parameters.notNull("queries", queries); //NOI18N @@ -254,9 +266,7 @@ public void queryDocTerms( Parameters.notNull("convertor", convertor); //NOI18N Parameters.notNull("termConvertor", termConvertor); //NOI18N Parameters.notNull("result", result); //NOI18N - if (selector == null) { - selector = AllFieldsSelector.INSTANCE; - } + IndexReader in = null; try { in = dirCache.acquireReader(); @@ -264,70 +274,102 @@ public void queryDocTerms( LOGGER.log(Level.FINE, "{0} is invalid!", this); return; } - final BitSet bs = new BitSet(in.maxDoc()); - final Collector c = new BitSetCollector(bs); - final IndexSearcher searcher = new IndexSearcher(in); - final TermCollector termCollector = new TermCollector(c); - try { + for (LeafReaderContext lrc : in.leaves()) { + final LeafReader lr = lrc.reader(); + Map> docTermMap = new HashMap<>(); for (Query q : queries) { if (cancel != null && cancel.get()) { - throw new InterruptedException (); + throw new InterruptedException(); } - if (q instanceof TermCollector.TermCollecting termCollecting) { - termCollecting.attach(termCollector); - } else { - throw new IllegalArgumentException ( + switch (q) { + case TermQuery tq -> { + Terms terms = lr.terms(tq.getTerm().field()); + if (terms != null) { + TermsEnum te = terms.iterator(); + if (te.seekExact(tq.getTerm().bytes())) { + PostingsEnum pe = te.postings(null); + for (int doc = pe.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = pe.nextDoc()) { + docTermMap.computeIfAbsent(doc, s -> new HashSet<>()) + .add(tq.getTerm().bytes()); + } + } + } + } + case PrefixQuery pq -> { + Terms terms = lr.terms(pq.getField()); + if (terms != null) { + TermsEnum te = pq.getCompiled().getTermsEnum(terms); + for (BytesRef termValue = te.next(); termValue != null; termValue = te.next()) { + PostingsEnum pe = te.postings(null); + for (int doc = pe.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = pe.nextDoc()) { + docTermMap.computeIfAbsent(doc, s -> new HashSet<>()) + .add(termValue); + } + } + } + } + case RegexpFilter rf -> { + Terms terms = lr.terms(rf.getField()); + if (terms != null) { + TermsEnum te = rf.getTermsEnum(terms); + for (BytesRef termValue = te.next(); termValue != null; termValue = te.next()) { + PostingsEnum pe = te.postings(null); + for (int doc = pe.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = pe.nextDoc()) { + docTermMap.computeIfAbsent(doc, s -> new HashSet<>()) + .add(termValue); + } + } + } + } + default -> throw new IllegalArgumentException( "Query: %s does not implement TermCollecting".formatted(q.getClass().getName())); //NOI18N } - searcher.search(q, termCollector); } - } finally { - searcher.close(); - } - - boolean logged = false; - changeIndexReader(convertor, in); - try { - changeIndexReader(termConvertor, in); + + boolean logged = false; + changeIndexReader(convertor, in); try { - for (int docNum = bs.nextSetBit(0); docNum >= 0; docNum = bs.nextSetBit(docNum+1)) { - if (cancel != null && cancel.get()) { - throw new InterruptedException (); - } - final Document doc = in.document(docNum, selector); - final T value = convertor.convert(doc); - if (value != null) { - final Set terms = termCollector.get(docNum); - if (terms != null) { - result.put (value, convertTerms(termConvertor, terms)); - } else { - if (!logged) { - LOGGER.log(Level.WARNING, "Index info [maxDoc: {0} numDoc: {1} docs: {2}]", - new Object[] { - in.maxDoc(), - in.numDocs(), - termCollector.docs() - }); - logged = true; + changeIndexReader(termConvertor, in); + try { + for (Entry> docNum: docTermMap.entrySet()) { + if (cancel != null && cancel.get()) { + throw new InterruptedException(); + } + final Document doc = lr.storedFields().document(docNum.getKey(), selector); + final T value = convertor.convert(doc); + if (value != null) { + final Set terms = docNum.getValue(); + if (terms != null) { + result.put(value, convertTerms(termConvertor, terms)); + } else { + if (!logged) { + LOGGER.log(Level.WARNING, "Index info [maxDoc: {0} numDoc: {1} docs: {2}]", + new Object[] { + in.maxDoc(), + in.numDocs(), + docNum.getValue() + }); + logged = true; + } + LOGGER.log(Level.WARNING, "No terms found for doc: {0}", docNum); } - LOGGER.log(Level.WARNING, "No terms found for doc: {0}", docNum); } } + } finally { + changeIndexReader(termConvertor, null); } } finally { - changeIndexReader(termConvertor, null); + changeIndexReader(convertor, null); } - } finally { - changeIndexReader(convertor, null); } } finally { dirCache.releaseReader(in); } } - private static Set convertTerms(final Convertor convertor, final Set terms) { + private static Set convertTerms(final Convertor convertor, final Set terms) { final Set result = new HashSet(terms.size()); - for (Term term : terms) { + for (BytesRef term : terms) { result.add(convertor.convert(term)); } return result; @@ -391,7 +433,7 @@ private void _doStore( for (S td : toDelete) { out.deleteDocuments(queryConvertor.convert(td)); } - } + } if (data.isEmpty()) { return; } @@ -401,11 +443,10 @@ private void _doStore( if (lmListener.isLowMemory()) { activeOut = out; } else { - memDir = new RAMDirectory (); + memDir = new ByteBuffersDirectory (); activeOut = new IndexWriter ( memDir, new IndexWriterConfig( - Version.LUCENE_35, dirCache.getAnalyzer())); } for (Iterator it = fastRemoveIterable(data).iterator(); it.hasNext();) { @@ -416,11 +457,10 @@ private void _doStore( if (memDir != null && lmListener.isLowMemory()) { activeOut.close(); out.addIndexes(memDir); - memDir = new RAMDirectory (); + memDir = new ByteBuffersDirectory (); activeOut = new IndexWriter ( memDir, new IndexWriterConfig( - Version.LUCENE_35, dirCache.getAnalyzer())); } } @@ -574,7 +614,7 @@ private static final class DirCache implements Evictable { private final StoreCloseSynchronizer storeCloseSynchronizer; private volatile FSDirectory fsDir; //@GuardedBy("this") - private RAMDirectory memDir; + private ByteBuffersDirectory memDir; private CleanReference ref; private IndexReader reader; private volatile boolean closed; @@ -638,16 +678,13 @@ private synchronized void doClear() throws IOException { try { fsDir.deleteFile(file); } catch (IOException e) { - //Some temporary files - if (fsDir.fileExists(file)) { - dirty = true; - } + dirty = true; } } } if (dirty) { //Try to delete dirty files and log what's wrong - final File cacheDir = fsDir.getDirectory(); + final File cacheDir = fsDir.getDirectory().toFile(); final File[] children = cacheDir.listFiles(); if (children != null) { for (final File child : children) { @@ -678,23 +715,7 @@ void close(IndexWriter writer) throws IOException { } finally { LOGGER.log(Level.FINE, "TX writer cleared for {0}", this); indexWriterRef.release(); - try { - if (!success) { - if (IndexWriter.isLocked(fsDir)) { - IndexWriter.unlock(fsDir); - } - } - } catch (IOException ioe) { - LOGGER.log( - Level.WARNING, - "Cannot unlock index {0} while recovering, {1}.", //NOI18N - new Object[] { - folder.getAbsolutePath(), - ioe.getMessage() - }); - } finally { - refreshReader(); - } + refreshReader(); } } @@ -759,7 +780,7 @@ synchronized void doClose ( boolean exists() { try { - return IndexReader.indexExists(this.fsDir); + return DirectoryReader.indexExists((Directory) this.fsDir); } catch (IOException e) { return false; } catch (RuntimeException e) { @@ -851,7 +872,7 @@ IndexWriter acquireWriter () throws IOException { try { try { IndexWriter writer = indexWriterRef.acquire(() -> { - IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_36, analyzer); + IndexWriterConfig iwc = new IndexWriterConfig(analyzer); //Linux: The posix::fsync(int) is very slow on Linux ext3, //minimize number of files sync is done on. //http://netbeans.org/bugzilla/show_bug.cgi?id=208224 @@ -919,16 +940,22 @@ private synchronized IndexReader getReader () throws IOException { try { Directory source; if (cachePolicy.hasMemCache() && fitsIntoMem(fsDir)) { - memDir = new RAMDirectory(fsDir); + memDir = new ByteBuffersDirectory(); + for(String file: fsDir.listAll()) { + try(IndexOutput io = memDir.createOutput(file, IOContext.READONCE); + IndexInput ii = fsDir.openInput(file, IOContext.READONCE)) { + io.copyBytes(ii, ii.length()); + } + } if (cachePolicy == CachePolicy.DYNAMIC) { - ref = new CleanReference (new RAMDirectory[] {this.memDir}); + ref = new CleanReference (this.memDir); } source = memDir; } else { source = fsDir; } assert source != null; - this.reader = IndexReader.open(source); + this.reader = DirectoryReader.open(source); } catch (final FileNotFoundException | ClosedByInterruptException | InterruptedIOException e) { //Either the index dir does not exist or the thread is interrupted //pass - returns null @@ -957,7 +984,7 @@ void refreshReader() throws IOException { try { synchronized (this) { if (reader != null) { - IndexReader newReader = IndexReader.openIfChanged(reader); + IndexReader newReader = DirectoryReader.openIfChanged((DirectoryReader) reader); if (newReader != null) { reader.close(); reader = newReader; @@ -1031,13 +1058,12 @@ private static FSDirectory createFSDirectory ( assert lockFactory != null; final String dirType = System.getProperty(PROP_DIR_TYPE); if (dirType == null) { - return FSDirectory.open(indexFolder, lockFactory); + return FSDirectory.open(indexFolder.toPath(), lockFactory); } return switch (dirType) { - case DIR_TYPE_MMAP -> new MMapDirectory(indexFolder, lockFactory); - case DIR_TYPE_NIO -> new NIOFSDirectory(indexFolder, lockFactory); - case DIR_TYPE_IO -> new SimpleFSDirectory(indexFolder, lockFactory); - default -> FSDirectory.open(indexFolder, lockFactory); + case DIR_TYPE_MMAP -> new MMapDirectory(indexFolder.toPath(), lockFactory); + case DIR_TYPE_NIO -> new NIOFSDirectory(indexFolder.toPath(), lockFactory); + default -> FSDirectory.open(indexFolder.toPath(), lockFactory); }; } @@ -1159,21 +1185,28 @@ private static boolean assertsEnabled() { } } - private final class CleanReference extends SoftReference implements Runnable { + private final class CleanReference extends SoftReference implements Runnable { @SuppressWarnings("VolatileArrayField") - private volatile Directory[] hardRef; //clearHRef may be called by more concurrently (read lock). + private volatile Directory hardRef; //clearHRef may be called by more concurrently (read lock). private final AtomicLong size = new AtomicLong(); //clearHRef may be called by more concurrently (read lock). - private CleanReference(final RAMDirectory[] dir) { + private CleanReference(final ByteBuffersDirectory dir) { super (dir, BaseUtilities.activeReferenceQueue()); final IndexCacheFactory.RAMController c = IndexCacheFactory.getDefault().getRAMController(); final boolean doHardRef = !c.isFull(); if (doHardRef) { - this.hardRef = dir; - long _size = dir[0].sizeInBytes(); - size.set(_size); - c.acquire(_size); + try { + this.hardRef = dir; + long _size = 0; + for(String file: dir.listAll()) { + _size += dir.fileLength(file); + } + size.set(_size); + c.acquire(_size); + } catch (IOException ex) { + Exceptions.printStackTrace(ex); + } } LOGGER.log(Level.FINEST, "Caching index: {0} cache policy: {1}", //NOI18N new Object[]{ @@ -1227,7 +1260,13 @@ public FlushIndexWriter( */ void callFlush(boolean triggerMerges, boolean flushDeletes) throws IOException { // flushStores ignored in Lucene 3.5 - super.flush(triggerMerges, true, flushDeletes); + super.flush(); + if(flushDeletes) { + super.forceMergeDeletes(true); + } + if(triggerMerges) { + super.maybeMerge(); + } } } @@ -1337,10 +1376,5 @@ public boolean isDirectory() { public boolean isFile() { return !isDirectory(); } - - @Override - public Path toPath() { - throw IPE; - } } } diff --git a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/MemoryIndex.java b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/MemoryIndex.java index 20f237d5c1d6..08e2d85420ba 100644 --- a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/MemoryIndex.java +++ b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/MemoryIndex.java @@ -18,26 +18,31 @@ */ package org.netbeans.modules.parsing.lucene; -import java.io.FileNotFoundException; import java.io.IOException; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.LimitTokenCountAnalyzer; +import org.apache.lucene.analysis.miscellaneous.LimitTokenCountAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.Collector; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.util.Version; +import org.apache.lucene.util.BytesRef; import org.netbeans.api.annotations.common.CheckForNull; import org.netbeans.api.annotations.common.NonNull; import org.netbeans.api.annotations.common.NullAllowed; @@ -46,6 +51,7 @@ import org.netbeans.modules.parsing.lucene.support.StoppableConvertor; import org.openide.util.Parameters; + /** * * @author Tomas Zezula @@ -55,7 +61,7 @@ public class MemoryIndex implements Index { private final Analyzer analyzer; private final ReentrantReadWriteLock lock; //@GuardedBy("this") - private RAMDirectory dir; + private ByteBuffersDirectory dir; //@GuardedBy("this") private IndexReader cachedReader; @@ -81,38 +87,33 @@ public Status getStatus(boolean tryOpen) throws IOException { public void query( @NonNull Collection result, @NonNull Convertor convertor, - @NullAllowed FieldSelector selector, + @NullAllowed Set selector, @NullAllowed AtomicBoolean cancel, @NonNull Query... queries) throws IOException, InterruptedException { Parameters.notNull("queries", queries); //NOI18N Parameters.notNull("convertor", convertor); //NOI18N Parameters.notNull("result", result); //NOI18N - - if (selector == null) { - selector = AllFieldsSelector.INSTANCE; - } - + lock.readLock().lock(); try { IndexReader in = getReader(); if (in == null) { return; } + IndexSearcher searcher = new IndexSearcher(in); BitSet bs = new BitSet(in.maxDoc()); Collector c = new BitSetCollector(bs); - try (IndexSearcher searcher = new IndexSearcher(in)) { - for (Query q : queries) { - if (cancel != null && cancel.get()) { - throw new InterruptedException (); - } - searcher.search(q, c); + for (Query q : queries) { + if (cancel != null && cancel.get()) { + throw new InterruptedException(); } - } + searcher.search(q, c); + } for (int docNum = bs.nextSetBit(0); docNum >= 0; docNum = bs.nextSetBit(docNum+1)) { if (cancel != null && cancel.get()) { throw new InterruptedException (); } - Document doc = in.document(docNum, selector); + Document doc = in.storedFields().document(docNum, selector); T value = convertor.convert(doc); if (value != null) { result.add (value); @@ -127,54 +128,83 @@ public void query( public void queryDocTerms( @NonNull Map> result, @NonNull Convertor convertor, - @NonNull Convertor termConvertor, - @NullAllowed FieldSelector selector, + @NonNull Convertor termConvertor, + @NullAllowed Set selector, @NullAllowed AtomicBoolean cancel, @NonNull Query... queries) throws IOException, InterruptedException { Parameters.notNull("result", result); //NOI18N Parameters.notNull("convertor", convertor); //NOI18N Parameters.notNull("termConvertor", termConvertor); //NOI18N Parameters.notNull("queries", queries); //NOI18N - - - if (selector == null) { - selector = AllFieldsSelector.INSTANCE; - } lock.readLock().lock(); - try { - IndexReader in = getReader(); + try (IndexReader in = getReader()) { if (in == null) { return; } - BitSet bs = new BitSet(in.maxDoc()); - Collector c = new BitSetCollector(bs); - TermCollector termCollector = new TermCollector(c); - try (IndexSearcher searcher = new IndexSearcher(in)) { + for (LeafReaderContext lrc : in.leaves()) { + LeafReader lr = lrc.reader(); + Map> docTermMap = new HashMap<>(); for (Query q : queries) { if (cancel != null && cancel.get()) { - throw new InterruptedException (); + throw new InterruptedException(); } - if (q instanceof TermCollector.TermCollecting termCollecting) { - termCollecting.attach(termCollector); - } else { - throw new IllegalArgumentException ( + switch (q) { + case TermQuery tq -> { + Terms terms = lr.terms(tq.getTerm().field()); + if (terms != null) { + TermsEnum te = terms.iterator(); + if (te.seekExact(tq.getTerm().bytes())) { + PostingsEnum pe = te.postings(null); + for (int doc = pe.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = pe.nextDoc()) { + docTermMap.computeIfAbsent(doc, s -> new HashSet<>()) + .add(tq.getTerm().bytes()); + } + } + } + } + case PrefixQuery pq -> { + Terms terms = lr.terms(pq.getField()); + if (terms != null) { + TermsEnum te = pq.getCompiled().getTermsEnum(terms); + for (BytesRef termValue = te.next(); termValue != null; termValue = te.next()) { + PostingsEnum pe = te.postings(null); + for (int doc = pe.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = pe.nextDoc()) { + docTermMap.computeIfAbsent(doc, s -> new HashSet<>()) + .add(termValue); + } + } + } + } + case RegexpFilter rf -> { + Terms terms = lr.terms(rf.getField()); + if (terms != null) { + TermsEnum te = rf.getTermsEnum(terms); + for (BytesRef termValue = te.next(); termValue != null; termValue = te.next()) { + PostingsEnum pe = te.postings(null); + for (int doc = pe.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = pe.nextDoc()) { + docTermMap.computeIfAbsent(doc, s -> new HashSet<>()) + .add(termValue); + } + } + } + } + default -> throw new IllegalArgumentException( "Query: %s does not implement TermCollecting".formatted(q.getClass().getName())); //NOI18N } - searcher.search(q, termCollector); } - } - for (int docNum = bs.nextSetBit(0); docNum >= 0; docNum = bs.nextSetBit(docNum+1)) { - if (cancel != null && cancel.get()) { - throw new InterruptedException (); - } - Document doc = in.document(docNum, selector); - T value = convertor.convert(doc); - if (value != null) { - final Set terms = termCollector.get(docNum); - if (terms != null) { - result.put (value, convertTerms(termConvertor, terms)); + for (Map.Entry> docNum: docTermMap.entrySet()) { + if (cancel != null && cancel.get()) { + throw new InterruptedException(); + } + Document doc = lr.storedFields().document(docNum.getKey(), selector); + T value = convertor.convert(doc); + if (value != null) { + final Set terms = docNum.getValue(); + if (terms != null) { + result.put(value, convertTerms(termConvertor, terms)); + } } } } @@ -186,34 +216,43 @@ public void queryDocTerms( @Override public void queryTerms( @NonNull Collection result, - @NullAllowed Term start, - @NonNull StoppableConvertor filter, + @NonNull String field, + @NullAllowed String startValue, + @NonNull StoppableConvertor filter, @NullAllowed AtomicBoolean cancel) throws IOException, InterruptedException { Parameters.notNull("result", result); //NOI18N Parameters.notNull("filter", filter); //NOI18N - + + BytesRef startBytesRef; + + if (startValue == null) { + startBytesRef = new BytesRef(""); + } else { + startBytesRef = new BytesRef(startValue); + } + lock.readLock().lock(); try { IndexReader in = getReader(); if (in == null) { return; } - try (TermEnum terms = start == null ? in.terms() : in.terms(start)) { - do { - if (cancel != null && cancel.get()) { - throw new InterruptedException (); - } - Term currentTerm = terms.term(); - if (currentTerm != null) { - T vote = filter.convert(currentTerm); + for(LeafReaderContext lrc: in.leaves()) { + TermsEnum te = lrc.reader().terms(field).iterator(); + if (te.seekCeil(startBytesRef) != TermsEnum.SeekStatus.END) { + do { + if (cancel != null && cancel.get()) { + throw new InterruptedException(); + } + T vote = filter.convert(te.term()); if (vote != null) { result.add(vote); } - } - } while (terms.next()); - } catch (StoppableConvertor.Stop stop) { - //Stop iteration of TermEnum + } while (te.next() != null); + } } + } catch (StoppableConvertor.Stop stop) { + //Stop iteration of TermEnum } finally { lock.readLock().unlock(); } @@ -278,11 +317,7 @@ public void close() throws IOException { @CheckForNull private synchronized IndexReader getReader() throws IOException { if (cachedReader == null) { - try { - cachedReader = IndexReader.open(getDirectory()); - } catch (FileNotFoundException fnf) { - //pass - returns null - } + cachedReader = DirectoryReader.open(getDirectory()); } return cachedReader; } @@ -290,7 +325,7 @@ private synchronized IndexReader getReader() throws IOException { private synchronized void refreshReader() throws IOException { assert lock.isWriteLockedByCurrentThread(); if (cachedReader != null) { - IndexReader newReader = IndexReader.openIfChanged(cachedReader); + IndexReader newReader = DirectoryReader.openIfChanged((DirectoryReader) cachedReader); if (newReader != null) { cachedReader.close(); cachedReader = newReader; @@ -299,23 +334,20 @@ private synchronized void refreshReader() throws IOException { } private synchronized IndexWriter getWriter() throws IOException { - IndexWriterConfig conf = new IndexWriterConfig( - Version.LUCENE_36, - new LimitTokenCountAnalyzer(analyzer, IndexWriter.DEFAULT_MAX_FIELD_LENGTH) - ); + IndexWriterConfig conf = new IndexWriterConfig(new LimitTokenCountAnalyzer(analyzer, 10_000)); return new IndexWriter (getDirectory(), conf); } private synchronized Directory getDirectory() { if (dir == null) { - dir = new RAMDirectory(); + dir = new ByteBuffersDirectory(); } return dir; } - private static Set convertTerms(final Convertor convertor, final Set terms) { + private static Set convertTerms(final Convertor convertor, final Set terms) { Set result = new HashSet<>(terms.size()); - for (Term term : terms) { + for (BytesRef term : terms) { result.add(convertor.convert(term)); } return result; diff --git a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/RecordOwnerLockFactory.java b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/RecordOwnerLockFactory.java index e0ead6f3660b..99c01c1b18fc 100644 --- a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/RecordOwnerLockFactory.java +++ b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/RecordOwnerLockFactory.java @@ -20,12 +20,11 @@ import java.io.File; import java.io.IOException; -import java.util.ArrayDeque; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; +import java.util.HashSet; import java.util.Map; -import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; import org.apache.lucene.store.LockObtainFailedException; @@ -40,53 +39,32 @@ //@ThreadSafe class RecordOwnerLockFactory extends LockFactory { - private final Map locks = - new HashMap<>(); + private static final Set lockHolder = ConcurrentHashMap.newKeySet(); RecordOwnerLockFactory() throws IOException { super(); } @Override - public Lock makeLock(String lockName) { - synchronized (locks) { - return locks.computeIfAbsent(lockName, k -> new RecordOwnerLock()); - } - } - - @Override - public void clearLock(String lockName) throws IOException { - synchronized (locks) { - final RecordOwnerLock lock = locks.remove(lockName); - if (lock != null) { - lock.release(); - } + public Lock obtainLock(Directory directory, String lock) throws IOException { + DirectoryLockPair dlp = new DirectoryLockPair(directory, lock); + if(! lockHolder.add(dlp)) { + throw new LockObtainFailedException("Pair already locked: " + dlp); } + return new RecordOwnerLock(dlp); } boolean hasLocks() { - synchronized (locks) { - boolean res = false; - for (RecordOwnerLock lock : locks.values()) { - res|=lock.isLocked(); - } - return res; - } + return ! lockHolder.isEmpty(); } - Collection forceClearLocks() { - synchronized (locks) { - final Queue locked = new ArrayDeque<>(); - for (Iterator it = locks.values().iterator(); - it.hasNext();) { - RecordOwnerLock lock = it.next(); - if (lock.isLocked()) { - it.remove(); - locked.offer(lock); - } - } - return locked; + Set forceClearLocks() { + Set oldLocked; + synchronized (lockHolder) { + oldLocked = new HashSet<>(lockHolder); + lockHolder.clear(); } + return oldLocked; } @Override @@ -94,17 +72,7 @@ public String toString() { final StringBuilder sb = new StringBuilder(); sb.append(getClass().getSimpleName()); sb.append('['); //NOI18N - synchronized (locks) { - boolean first = true; - for (Map.Entry e : locks.entrySet()) { - if (!first) { - sb.append('\n'); //NOI18N - } else { - first = false; - } - sb.append("name: ").append(e.getKey()).append("->").append(e.getValue()); //NOI18N - } - } + sb.append(lockHolder.toString()); sb.append("]\n"); //NOI18N return sb.toString(); } @@ -112,66 +80,45 @@ public String toString() { private final class RecordOwnerLock extends Lock { - //@GuardedBy("locks") - private Thread owner; - //@GuardedBy("locks") - private Exception caller; + private final Thread owner; + private final Exception caller; + private final DirectoryLockPair lockedPair; - private RecordOwnerLock() { + private RecordOwnerLock(DirectoryLockPair lockedPair) { + this.lockedPair = lockedPair; + this.owner = Thread.currentThread(); + this.caller = new Exception(); } @Override - public boolean obtain() { - synchronized (RecordOwnerLockFactory.this.locks) { - if (this.owner == null) { - this.owner = Thread.currentThread(); - this.caller = new Exception(); - return true; - } else { - return false; - } - } - } - - @Override - public boolean obtain(long lockWaitTimeout) throws LockObtainFailedException, IOException { - try { - return super.obtain(lockWaitTimeout); - } catch (LockObtainFailedException e) { - throw annotateException( - e, - (File) null, - Thread.getAllStackTraces(), - RecordOwnerLockFactory.this); - } + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append(this.getClass().getSimpleName()); + sb.append("owned by:["); //NOI18N + sb.append(owner); + sb.append('(').append(owner == null ? -1 : owner.getId()).append(')'); //NOI18N + sb.append("created from:\n"); + stackTrace(caller == null ? new StackTraceElement[0] : caller.getStackTrace(), sb); + return sb.toString(); } @Override - public void release() { - synchronized (RecordOwnerLockFactory.this.locks) { - this.owner = null; - this.caller = null; - } + public void close() throws IOException { + lockHolder.remove(this.lockedPair); } @Override - public boolean isLocked() { - synchronized (RecordOwnerLockFactory.this.locks) { - return this.owner != null; - } - } - - @Override - public String toString() { - synchronized (RecordOwnerLockFactory.this.locks) { + public void ensureValid() throws IOException { + if (!lockHolder.contains(lockedPair)) { final StringBuilder sb = new StringBuilder(); - sb.append(this.getClass().getSimpleName()); + sb.append(this.lockedPair); sb.append("owned by:["); //NOI18N sb.append(owner); sb.append('(').append(owner == null ? -1 : owner.getId()).append(')'); //NOI18N sb.append("created from:\n"); stackTrace(caller == null ? new StackTraceElement[0] : caller.getStackTrace(), sb); - return sb.toString(); + sb.append(" not valid anymore"); + throw new IOException(sb.toString()); } } } @@ -230,4 +177,25 @@ private static void stackTrace( sb.append('\t').append(se).append('\n'); //NOI18N } } + + record DirectoryLockPair(Directory directory, String lock) { + + private static final Set RECURSION_PROTECTION = ConcurrentHashMap.newKeySet(); + + @Override + public String toString() { + // The directory also outputs the lock in toString(), we need to + // protect unlimited recursion + if(RECURSION_PROTECTION.contains(Thread.currentThread())) { + return "<>"; + } + try { + RECURSION_PROTECTION.add(Thread.currentThread()); + return "DirectoryLockPair{" + directory + ", " + lock + "}"; + } finally { + RECURSION_PROTECTION.remove(Thread.currentThread()); + } + } + + } } diff --git a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/RegexpFilter.java b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/RegexpFilter.java new file mode 100644 index 000000000000..4c2004e09227 --- /dev/null +++ b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/RegexpFilter.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.netbeans.modules.parsing.lucene; + +import java.io.IOException; +import java.util.BitSet; +import java.util.List; +import java.util.regex.Pattern; +import org.apache.lucene.index.FilteredTermsEnum; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.util.AttributeSource; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Automata; +import org.apache.lucene.util.automaton.CompiledAutomaton; +import org.apache.lucene.util.automaton.Operations; + + +public class RegexpFilter extends MultiTermQuery { + + private static final BitSet SPECIAL_CHARS = new BitSet(126); + static { + final char[] specials = new char[]{'{', '}', '[', ']', '(', ')', '\\', '.', '*', '?', '+'}; //NOI18N + for (char c : specials) { + SPECIAL_CHARS.set(c); + } + } + private static final BitSet QUANTIFIER_CHARS = new BitSet(126); + static { + final char[] specials = new char[]{'{', '*', '?'}; //NOI18N + for (char c : specials) { + QUANTIFIER_CHARS.set(c); + } + } + private final CompiledAutomaton startPrefix; + private final Pattern pattern; + + public RegexpFilter(final String fieldName, final String regexp, final boolean caseSensitive) { + super(fieldName, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE); + this.pattern = caseSensitive ? Pattern.compile(regexp) : Pattern.compile(regexp, Pattern.CASE_INSENSITIVE); + this.startPrefix = new CompiledAutomaton(Operations.concatenate(List.of(Automata.makeString(getStartText(regexp)), Automata.makeAnyString()))); + } + + @Override + protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException { + return new FilteredTermsEnum(startPrefix.getTermsEnum(terms), false) { + @Override + protected FilteredTermsEnum.AcceptStatus accept(BytesRef term) throws IOException { + if (pattern.matcher(term.utf8ToString()).matches()) { + return FilteredTermsEnum.AcceptStatus.YES; + } + return FilteredTermsEnum.AcceptStatus.NO; + } + + @Override + protected BytesRef nextSeekTerm(BytesRef currentTerm) throws IOException { + if(currentTerm == null || currentTerm == tenum.term()) { + return tenum.next(); + } else { + throw new IllegalStateException("TermsEnum does not enable seeking"); + } + } + }; + } + + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(field)) { + visitor.visitLeaf(this); + } + } + + static String getStartText(final String regexp) { + final StringBuilder startBuilder = new StringBuilder(); + boolean quoted = false; + for (int i = 0; i < regexp.length(); i++) { + char c = regexp.charAt(i); + if ((!quoted) && i < (regexp.length() - 1)) { + char lookAhead = regexp.charAt(i + 1); + if (QUANTIFIER_CHARS.get(lookAhead)) { + break; + } + } + if (c == '\\' && (i + 1) < regexp.length()) { + //NOI18N + char cn = regexp.charAt(i + 1); + if (!quoted && cn == 'Q') { + //NOI18N + quoted = true; + i++; + continue; + } else if (cn == 'E') { + //NOI18N + quoted = false; + i++; + continue; + } + } else if (!quoted && (c == '^' || c == '$')) { + //NOI18N + continue; + } + if (!quoted && SPECIAL_CHARS.get(c)) { + break; + } + startBuilder.append(c); + } + return startBuilder.toString(); + } + + @Override + public String toString(String field) { + StringBuilder buffer = new StringBuilder(); + if (this.getField() != null) { + buffer.append(this.getField()); + buffer.append(":"); + } + buffer.append('/'); + buffer.append(pattern.toString()); + buffer.append('/'); + return buffer.toString(); + } + +} diff --git a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/SupportAccessor.java b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/SupportAccessor.java index 1a6fe0801ba6..d510f3a44a19 100644 --- a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/SupportAccessor.java +++ b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/SupportAccessor.java @@ -18,7 +18,7 @@ */ package org.netbeans.modules.parsing.lucene; -import org.apache.lucene.index.Term; +import org.apache.lucene.util.BytesRef; import org.netbeans.api.annotations.common.NonNull; import org.netbeans.modules.parsing.lucene.support.Index; import org.netbeans.modules.parsing.lucene.support.IndexManager; @@ -59,7 +59,7 @@ public static void setInstance(@NonNull SupportAccessor _instance) { @NonNull public abstract Index.WithTermFrequencies.TermFreq setTermFreq( @NonNull Index.WithTermFrequencies.TermFreq into, - @NonNull Term term, + @NonNull BytesRef term, int freq); } diff --git a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/TermCollector.java b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/TermCollector.java index e009f3815b2c..2491c72e8199 100644 --- a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/TermCollector.java +++ b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/TermCollector.java @@ -25,10 +25,11 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.SimpleCollector; import org.netbeans.api.annotations.common.NonNull; /** @@ -39,43 +40,35 @@ * * @author Tomas Zezula */ -public final class TermCollector extends Collector { - private final Collector delegate; +public final class TermCollector extends SimpleCollector { + private final SimpleCollector delegate; private final Map> doc2Terms; - private int indexOffset; - - TermCollector(Collector collector) { + + TermCollector(SimpleCollector collector) { this.delegate = collector; doc2Terms = new HashMap<>(); } - + public void add (final int docId, final @NonNull Term term) { - final int realId = docId + indexOffset; - doc2Terms.computeIfAbsent(realId, k -> new HashSet<>()) + doc2Terms.computeIfAbsent(docId, k -> new HashSet<>()) .add(term); } - + Set get(final int docId) { return doc2Terms.get(docId); } - + Set docs() { return Collections.unmodifiableSet(doc2Terms.keySet()); - } - - public static interface TermCollecting { - void attach (TermCollector collector); } @Override - public void setScorer(Scorer scorer) throws IOException { - delegate.setScorer(scorer); + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; } - @Override - public void setNextReader(IndexReader reader, int i) throws IOException { - delegate.setNextReader(reader, i); - indexOffset = i; + public static interface TermCollecting { + void attach (TermCollector collector); } @Override @@ -84,8 +77,13 @@ public void collect(int i) throws IOException { } @Override - public boolean acceptsDocsOutOfOrder() { - return delegate.acceptsDocsOutOfOrder(); + public void setScorer(Scorable scorer) throws IOException { + super.setScorer(scorer); + } + + @Override + protected void doSetNextReader(LeafReaderContext context) throws IOException { + super.doSetNextReader(context); } } diff --git a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/Index.java b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/Index.java index eb6b590d15fe..04d1ff4419a0 100644 --- a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/Index.java +++ b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/Index.java @@ -25,9 +25,9 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldSelector; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; import org.netbeans.api.annotations.common.NonNull; import org.netbeans.api.annotations.common.NullAllowed; @@ -93,7 +93,7 @@ public static final class IndexClosedException extends IOException { * @throws IOException in case of IO problem * @throws InterruptedException when query was canceled */ - void query (Collection result, @NonNull Convertor convertor, @NullAllowed FieldSelector selector, @NullAllowed AtomicBoolean cancel, @NonNull Query... queries) throws IOException, InterruptedException; + void query (Collection result, @NonNull Convertor convertor, @NullAllowed Set selector, @NullAllowed AtomicBoolean cancel, @NonNull Query... queries) throws IOException, InterruptedException; /** * Queries the {@link Index} by given queries. In addition to documents it also collects the terms which matched the queries. @@ -107,11 +107,18 @@ public static final class IndexClosedException extends IOException { * @throws IOException in case of IO problem * @throws InterruptedException when query was canceled */ - void queryDocTerms(Map> result, @NonNull Convertor convertor, @NonNull Convertor termConvertor,@NullAllowed FieldSelector selector, @NullAllowed AtomicBoolean cancel, @NonNull Query... queries) throws IOException, InterruptedException; + void queryDocTerms( + Map> result, + @NonNull Convertor convertor, + @NonNull Convertor termConvertor, + @NullAllowed Set selector, + @NullAllowed AtomicBoolean cancel, + @NonNull Query... queries) throws IOException, InterruptedException; /** * Queries the {@link Index}'s b-tree for terms starting by the start term and accepted by the filter. * @param result the {@link Collection} to store results into + * @param field the field to analyze * @param start the first term to start the b-tree iteration with, if null the iteration start on the first term. * @param filter converting the terms into the user objects which are added into the result or null to skeep them. * The filter can stop the iteration by throwing the {@link StoppableConvertor.Stop}. @@ -120,7 +127,7 @@ public static final class IndexClosedException extends IOException { * @throws IOException in case of IO problem * @throws InterruptedException when query was canceled */ - void queryTerms(@NonNull Collection result, @NullAllowed Term start, @NonNull StoppableConvertor filter, @NullAllowed AtomicBoolean cancel) throws IOException, InterruptedException; + void queryTerms(@NonNull Collection result, @NonNull String field, @NullAllowed String start, @NonNull StoppableConvertor filter, @NullAllowed AtomicBoolean cancel) throws IOException, InterruptedException; /** * Updates the {@link Index} by adding the toAdd objects and deleting toDelete objects. @@ -199,11 +206,11 @@ public interface WithTermFrequencies extends Index { */ public final class TermFreq { private int freq; - private Term term; + private BytesRef term; TermFreq() {} - void setTerm(@NonNull final Term term) { + void setTerm(@NonNull final BytesRef term) { this.term = term; } @@ -216,7 +223,7 @@ void setFreq(final int freq) { * @return the term. */ @NonNull - public Term getTerm() { + public BytesRef getTerm() { return term; } @@ -233,6 +240,7 @@ public int getFreq() { /** * Queries the {@link Index}'s b-tree for terms and frequencies estimate starting by the start term and accepted by the filter. * @param result the {@link Collection} to store results into + * @param field the field to analyze * @param start the first term to start the b-tree iteration with, if null the iteration start on the first term. * @param filter converting the terms into the user objects which are added into the result or null to skeep them. * The filter can stop the iteration by throwing the {@link StoppableConvertor.Stop}. @@ -243,7 +251,8 @@ public int getFreq() { */ void queryTermFrequencies( @NonNull Collection result, - @NullAllowed Term start, + @NonNull String field, + @NullAllowed String start, @NonNull StoppableConvertor filter, @NullAllowed AtomicBoolean cancel) throws IOException, InterruptedException; } diff --git a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/IndexManager.java b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/IndexManager.java index a751c06d1dd5..1c5bcf4ebc34 100644 --- a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/IndexManager.java +++ b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/IndexManager.java @@ -30,8 +30,8 @@ import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.KeywordAnalyzer; -import org.apache.lucene.index.Term; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.util.BytesRef; import org.netbeans.api.annotations.common.NonNull; import org.netbeans.modules.masterfs.providers.ProvidedExtensions; import org.netbeans.modules.parsing.lucene.DocumentIndexImpl; @@ -489,7 +489,7 @@ public TermFreq newTermFreq() { } @Override - public Index.WithTermFrequencies.TermFreq setTermFreq(@NonNull TermFreq into, @NonNull Term term, int freq) { + public Index.WithTermFrequencies.TermFreq setTermFreq(@NonNull TermFreq into, @NonNull BytesRef term, int freq) { into.setTerm(term); into.setFreq(freq); return into; diff --git a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/Queries.java b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/Queries.java index 9a0c24b20c2e..bbe0622fd1e8 100644 --- a/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/Queries.java +++ b/ide/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/Queries.java @@ -19,34 +19,24 @@ package org.netbeans.modules.parsing.lucene.support; -import java.io.IOException; -import java.util.BitSet; +import org.netbeans.modules.parsing.lucene.RegexpFilter; import java.util.Collections; +import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.lucene.document.FieldSelector; -import org.apache.lucene.document.FieldSelectorResult; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.DocIdSet; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.FilteredQuery; -import org.apache.lucene.search.FilteredTermEnum; +import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.PrefixQuery; -import org.apache.lucene.search.PrefixTermEnum; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.util.OpenBitSet; import org.netbeans.api.annotations.common.CheckForNull; import org.netbeans.api.annotations.common.NonNull; import org.netbeans.api.annotations.common.NullAllowed; -import org.netbeans.modules.parsing.lucene.TermCollector; import org.openide.util.Pair; import org.openide.util.Parameters; @@ -215,7 +205,7 @@ public static Query createTermCollectingQuery( Parameters.notNull("value", value); //NOI18N Parameters.notNull("kind", kind); //NOI18N Parameters.notNull("options", options); //NOI18N - return createQueryImpl(fieldName, caseInsensitiveFieldName, value, kind, new TCQueryFactory(), options); + return createQueryImpl(fieldName, caseInsensitiveFieldName, value, kind, new StandardQueryFactory(), options); } /** @@ -223,8 +213,8 @@ public static Query createTermCollectingQuery( * @param fieldsToLoad the fields to be loaded into the document. * @return the created FieldSelector */ - public static FieldSelector createFieldSelector(final @NonNull String... fieldsToLoad) { - return new FieldSelectorImpl(fieldsToLoad); + public static Set createFieldSelector(final @NonNull String... fieldsToLoad) { + return new HashSet<>(List.of(fieldsToLoad)); } /** @@ -381,292 +371,11 @@ private static int findNextSeparator( return -1; } - private abstract static class TCFilter extends Filter { - public abstract void attach (TermCollector collector); - } - - private abstract static class AbstractTCFilter extends TCFilter { - - private TermCollector termCollector; - - @Override - public final DocIdSet getDocIdSet(IndexReader reader) throws IOException { - final FilteredTermEnum enumerator = getTermEnum(reader); - // if current term in enum is null, the enum is empty -> shortcut - if (enumerator.term() == null) { - return DocIdSet.EMPTY_DOCIDSET; - } - try { - // else fill into a OpenBitSet - final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc()); - final int[] docs = new int[32]; - final int[] freqs = new int[32]; - try (TermDocs termDocs = reader.termDocs()) { - do { - final Term term = enumerator.term(); - if (term == null) { - break; - } - termDocs.seek(term); - while (true) { - final int count = termDocs.read(docs, freqs); - if (count != 0) { - for (int i = 0; i < count; i++) { - bitSet.set(docs[i]); - if (termCollector != null) { - termCollector.add(docs[i], term); - } - } - } else { - break; - } - } - } while (enumerator.next()); - } - return bitSet; - } finally { - enumerator.close(); - } - } - - @Override - public final void attach(final TermCollector tc) { - this.termCollector = tc; - } - - protected abstract FilteredTermEnum getTermEnum(IndexReader reader) throws IOException; - - } - - private static class RegexpTermEnum extends FilteredTermEnum { - - private final String fieldName; - private final String startPrefix; - private final Pattern pattern; - private boolean endEnum; - - public RegexpTermEnum( - final IndexReader in, - final String fieldName, - final Pattern pattern, - final String startPrefix) throws IOException { - final Term term = new Term(fieldName,startPrefix); - this.fieldName = term.field(); - this.pattern = pattern; - this.startPrefix = startPrefix; - setEnum(in.terms(term)); - } - - @Override - protected boolean termCompare(Term term) { - if (fieldName == term.field()) { - String searchText = term.text(); - if (searchText.startsWith(startPrefix)) { - return pattern.matcher(term.text()).matches(); - } - } - endEnum = true; - return false; - } - - @Override - public float difference() { - return 1.0f; - } - - @Override - protected boolean endEnum() { - return endEnum; - } - } - - static class RegexpFilter extends AbstractTCFilter { - private static final BitSet SPECIAL_CHARS = new BitSet(126); - static { - final char[] specials = new char[] {'{','}','[',']','(',')','\\','.','*','?', '+'}; //NOI18N - for (char c : specials) { - SPECIAL_CHARS.set(c); - } - } - private static final BitSet QUANTIFIER_CHARS = new BitSet(126); - static { - final char[] specials = new char[] {'{','*','?'}; //NOI18N - for (char c : specials) { - QUANTIFIER_CHARS.set(c); - } - } - - private final String fieldName; - private final String startPrefix; - private final Pattern pattern; - - public RegexpFilter(final String fieldName, final String regexp, final boolean caseSensitive) { - this.fieldName = fieldName; - this.pattern = caseSensitive ? Pattern.compile(regexp) : Pattern.compile(regexp, Pattern.CASE_INSENSITIVE); - this.startPrefix = getStartText(regexp); - } - - @Override - protected FilteredTermEnum getTermEnum(final @NonNull IndexReader reader) throws IOException { - return new RegexpTermEnum(reader, fieldName, pattern, startPrefix); - } - - private static String getStartText(final String regexp) { - final StringBuilder startBuilder = new StringBuilder (); - boolean quoted = false; - for (int i = 0; i < regexp.length(); i++) { - char c = regexp.charAt(i); - if ((!quoted) && i < (regexp.length() - 1)) { - char lookAhead = regexp.charAt(i + 1); - if (QUANTIFIER_CHARS.get(lookAhead)) { - break; - } - } - if (c == '\\' && (i+1) < regexp.length()) { //NOI18N - char cn = regexp.charAt(i+1); - if (!quoted && cn == 'Q') { //NOI18N - quoted = true; - i++; - continue; - } else if (cn == 'E') { //NOI18N - quoted = false; - i++; - continue; - } - } else if (!quoted && (c == '^' || c == '$')) { //NOI18N - continue; - } - if (!quoted && SPECIAL_CHARS.get(c)) { - break; - } - startBuilder.append(c); - } - return startBuilder.toString(); - } - } - - private static class PrefixFilter extends AbstractTCFilter { - - protected final Term term; - - public PrefixFilter(final @NonNull String fieldName, final @NonNull String prefix) { - this.term = new Term(fieldName, prefix); - } - - @Override - protected FilteredTermEnum getTermEnum(final @NonNull IndexReader reader) throws IOException { - return new PrefixTermEnum(reader, term); - } - } - - private static class TermFilter extends PrefixFilter { - - public TermFilter (final String fieldName, final String value) { - super(fieldName, value); - } - - @Override - protected FilteredTermEnum getTermEnum(IndexReader reader) throws IOException { - return new PrefixTermEnum(reader, term) { - - private boolean endEnum; - - @Override - protected boolean termCompare(Term term) { - if (TermFilter.this.term.field() == term.field() && TermFilter.this.term.text().equals(term.text())) { - return true; - } - endEnum = true; - return false; - } - - @Override - protected boolean endEnum() { - return endEnum; - } - - }; - } - } - - private static class HasFieldFilter extends PrefixFilter { - - public HasFieldFilter (final String fieldName) { - super (fieldName, ""); //NOI18N - } - - @Override - protected FilteredTermEnum getTermEnum(IndexReader reader) throws IOException { - return new PrefixTermEnum(reader, term) { - - private boolean endEnum; - - @Override - protected boolean termCompare(Term term) { - if (HasFieldFilter.this.term.field() == term.field()) { - return true; - } - endEnum = true; - return false; - } - - @Override - protected boolean endEnum() { - return endEnum; - } - }; - } - } - - private static class TCFilteredQuery extends FilteredQuery implements TermCollector.TermCollecting { - private TCFilteredQuery(final Query query, final TCFilter filter) { - super (query, filter); - } - - @Override - public void attach(TermCollector collector) { - ((TCFilter)getFilter()).attach(collector); - } - } - - private static class TCBooleanQuery extends BooleanQuery implements TermCollector.TermCollecting { - - private TermCollector collector; - - @Override - public void attach(TermCollector collector) { - this.collector = collector; - if (this.collector != null) { - attach(this, this.collector); - } - } - - @Override - public Query rewrite(IndexReader reader) throws IOException { - final Query result = super.rewrite(reader); - if (this.collector != null) { - attach(this,this.collector); - } - return result; - } - - private static void attach (final BooleanQuery query, final TermCollector collector) { - for (BooleanClause clause : query.getClauses()) { - final Query q = clause.getQuery(); - if (!(q instanceof TermCollector.TermCollecting)) { - throw new IllegalArgumentException(); - } - ((TermCollector.TermCollecting)q).attach(collector); - } - } - } - private static interface QueryFactory { Query createTermQuery(@NonNull String name, @NonNull String value); Query createPrefixQuery(@NonNull String name, @NonNull String value); Query createRegExpQuery(@NonNull String name, @NonNull String value, boolean caseSensitive); Query createAllDocsQuery(@NonNull String name); - BooleanQuery createBooleanQuery(); } private static class StandardQueryFactory implements QueryFactory { @@ -678,14 +387,13 @@ public Query createTermQuery(final @NonNull String name, final @NonNull String v @Override public Query createPrefixQuery(final @NonNull String name, final @NonNull String value) { - final PrefixQuery pq = new PrefixQuery(new Term(name, value)); - pq.setRewriteMethod(PrefixQuery.CONSTANT_SCORE_FILTER_REWRITE); + final PrefixQuery pq = new PrefixQuery(new Term(name, value), PrefixQuery.CONSTANT_SCORE_REWRITE); return pq; } @Override public Query createRegExpQuery(final @NonNull String name, final @NonNull String value, final boolean caseSensitive) { - return new FilteredQuery(new MatchAllDocsQuery(), new RegexpFilter(name, value, caseSensitive)); + return new RegexpFilter(name, value, caseSensitive); } @Override @@ -693,64 +401,10 @@ public Query createAllDocsQuery(final @NonNull String name) { if (name.isEmpty()) { return new MatchAllDocsQuery(); } else { - return new FilteredQuery(new MatchAllDocsQuery(), new HasFieldFilter(name)); - } - } - - @Override - public BooleanQuery createBooleanQuery() { - return new BooleanQuery(); - } - } - - private static class TCQueryFactory implements QueryFactory { - - @Override - public Query createTermQuery(final @NonNull String name, final @NonNull String value) { - return new TCFilteredQuery(new MatchAllDocsQuery(), new TermFilter(name,value)); - } - - @Override - public Query createPrefixQuery(final @NonNull String name, final @NonNull String value) { - return new TCFilteredQuery(new MatchAllDocsQuery(), new PrefixFilter(name, value)); - } - - @Override - public Query createRegExpQuery(final @NonNull String name, final @NonNull String value, final boolean caseSensitive) { - return new TCFilteredQuery(new MatchAllDocsQuery(), new RegexpFilter(name, value, caseSensitive)); - } - - @Override - public Query createAllDocsQuery(final @NonNull String name) { - throw new IllegalArgumentException (); - } - - @Override - public BooleanQuery createBooleanQuery() { - return new TCBooleanQuery(); - } - } - - private static class FieldSelectorImpl implements FieldSelector { - - private final Term[] terms; - - FieldSelectorImpl(String... fieldNames) { - terms = new Term[fieldNames.length]; - for (int i=0; i< fieldNames.length; i++) { - terms[i] = new Term (fieldNames[i],""); //NOI18N + return new FieldExistsQuery(name); } } - @Override - public FieldSelectorResult accept(String fieldName) { - for (Term t : terms) { - if (fieldName == t.field()) { - return FieldSelectorResult.LOAD; - } - } - return FieldSelectorResult.NO_LOAD; - } } private Queries() {} diff --git a/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/AsyncCloseTest.java b/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/AsyncCloseTest.java index be5b1d8385ef..d99a89f627b6 100644 --- a/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/AsyncCloseTest.java +++ b/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/AsyncCloseTest.java @@ -26,9 +26,11 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; @@ -181,7 +183,11 @@ public Document convert(String p) { } final Document doc = new Document(); - doc.add(new Field(FLD_KEY, p, Field.Store.YES, Field.Index.ANALYZED_NO_NORMS)); //NOI18N + FieldType ft = new FieldType(); + ft.setStored(true); + ft.setTokenized(true); + ft.setIndexOptions(IndexOptions.DOCS); + doc.add(new Field(FLD_KEY, p, ft)); return doc; } } diff --git a/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/IndexTransactionTest.java b/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/IndexTransactionTest.java index d1e0c578eab9..d3981b593742 100644 --- a/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/IndexTransactionTest.java +++ b/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/IndexTransactionTest.java @@ -26,9 +26,11 @@ import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; @@ -143,7 +145,7 @@ public void testAddDocumentAndQuery() throws Exception { final SimpleDocumentIndexCache cache = new SimpleDocumentIndexCache(); DocumentIndex docIndex = IndexManager.createDocumentIndex(index, cache); IndexDocument doc = IndexManager.createDocument("manicka"); - doc.addPair("name", "manicka", true, false); + doc.addPair("name", "manicka", true, true); doc.addPair("age", "10", true, true); Collection results = @@ -183,7 +185,7 @@ public void testRemoveDocumentAndQuery() throws Exception { final SimpleDocumentIndexCache cache = new SimpleDocumentIndexCache(); DocumentIndex docIndex = IndexManager.createDocumentIndex(index, cache); IndexDocument doc = IndexManager.createDocument("manicka"); - doc.addPair("name", "manicka", true, false); + doc.addPair("name", "manicka", true, true); doc.addPair("age", "10", true, true); Collection results; @@ -253,7 +255,11 @@ public StrToDocConvertor(final String name) { @Override public Document convert(final String p) { final Document doc = new Document(); - doc.add(new Field(name, p, Field.Store.YES, Field.Index.ANALYZED)); + FieldType ft = new FieldType(); + ft.setStored(true); + ft.setTokenized(true); + ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + doc.add(new Field(name, p, ft)); return doc; } } diff --git a/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/LuceneIndexTest.java b/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/LuceneIndexTest.java index dfdbe9ea54df..914c415753b6 100644 --- a/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/LuceneIndexTest.java +++ b/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/LuceneIndexTest.java @@ -30,9 +30,10 @@ import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.Logger; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; @@ -247,7 +248,7 @@ private void createLock(final LuceneIndex index) throws NoSuchFieldException, Il final java.lang.reflect.Field directory = o.getClass().getDeclaredField("fsDir"); //NOI18N directory.setAccessible(true); Directory dir = (Directory) directory.get(o); - dir.makeLock("test").obtain(); //NOI18N + dir.obtainLock("test"); //NOI18N } @@ -276,7 +277,10 @@ public StrToDocConvertor(final String name) { @Override public Document convert(final String p) { final Document doc = new Document(); - doc.add(new Field(name, p, Field.Store.YES, Field.Index.ANALYZED)); + FieldType ft = new FieldType(); + ft.setStored(true); + ft.setTokenized(true); + doc.add(new Field(name, p, ft)); return doc; } } diff --git a/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/NativeFSLockFactoryTest.java b/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/NativeFSLockFactoryTest.java index cf45c9229611..2d6b87516551 100644 --- a/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/NativeFSLockFactoryTest.java +++ b/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/NativeFSLockFactoryTest.java @@ -27,9 +27,10 @@ import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.Logger; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; import org.apache.lucene.search.Query; import org.netbeans.api.annotations.common.NonNull; import org.netbeans.junit.NbTestCase; @@ -61,6 +62,9 @@ public void testLockFreedDuringStoreError() throws Exception { final LuceneIndex index = LuceneIndex.create(indexFolder, new KeywordAnalyzer()); final Collection dataSet = generateDataSet(1000); final Logger log = Logger.getLogger(LuceneIndex.class.getName()); + final FieldType ft = new FieldType(); + ft.setStored(true); + ft.setTokenized(true); final TestHandler handler = new TestHandler( new Runnable() { @Override @@ -88,12 +92,11 @@ public Document convert(Integer p) { doc.add(new Field( "val", //NOI18N Integer.toString(p), - Field.Store.YES, - Field.Index.ANALYZED_NO_NORMS)); + ft)); return doc; } }, - new Convertor() { + new Convertor<>() { @Override public Query convert(String p) { throw new UnsupportedOperationException(); @@ -119,8 +122,7 @@ public Document convert(Integer p) { doc.add(new Field( "val", //NOI18N Integer.toString(p), - Field.Store.YES, - Field.Index.ANALYZED_NO_NORMS)); + ft)); return doc; } }, diff --git a/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/RawIndexPerfTest.java b/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/RawIndexPerfTest.java index cf7772e50973..98209581e74c 100644 --- a/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/RawIndexPerfTest.java +++ b/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/RawIndexPerfTest.java @@ -24,9 +24,12 @@ import java.util.List; import java.util.Random; import java.util.concurrent.atomic.AtomicLong; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; @@ -127,13 +130,33 @@ private static void deleteContent(File folder) { } } + private static final IndexableFieldType STORED_NOT_ANALYZED_NO_NORMS; + private static final IndexableFieldType NOT_STORED_NOT_ANALYZED_NO_NORMS; + private static final IndexableFieldType STORED_NOT_ANALYZED; + static { + STORED_NOT_ANALYZED_NO_NORMS = new FieldType(); + ((FieldType) STORED_NOT_ANALYZED_NO_NORMS).setStored(true); + ((FieldType) STORED_NOT_ANALYZED_NO_NORMS).setIndexOptions(IndexOptions.DOCS_AND_FREQS); + ((FieldType) STORED_NOT_ANALYZED_NO_NORMS).setTokenized(false); + ((FieldType) STORED_NOT_ANALYZED_NO_NORMS).setOmitNorms(true); + NOT_STORED_NOT_ANALYZED_NO_NORMS = new FieldType(); + ((FieldType) NOT_STORED_NOT_ANALYZED_NO_NORMS).setStored(false); + ((FieldType) NOT_STORED_NOT_ANALYZED_NO_NORMS).setIndexOptions(IndexOptions.DOCS_AND_FREQS); + ((FieldType) NOT_STORED_NOT_ANALYZED_NO_NORMS).setTokenized(false); + ((FieldType) NOT_STORED_NOT_ANALYZED_NO_NORMS).setOmitNorms(true); + STORED_NOT_ANALYZED = new FieldType(); + ((FieldType) STORED_NOT_ANALYZED).setStored(true); + ((FieldType) STORED_NOT_ANALYZED).setIndexOptions(IndexOptions.NONE); + ((FieldType) STORED_NOT_ANALYZED).setTokenized(false); + } + private static final class LongToDoc implements Convertor { @Override public Document convert(Long p) { Document doc = new Document(); - doc.add(new Field("dec", Long.toString(p), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); - doc.add(new Field("hex", Long.toHexString(p), Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS)); - doc.add(new Field("bin", Long.toBinaryString(p), Field.Store.YES, Field.Index.NO)); + doc.add(new Field("dec", Long.toString(p), STORED_NOT_ANALYZED_NO_NORMS)); + doc.add(new Field("hex", Long.toHexString(p), NOT_STORED_NOT_ANALYZED_NO_NORMS)); + doc.add(new Field("bin", Long.toBinaryString(p), STORED_NOT_ANALYZED)); return doc; } } @@ -148,7 +171,7 @@ public Query convert(Long p) { private static final class DocToLong implements Convertor { @Override public Long convert(Document p) { - return Long.valueOf(p.getFieldable("dec").stringValue()); + return Long.valueOf(p.get("dec")); } } } diff --git a/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/RecordOwnerLockFactoryTest.java b/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/RecordOwnerLockFactoryTest.java index 7885f59ac60f..06a875c07134 100644 --- a/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/RecordOwnerLockFactoryTest.java +++ b/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/RecordOwnerLockFactoryTest.java @@ -22,6 +22,8 @@ import java.io.IOException; import java.util.Collection; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; import org.netbeans.junit.NbTestCase; @@ -43,72 +45,60 @@ protected void setUp() throws Exception { this.lockFactory = new RecordOwnerLockFactory(); } - public void testLock() throws IOException { - final Lock lock = lockFactory.makeLock("test"); //NOI18N - assertFalse(lock.isLocked()); - lock.obtain(); - assertTrue(lock.isLocked()); - lock.release(); - assertFalse(lock.isLocked()); + public void testClosedLockIsDetected() throws IOException { + Directory dir = new ByteBuffersDirectory(); + Lock lock = lockFactory.obtainLock(dir, "test"); + assertNotNull(lock); + lock.close(); + boolean ioExceptionRecorded = false; + try { + lock.ensureValid(); + assertFalse("Lock was not invalidated", true); + } catch (IOException ex) { + ioExceptionRecorded = true; + } + assertTrue("IOException was expected but not thrown", ioExceptionRecorded); } - public void testLockInstances() throws IOException { - final Lock lock1 = lockFactory.makeLock("test"); //NOI18N - final Lock lock2 = lockFactory.makeLock("test"); //NOI18N - assertFalse(lock1.isLocked()); - assertFalse(lock2.isLocked()); - lock1.obtain(); - assertTrue(lock1.isLocked()); - assertTrue(lock2.isLocked()); - lock2.release(); - assertFalse(lock1.isLocked()); - assertFalse(lock2.isLocked()); + public void testLockDetectsDuplicate() throws IOException { + Directory dir = new ByteBuffersDirectory(); + Lock lock1 = lockFactory.obtainLock(dir, "test"); //NOI18N + assertNotNull(lock1); + lock1.ensureValid(); + boolean ioExceptionRecorded = false; + try { + Lock lock2 = lockFactory.obtainLock(dir, "test"); //NOI18N + } catch (IOException ex) { + ioExceptionRecorded = true; + } + assertTrue("IOException was expected but not thrown", ioExceptionRecorded); + lock1.close(); } public void testClearLock() throws IOException { - Lock lock = lockFactory.makeLock("test"); //NOI18N - assertFalse(lock.isLocked()); - lock.obtain(); - assertTrue(lock.isLocked()); - lockFactory.clearLock("test"); //NOI18N - assertFalse(lock.isLocked()); - } - - public void testHasLocks() throws IOException { - assertFalse(lockFactory.hasLocks()); - final Lock lock1 = lockFactory.makeLock("test1"); //NOI18N - final Lock lock2 = lockFactory.makeLock("test2"); //NOI18N - final Lock lock3 = lockFactory.makeLock("test3"); //NOI18N - final Lock lock4 = lockFactory.makeLock("test4"); //NOI18N - assertFalse(lockFactory.hasLocks()); - assertTrue(lock2.obtain()); - assertTrue(lockFactory.hasLocks()); - lock2.release(); - assertFalse(lockFactory.hasLocks()); - assertTrue(lock3.obtain()); - assertTrue(lockFactory.hasLocks()); - assertTrue(lock4.obtain()); - assertTrue(lockFactory.hasLocks()); - lockFactory.clearLock("test3"); //NOI18N - assertTrue(lockFactory.hasLocks()); - assertTrue(lock2.obtain()); - lockFactory.clearLock("test4"); //NOI18N - assertTrue(lockFactory.hasLocks()); - lock2.release(); - assertFalse(lockFactory.hasLocks()); + Directory dir = new ByteBuffersDirectory(); + Lock lock = lockFactory.obtainLock(dir, "test"); + assertNotNull(lock); + lock.ensureValid(); + lock.close(); + assertTrue(lockFactory.forceClearLocks().isEmpty()); } public void testForceClearLocks() throws IOException { - final Lock lock1 = lockFactory.makeLock("test1"); //NOI18N - assertTrue(lock1.obtain()); + Directory dir = new ByteBuffersDirectory(); + Lock lock = lockFactory.obtainLock(dir, "test1"); + lock.ensureValid(); assertTrue(lockFactory.hasLocks()); - lockFactory.makeLock("test2"); //NOI18N - assertTrue(lockFactory.makeLock("test3").obtain()); //NOI18N - lockFactory.makeLock("test3").release(); //NOI18N + Lock lock2 = lockFactory.obtainLock(dir, "test2"); + lock2.ensureValid(); + Lock lock3 = lockFactory.obtainLock(dir, "test3"); + lock3.ensureValid(); + lock2.close(); + lock3.close(); assertTrue(lockFactory.hasLocks()); - Collection locks = lockFactory.forceClearLocks(); + Collection locks = lockFactory.forceClearLocks(); assertEquals(1, locks.size()); - assertEquals(lock1, locks.iterator().next()); + assertEquals(new RecordOwnerLockFactory.DirectoryLockPair(dir, "test1"), locks.iterator().next()); assertFalse(lockFactory.hasLocks()); } diff --git a/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/support/QueriesTest.java b/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/RegexpFilterTest.java similarity index 78% rename from ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/support/QueriesTest.java rename to ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/RegexpFilterTest.java index dbabcef16afc..4bfa4cd045e8 100644 --- a/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/support/QueriesTest.java +++ b/ide/parsing.lucene/test/unit/src/org/netbeans/modules/parsing/lucene/RegexpFilterTest.java @@ -16,18 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -package org.netbeans.modules.parsing.lucene.support; +package org.netbeans.modules.parsing.lucene; -import java.lang.reflect.Field; import java.util.regex.Pattern; -import org.netbeans.modules.parsing.lucene.support.Queries.RegexpFilter; -import static org.junit.Assert.assertEquals; import org.junit.Test; +import static org.junit.Assert.*; -public class QueriesTest { +public class RegexpFilterTest { - public QueriesTest() { + public RegexpFilterTest() { } @Test @@ -46,10 +44,7 @@ public void testRegexpStartText() throws Exception { } private String getRegexpFilterPrefixForPattern(String pattern) throws Exception { - RegexpFilter rf = new RegexpFilter("dummy", pattern, true); - Field startPrefix = RegexpFilter.class.getDeclaredField("startPrefix"); - startPrefix.setAccessible(true); - return (String) startPrefix.get(rf); + return RegexpFilter.getStartText(pattern); } } diff --git a/java/java.hints/nbproject/project.xml b/java/java.hints/nbproject/project.xml index a9fe67477f43..fd59fc2ba2f4 100644 --- a/java/java.hints/nbproject/project.xml +++ b/java/java.hints/nbproject/project.xml @@ -91,8 +91,8 @@ - 3 - 3.0 + 10 + 10.0 diff --git a/java/java.lsp.server/nbproject/project.xml b/java/java.lsp.server/nbproject/project.xml index 9a8acdcab18b..0f1cc68adcad 100644 --- a/java/java.lsp.server/nbproject/project.xml +++ b/java/java.lsp.server/nbproject/project.xml @@ -509,8 +509,8 @@ - 2 - 2.47 + 3 + 3.0 diff --git a/java/java.source.base/nbproject/project.xml b/java/java.source.base/nbproject/project.xml index 47e43422d6d8..3e5172dcdb77 100644 --- a/java/java.source.base/nbproject/project.xml +++ b/java/java.source.base/nbproject/project.xml @@ -98,8 +98,8 @@ - 3 - 3.0 + 10 + 10.0 @@ -221,8 +221,8 @@ - 2 - 2.33 + 3 + 3.0 diff --git a/java/java.source.base/src/org/netbeans/modules/java/source/usages/ClassIndexImpl.java b/java/java.source.base/src/org/netbeans/modules/java/source/usages/ClassIndexImpl.java index d3e768f20480..4b685c4a1827 100644 --- a/java/java.source.base/src/org/netbeans/modules/java/source/usages/ClassIndexImpl.java +++ b/java/java.source.base/src/org/netbeans/modules/java/source/usages/ClassIndexImpl.java @@ -36,7 +36,6 @@ import javax.lang.model.element.ModuleElement; import javax.lang.model.element.TypeElement; import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldSelector; import org.netbeans.api.annotations.common.CheckForNull; import org.netbeans.api.annotations.common.NonNull; import org.netbeans.api.annotations.common.NullAllowed; @@ -107,7 +106,7 @@ public abstract void getDeclaredElements ( @NonNull String name, @NonNull ClassIndex.NameKind kind, @NonNull Set scope, - @NonNull FieldSelector selector, + @NonNull Set selector, @NonNull Convertor convertor, @NonNull Collection result) throws IOException, InterruptedException; diff --git a/java/java.source.base/src/org/netbeans/modules/java/source/usages/DocumentUtil.java b/java/java.source.base/src/org/netbeans/modules/java/source/usages/DocumentUtil.java index 86387ebe96fc..e583e4287591 100644 --- a/java/java.source.base/src/org/netbeans/modules/java/source/usages/DocumentUtil.java +++ b/java/java.source.base/src/org/netbeans/modules/java/source/usages/DocumentUtil.java @@ -25,21 +25,25 @@ import java.util.Arrays; import java.util.EnumSet; import java.util.List; +import java.util.Map; import java.util.Set; import javax.lang.model.element.Element; import javax.lang.model.element.ElementKind; import javax.lang.model.element.ModuleElement; import javax.lang.model.element.TypeElement; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.CharTokenizer; -import org.apache.lucene.analysis.KeywordAnalyzer; -import org.apache.lucene.analysis.PerFieldAnalyzerWrapper; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; +import org.apache.lucene.analysis.util.CharTokenizer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldSelector; -import org.apache.lucene.document.Fieldable; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; @@ -75,7 +79,7 @@ public class DocumentUtil { static final String FIELD_CASE_INSENSITIVE_FEATURE_IDENTS = "cifids"; //NOI18N private static final String FIELD_BINARY_NAME = "binaryName"; //NOI18N private static final String FIELD_SOURCE = "source"; //NOI18N - private static final String FIELD_REFERENCES = "references"; //NOI18N + public static final String FIELD_REFERENCES = "references"; //NOI18N private static final char NO = '-'; //NOI18N private static final char YES = '+'; //NOI18N @@ -95,14 +99,37 @@ public class DocumentUtil { private static final char EK_MODULE = 'M'; //NOI18N private static final int SIZE = ClassIndexImpl.UsageType.values().length; + private static final IndexableFieldType NOT_ANALYZED_STORED; + private static final IndexableFieldType NOT_ANALYZED_NOT_STORED; + private static final IndexableFieldType ANALYZED_NOT_STORED; + + static { + NOT_ANALYZED_STORED = new FieldType(); + ((FieldType) NOT_ANALYZED_STORED).setTokenized(false); + ((FieldType) NOT_ANALYZED_STORED).setIndexOptions(IndexOptions.DOCS_AND_FREQS); + ((FieldType) NOT_ANALYZED_STORED).setStored(true); + ((FieldType) NOT_ANALYZED_STORED).freeze(); + NOT_ANALYZED_NOT_STORED = new FieldType(); + ((FieldType) NOT_ANALYZED_NOT_STORED).setTokenized(false); + ((FieldType) NOT_ANALYZED_NOT_STORED).setIndexOptions(IndexOptions.DOCS_AND_FREQS); + ((FieldType) NOT_ANALYZED_NOT_STORED).setStored(false); + ((FieldType) NOT_ANALYZED_NOT_STORED).freeze(); + ANALYZED_NOT_STORED = new FieldType(); + ((FieldType) ANALYZED_NOT_STORED).setTokenized(true); + ((FieldType) ANALYZED_NOT_STORED).setIndexOptions(IndexOptions.DOCS_AND_FREQS); + ((FieldType) ANALYZED_NOT_STORED).setStored(true); + ((FieldType) ANALYZED_NOT_STORED).freeze(); + } + private DocumentUtil () { } public static Analyzer createAnalyzer() { - final PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new KeywordAnalyzer()); - analyzer.addAnalyzer(DocumentUtil.FIELD_IDENTS, new WhitespaceAnalyzer()); - analyzer.addAnalyzer(DocumentUtil.FIELD_FEATURE_IDENTS, new WhitespaceAnalyzer()); - analyzer.addAnalyzer(DocumentUtil.FIELD_CASE_INSENSITIVE_FEATURE_IDENTS, new DocumentUtil.LCWhitespaceAnalyzer()); + final PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new KeywordAnalyzer(), Map.of( + DocumentUtil.FIELD_IDENTS, new WhitespaceAnalyzer(), + DocumentUtil.FIELD_FEATURE_IDENTS, new WhitespaceAnalyzer(), + DocumentUtil.FIELD_CASE_INSENSITIVE_FEATURE_IDENTS, new DocumentUtil.LCWhitespaceAnalyzer() + )); return analyzer; } @@ -158,8 +185,8 @@ static String getBinaryName (final Document doc) { static String getBinaryName (final Document doc, final ElementKind[] kind) { assert doc != null; - final Field pkgField = doc.getField(FIELD_PACKAGE_NAME); - final Field snField = doc.getField (FIELD_BINARY_NAME); + final IndexableField pkgField = doc.getField(FIELD_PACKAGE_NAME); + final IndexableField snField = doc.getField (FIELD_BINARY_NAME); if (snField == null) { return null; } @@ -184,7 +211,7 @@ static String getBinaryName (final Document doc, final ElementKind[] kind) { public static String getSimpleBinaryName (final Document doc) { assert doc != null; - Fieldable field = doc.getFieldable(FIELD_BINARY_NAME); + IndexableField field = doc.getField(FIELD_BINARY_NAME); if (field == null) { return null; } else { @@ -194,14 +221,14 @@ public static String getSimpleBinaryName (final Document doc) { } public static String getSimpleName(final Document doc) { - final Fieldable field = doc.getFieldable(FIELD_SIMPLE_NAME); + final IndexableField field = doc.getField(FIELD_SIMPLE_NAME); return field == null ? null : field.stringValue(); } public static boolean isLocal(@NonNull final Document doc) { - Fieldable fld = doc.getFieldable(FIELD_BINARY_NAME); + IndexableField fld = doc.getField(FIELD_BINARY_NAME); if (fld == null) { return false; } else { @@ -220,14 +247,14 @@ public static boolean isLocal(@NonNull final Document doc) { static String getPackageName (final Document doc) { assert doc != null; - Field field = doc.getField(FIELD_PACKAGE_NAME); + IndexableField field = doc.getField(FIELD_PACKAGE_NAME); return field == null ? null : field.stringValue(); } //Term and query factories - static Query binaryNameQuery (final String resourceName) { - final BooleanQuery query = new BooleanQuery (); + static BooleanQuery.Builder binaryNameQuery (final String resourceName) { + final BooleanQuery.Builder query = new BooleanQuery.Builder(); int index = resourceName.lastIndexOf(BinaryName.PKG_SEPARATOR); // NOI18N String pkgName, sName; if (index < 0) { @@ -280,31 +307,31 @@ private static Document createDocument (final BinaryName name, ", FileName with type char: " + fileName; String simpleName = name.getSimpleName(); final String caseInsensitiveName = simpleName.toLowerCase(); //XXX: I18N, Locale - Document doc = new Document (); - Field field = new Field (FIELD_BINARY_NAME,fileName,Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); - doc.add (field); - field = new Field (FIELD_PACKAGE_NAME,pkgName,Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); - doc.add (field); - field = new Field (FIELD_SIMPLE_NAME,simpleName, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); - doc.add (field); - field = new Field (FIELD_CASE_INSENSITIVE_NAME, caseInsensitiveName, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); - doc.add (field); + Document doc = new Document(); + Field field = new Field(FIELD_BINARY_NAME, fileName, NOT_ANALYZED_STORED); + doc.add(field); + field = new Field(FIELD_PACKAGE_NAME, pkgName, NOT_ANALYZED_STORED); + doc.add(field); + field = new Field(FIELD_SIMPLE_NAME, simpleName, NOT_ANALYZED_STORED); + doc.add(field); + field = new Field(FIELD_CASE_INSENSITIVE_NAME, caseInsensitiveName, NOT_ANALYZED_STORED); + doc.add(field); for (String reference : references) { - field = new Field (FIELD_REFERENCES,reference,Field.Store.NO,Field.Index.NOT_ANALYZED_NO_NORMS); + field = new Field(FIELD_REFERENCES, reference, NOT_ANALYZED_NOT_STORED); doc.add(field); } if (featureIdents != null) { - field = new Field(FIELD_FEATURE_IDENTS, featureIdents, Field.Store.NO, Field.Index.ANALYZED_NO_NORMS); + field = new Field(FIELD_FEATURE_IDENTS, featureIdents, ANALYZED_NOT_STORED); doc.add(field); - field = new Field(FIELD_CASE_INSENSITIVE_FEATURE_IDENTS, featureIdents, Field.Store.NO, Field.Index.ANALYZED_NO_NORMS); + field = new Field(FIELD_CASE_INSENSITIVE_FEATURE_IDENTS, featureIdents, ANALYZED_NOT_STORED); doc.add(field); } if (idents != null) { - field = new Field(FIELD_IDENTS, idents, Field.Store.NO, Field.Index.ANALYZED_NO_NORMS); + field = new Field(FIELD_IDENTS, idents, ANALYZED_NOT_STORED); doc.add(field); } if (source != null) { - field = new Field (FIELD_SOURCE,source,Field.Store.YES,Field.Index.NOT_ANALYZED_NO_NORMS); + field = new Field(FIELD_SOURCE, source, NOT_ANALYZED_STORED); doc.add(field); } return doc; @@ -404,7 +431,7 @@ static char encodeKind ( } - public static FieldSelector declaredTypesFieldSelector ( + public static Set declaredTypesFieldSelector ( final boolean includeSource, final boolean includeSimpleName) { return includeSource ? @@ -416,7 +443,7 @@ public static FieldSelector declaredTypesFieldSelector ( Queries.createFieldSelector(FIELD_PACKAGE_NAME, FIELD_BINARY_NAME); } - static FieldSelector sourceNameFieldSelector () { + static Set sourceNameFieldSelector () { return Queries.createFieldSelector(FIELD_SOURCE); } @@ -436,9 +463,6 @@ static Queries.QueryKind translateQueryKind(final ClassIndex.NameKind kind) { // // in Lucene 3.5, WhitespaceTokenizer became final class; isTokenChar was copied. private static class LCWhitespaceTokenizer extends CharTokenizer { - LCWhitespaceTokenizer (final Reader r) { - super (r); - } @Override protected boolean isTokenChar(int c) { @@ -451,8 +475,9 @@ protected char normalize(char c) { } static final class LCWhitespaceAnalyzer extends Analyzer { - public TokenStream tokenStream(String fieldName, Reader reader) { - return new LCWhitespaceTokenizer(reader); + @Override + public TokenStreamComponents createComponents(String fieldName) { + return new TokenStreamComponents(new LCWhitespaceTokenizer()); } } @@ -620,7 +645,7 @@ private static class SourceNameConvertor implements Convertor { @Override public String convert(Document doc) { - Field field = doc.getField(FIELD_SOURCE); + IndexableField field = doc.getField(FIELD_SOURCE); return field == null ? null : field.stringValue(); } } @@ -651,31 +676,36 @@ private QueryClassesWithEncConvertor(final boolean fileBased) { public Query convert(Pair p) { final String resourceName = p.first(); final String sourceName = p.second(); - return fileBased ? createClassesInFileQuery(resourceName,sourceName) : createClassWithEnclosedQuery(resourceName, sourceName); + if(fileBased) { + return createClassesInFileQuery(resourceName,sourceName); + } else { + return createClassWithEnclosedQuery(resourceName, sourceName); + } } - private static Query createClassWithEnclosedQuery (final String resourceName, final String sourceName) { - final BooleanQuery query = createFQNQuery(resourceName); + private static BooleanQuery createClassWithEnclosedQuery (final String resourceName, final String sourceName) { + final BooleanQuery.Builder result = new BooleanQuery.Builder(); + result.add(createFQNQuery(resourceName), Occur.MUST); if (sourceName != null) { - query.add (new TermQuery(new Term (DocumentUtil.FIELD_SOURCE,sourceName)), Occur.MUST); + result.add (new TermQuery(new Term (DocumentUtil.FIELD_SOURCE,sourceName)), Occur.MUST); } - return query; + return result.build(); } - private static Query createClassesInFileQuery (final String resourceName, final String sourceName) { + private static BooleanQuery createClassesInFileQuery (final String resourceName, final String sourceName) { if (sourceName != null) { - final BooleanQuery result = new BooleanQuery(); + final BooleanQuery.Builder result = new BooleanQuery.Builder(); result.add(createFQNQuery(FileObjects.convertFolder2Package(FileObjects.stripExtension(sourceName))), Occur.SHOULD); result.add(new TermQuery(new Term (DocumentUtil.FIELD_SOURCE,sourceName)),Occur.SHOULD); - return result; + return result.build(); } else { - final BooleanQuery result = new BooleanQuery(); + final BooleanQuery.Builder result = new BooleanQuery.Builder(); result.add(createFQNQuery(resourceName), Occur.SHOULD); result.add(new TermQuery(new Term ( DocumentUtil.FIELD_SOURCE, new StringBuilder(FileObjects.convertPackage2Folder(resourceName)).append('.').append(FileObjects.JAVA).toString())), Occur.SHOULD); - return result; + return result.build(); } } @@ -690,16 +720,16 @@ private static BooleanQuery createFQNQuery(final String resourceName) { pkgName = resourceName.substring(0, index); sName = resourceName.substring(index+1); } - final BooleanQuery snQuery = new BooleanQuery(); + final BooleanQuery.Builder snQuery = new BooleanQuery.Builder(); snQuery.add (new WildcardQuery (new Term (DocumentUtil.FIELD_BINARY_NAME, sName + DocumentUtil.WILDCARD_QUERY_WILDCARD)),Occur.SHOULD); snQuery.add (new PrefixQuery (new Term (DocumentUtil.FIELD_BINARY_NAME, sName + '$')),Occur.SHOULD); //NOI18N if (pkgName.length() == 0) { - return snQuery; + return snQuery.build(); } - final BooleanQuery fqnQuery = new BooleanQuery(); + final BooleanQuery.Builder fqnQuery = new BooleanQuery.Builder(); fqnQuery.add(new TermQuery(new Term(DocumentUtil.FIELD_PACKAGE_NAME,pkgName)), Occur.MUST); - fqnQuery.add(snQuery, Occur.MUST); - return fqnQuery; + fqnQuery.add(snQuery.build(), Occur.MUST); + return fqnQuery.build(); } } @@ -715,21 +745,19 @@ private static Query binaryNameSourceNamePairQuery (final Pair bi final String binaryName = binaryNameSourceNamePair.first(); final String sourceName = binaryNameSourceNamePair.second(); assert binaryName != null || sourceName != null; - Query query = null; + BooleanQuery.Builder query = null; if (binaryName != null) { query = binaryNameQuery(binaryName); } if (sourceName != null) { if (query == null) { - query = new TermQuery(new Term (FIELD_SOURCE,sourceName)); + return new TermQuery(new Term (FIELD_SOURCE,sourceName)); } else { - assert query instanceof BooleanQuery : "The DocumentUtil.binaryNameQuery was incompatibly changed!"; //NOI18N - final BooleanQuery bq = (BooleanQuery) query; - bq.add(new TermQuery(new Term (FIELD_SOURCE,sourceName)), BooleanClause.Occur.MUST); + query.add(new TermQuery(new Term (FIELD_SOURCE,sourceName)), BooleanClause.Occur.MUST); } } assert query != null; - return query; + return query.build(); } } diff --git a/java/java.source.base/src/org/netbeans/modules/java/source/usages/PersistentClassIndex.java b/java/java.source.base/src/org/netbeans/modules/java/source/usages/PersistentClassIndex.java index a1ca6fc41f47..637599e1d6f3 100644 --- a/java/java.source.base/src/org/netbeans/modules/java/source/usages/PersistentClassIndex.java +++ b/java/java.source.base/src/org/netbeans/modules/java/source/usages/PersistentClassIndex.java @@ -27,10 +27,9 @@ import java.util.logging.Logger; import javax.lang.model.element.ElementKind; import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldSelector; -import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; import org.netbeans.api.annotations.common.CheckForNull; import org.netbeans.api.annotations.common.NonNull; import org.netbeans.api.annotations.common.NullAllowed; @@ -57,6 +56,8 @@ import org.openide.util.Pair; import org.openide.util.Parameters; +import static org.netbeans.modules.java.source.usages.DocumentUtil.FIELD_REFERENCES; + /** * * @author Petr Hrebejk, Tomas Zezula @@ -183,7 +184,7 @@ public FileObject[] getBinaryRoots() { @Override public String getSourceName (final String binaryName) throws IOException, InterruptedException { try { - final Query q = DocumentUtil.binaryNameQuery(binaryName); + final Query q = DocumentUtil.binaryNameQuery(binaryName).build(); Set names = new HashSet<>(); index.query(names, DocumentUtil.sourceNameConvertor(), DocumentUtil.sourceNameFieldSelector(), cancel.get(), q); return names.isEmpty() ? null : names.iterator().next(); @@ -273,7 +274,7 @@ public void getDeclaredElements ( @NonNull final String simpleName, @NonNull final ClassIndex.NameKind kind, @NonNull final Set scope, - @NonNull final FieldSelector selector, + @NonNull final Set selector, @NonNull final Convertor convertor, @NonNull final Collection result) throws InterruptedException, IOException { final Pair,Index> ctu = indexPath.getPatch(convertor); @@ -316,7 +317,7 @@ public void getDeclaredElements ( index.queryDocTerms( result, ctu.first(), - Term::text, + BytesRef::utf8ToString, DocumentUtil.declaredTypesFieldSelector(false, false), cancel.get(), query); @@ -324,7 +325,7 @@ public void getDeclaredElements ( ctu.second().queryDocTerms( result, convertor, - Term::text, + BytesRef::utf8ToString, DocumentUtil.declaredTypesFieldSelector(false, false), cancel.get(), query); @@ -356,8 +357,8 @@ public void getPackageNames (final String prefix, final boolean directOnly, fina } else { collectInto = result; } - final Pair,Term> filter = QueryUtil.createPackageFilter(prefix,directOnly); - index.queryTerms(collectInto, filter.second(), filter.first(), cancel.get()); + final Pair,String> filter = QueryUtil.createPackageFilter(prefix,directOnly); + index.queryTerms(collectInto, DocumentUtil.FIELD_PACKAGE_NAME, filter.second(), filter.first(), cancel.get()); if (cacheOp) { synchronized (PersistentClassIndex.this) { if (rootPkgCache == null) { @@ -384,12 +385,12 @@ public void getReferencesFrequences ( } try { IndexManager.priorityAccess(() -> { - final Term startTerm = DocumentUtil.referencesTerm("", null, true); //NOI18N final StoppableConvertor convertor = new FreqCollector( - startTerm, typeFreq, pkgFreq); + FIELD_REFERENCES, typeFreq, pkgFreq); ((Index.WithTermFrequencies)index).queryTermFrequencies( Collections.emptyList(), - startTerm, + FIELD_REFERENCES, + null, convertor, cancel.get()); return null; @@ -710,10 +711,10 @@ private static final class FreqCollector implements StoppableConvertor typeFreqs, @NonNull final Map pkgFreq) { - this.fieldName = startTerm.field(); + this.fieldName = fieldName; this.typeFreq = typeFreqs; this.pkgFreq = pkgFreq; } @@ -722,12 +723,9 @@ private static final class FreqCollector implements StoppableConvertor new BytesRef(s)).collect(Collectors.toList()) + ), + Occur.MUST); + qFiltered.add(q, Occur.MUST); + return qFiltered.build(); + } - static Pair,Term> createPackageFilter( + static Pair,String> createPackageFilter( final @NullAllowed String prefix, final boolean directOnly) { - final Term startTerm = new Term (DocumentUtil.FIELD_PACKAGE_NAME, prefix); - final StoppableConvertor filter = new PackageFilter(startTerm, directOnly); - return Pair.of(filter,startTerm); + final StoppableConvertor filter = new PackageFilter(prefix, directOnly); + return Pair.of(filter, prefix); } // - private static final class PackageFilter implements StoppableConvertor { + private static final class PackageFilter implements StoppableConvertor { private static final Stop STOP = new Stop(); private final boolean directOnly; private final boolean all; - private final String fieldName; private final String value; - PackageFilter(final @NonNull Term startTerm, final boolean directOnly) { - this.fieldName = startTerm.field(); - this.value = startTerm.text(); + PackageFilter(final String value, final boolean directOnly) { + this.value = value; this.directOnly = directOnly; this.all = value.length() == 0; } @Override - public String convert(Term currentTerm) throws Stop { - if (fieldName != currentTerm.field()) { - throw STOP; - } - String currentText = currentTerm.text(); + public String convert(BytesRef currentTerm) throws Stop { + String currentText = currentTerm.utf8ToString(); if (all || currentText.startsWith(value)) { if (directOnly) { int index = currentText.indexOf('.', value.length()); //NOI18N @@ -204,98 +184,5 @@ public String convert(Term currentTerm) throws Stop { } } - private static final class PackagesFilter extends Filter { - - private final SortedSet pkgs; - - PackagesFilter(@NonNull final SortedSet pkgs) { - assert pkgs != null; - this.pkgs = pkgs; - } - - @NonNull - @Override - public DocIdSet getDocIdSet(@NonNull final IndexReader reader) throws IOException { - final TermEnum enumerator = getTermEnum(reader); - // if current term in enum is null, the enum is empty -> shortcut - if (enumerator.term() == null) { - return DocIdSet.EMPTY_DOCIDSET; - } - try { - // else fill into a OpenBitSet - final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc()); - final int[] docs = new int[32]; - final int[] freqs = new int[32]; - final TermDocs termDocs = reader.termDocs(); - try { - do { - final Term term = enumerator.term(); - if (term == null) { - break; - } - termDocs.seek(term); - while (true) { - final int count = termDocs.read(docs, freqs); - if (count != 0) { - for (int i = 0; i < count; i++) { - bitSet.set(docs[i]); - } - } else { - break; - } - } - } while (enumerator.next()); - } finally { - termDocs.close(); - } - return bitSet; - } finally { - enumerator.close(); - } - } - - private TermEnum getTermEnum(@NonNull final IndexReader reader) { - return new TermEnum () { - private Iterator pkgsIt = pkgs.iterator(); - private String current; - { - next(); - } - - @Override - public boolean next() { - if (pkgsIt == null) { - throw new IllegalStateException("Already closed."); //NOI18N - } - if (pkgsIt.hasNext()) { - current = pkgsIt.next(); - return true; - } else { - current = null; - return false; - } - } - - @Override - public Term term() { - return current == null ? - null : - new Term (DocumentUtil.FIELD_PACKAGE_NAME, current); - } - - @Override - public int docFreq() { - return current == null ? - -1 : - 0; - } - - @Override - public void close() throws IOException { - pkgsIt = null; - } - }; - } - } // } diff --git a/java/java.source.base/test/unit/src/org/netbeans/api/java/source/JavaSourceTest.java b/java/java.source.base/test/unit/src/org/netbeans/api/java/source/JavaSourceTest.java index e2b30e47548a..b1c7a792c260 100644 --- a/java/java.source.base/test/unit/src/org/netbeans/api/java/source/JavaSourceTest.java +++ b/java/java.source.base/test/unit/src/org/netbeans/api/java/source/JavaSourceTest.java @@ -49,9 +49,8 @@ import com.sun.tools.javac.api.JavacTaskImpl; import junit.framework.*; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.FieldSelector; -import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; import org.netbeans.api.java.classpath.ClassPath; import org.netbeans.api.java.classpath.GlobalPathRegistry; import org.netbeans.api.java.lexer.JavaTokenId; @@ -64,7 +63,6 @@ import org.netbeans.junit.NbTestSuite; import org.netbeans.modules.java.source.BootClassPathUtil; import org.netbeans.modules.java.source.JavaSourceAccessor; -import org.netbeans.modules.java.source.NoJavacHelper; import org.netbeans.modules.java.source.classpath.CacheClassPath; import org.netbeans.modules.java.source.parsing.CompilationInfoImpl; import org.netbeans.modules.java.source.parsing.DocPositionRegion; @@ -2381,32 +2379,31 @@ public Status getStatus(boolean tryOpen) throws IOException { public void query( Collection result, Convertor convertor, - FieldSelector selector, + Set selector, AtomicBoolean cancel, Query... queries) throws IOException, InterruptedException { await(cancel); } - + @Override public void queryTerms( Collection result, - Term start, - StoppableConvertor filter, + String field, String start, + StoppableConvertor filter, AtomicBoolean cancel) throws IOException, InterruptedException { await (cancel); } - + @Override public void queryDocTerms( Map> result, Convertor convertor, - Convertor termConvertor, - FieldSelector selector, + Convertor termConvertor, Set selector, AtomicBoolean cancel, Query... queries) throws IOException, InterruptedException { await(cancel); } - + @Override public void store(Collection toAdd, Collection toDelete, Convertor docConvertor, Convertor queryConvertor, boolean optimize) throws IOException { } diff --git a/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/usages/LucenePerformanceTest.java b/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/usages/LucenePerformanceTest.java index a5ebe3cd5465..91788dbf7a3a 100644 --- a/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/usages/LucenePerformanceTest.java +++ b/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/usages/LucenePerformanceTest.java @@ -40,6 +40,7 @@ import javax.lang.model.element.ElementKind; import javax.lang.model.element.TypeElement; import org.apache.lucene.index.Term; +import org.apache.lucene.util.BytesRef; import org.netbeans.api.java.source.ElementHandle; import org.netbeans.junit.NbTestCase; import org.netbeans.modules.parsing.lucene.support.Index; @@ -91,8 +92,8 @@ public void testPerformance () throws Exception { Set result = new HashSet(); startTime = System.currentTimeMillis(); - final Pair,Term> filter = QueryUtil.createPackageFilter("", true); - index.queryTerms(result, filter.second(), filter.first(), null); + final Pair,String> filter = QueryUtil.createPackageFilter("", true); + index.queryTerms(result, DocumentUtil.FIELD_PACKAGE_NAME, filter.second(), filter.first(), null); endTime = System.currentTimeMillis(); delta = (endTime-startTime); System.out.println("Packages: " + delta); diff --git a/java/java.source/nbproject/project.xml b/java/java.source/nbproject/project.xml index 07ae9ca032a0..8f6dfece2385 100644 --- a/java/java.source/nbproject/project.xml +++ b/java/java.source/nbproject/project.xml @@ -99,8 +99,8 @@ - 3 - 3.0 + 10 + 10.0 diff --git a/java/java.sourceui/nbproject/project.xml b/java/java.sourceui/nbproject/project.xml index e1dbf54c629e..fbec295fcb1f 100644 --- a/java/java.sourceui/nbproject/project.xml +++ b/java/java.sourceui/nbproject/project.xml @@ -99,8 +99,8 @@ - 3 - 3.0 + 10 + 10.0 @@ -212,8 +212,8 @@ - 2 - 2.9 + 3 + 3.0 diff --git a/java/spi.java.hints/nbproject/project.xml b/java/spi.java.hints/nbproject/project.xml index e543c2f4fe8a..9fd6809fedd2 100644 --- a/java/spi.java.hints/nbproject/project.xml +++ b/java/spi.java.hints/nbproject/project.xml @@ -82,8 +82,8 @@ - 3 - 3.0 + 10 + 10.0 diff --git a/java/whitelist/nbproject/project.xml b/java/whitelist/nbproject/project.xml index a86ffb8580c3..d020f59765d4 100644 --- a/java/whitelist/nbproject/project.xml +++ b/java/whitelist/nbproject/project.xml @@ -106,8 +106,8 @@ - 2 - 2.9 + 3 + 3.0 diff --git a/nbbuild/licenses/Apache-2.0-lucene b/nbbuild/licenses/Apache-2.0-lucene index 0b6e728a51a0..5906dd021595 100644 --- a/nbbuild/licenses/Apache-2.0-lucene +++ b/nbbuild/licenses/Apache-2.0-lucene @@ -202,16 +202,16 @@ -Some code in src/java/org/apache/lucene/util/UnicodeUtil.java was +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was derived from unicode conversion examples available at http://www.unicode.org/Public/PROGRAMS/CVTUTF. Here is the copyright from those sources: /* * Copyright 2001-2004 Unicode, Inc. - * + * * Disclaimer - * + * * This source code is provided as is by Unicode, Inc. No claims are * made as to fitness for any particular purpose. No warranties of any * kind are expressed or implied. The recipient agrees to determine @@ -219,9 +219,9 @@ from those sources: * purchased on magnetic or optical media from Unicode, Inc., the * sole remedy for any claim will be exchange of defective media * within 90 days of receipt. - * + * * Limitations on Rights to Redistribute This Code - * + * * Unicode, Inc. hereby grants the right to freely use the information * supplied in this file in the creation of products supporting the * Unicode Standard, and to make copies of this file in any form @@ -230,43 +230,106 @@ from those sources: */ -Some code in src/java/org/apache/lucene/util/ArrayUtil.java was +Some code in core/src/java/org/apache/lucene/util/ArrayUtil.java was derived from Python 2.4.2 sources available at http://www.python.org. Full license is here: http://www.python.org/download/releases/2.4.2/license/ +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from Python 3.1.2 sources available at +http://www.python.org. Full license is here: + + http://www.python.org/download/releases/3.1.2/license/ + +Some code in core/src/java/org/apache/lucene/util/automaton was +derived from Brics automaton sources available at +www.brics.dk/automaton/. Here is the copyright from those sources: -Some code in src/java/org/apache/lucene/util/UnicodeUtil.java was +/* + * Copyright (c) 2001-2009 Anders Moeller + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +The levenshtein automata tables in core/src/java/org/apache/lucene/util/automaton +were automatically generated with the moman/finenight FSA package. +Here is the copyright for those sources: + +# Copyright (c) 2010, Jean-Philippe Barrette-LaPierre, +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was derived from ICU (http://www.icu-project.org) -The full license is available here: - http://source.icu-project.org/repos/icu/icu/trunk/license.html +The full license is available here: + https://github.com/unicode-org/icu/blob/main/icu4c/LICENSE /* * Copyright (C) 1999-2010, International Business Machines * Corporation and others. All Rights Reserved. * - * Permission is hereby granted, free of charge, to any person obtaining a copy + * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, and/or sell copies of the - * Software, and to permit persons to whom the Software is furnished to do so, - * provided that the above copyright notice(s) and this permission notice appear + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * provided that the above copyright notice(s) and this permission notice appear * in all copies of the Software and that both the above copyright notice(s) and * this permission notice appear in supporting documentation. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. - * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE - * LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR - * ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER - * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE + * LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR + * ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER + * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. * - * Except as contained in this notice, the name of a copyright holder shall not - * be used in advertising or otherwise to promote the sale, use or other - * dealings in this Software without prior written authorization of the + * Except as contained in this notice, the name of a copyright holder shall not + * be used in advertising or otherwise to promote the sale, use or other + * dealings in this Software without prior written authorization of the * copyright holder. */ @@ -332,3 +395,111 @@ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The following license applies to the Morfologik project: + +Copyright (c) 2006 Dawid Weiss +Copyright (c) 2007-2011 Dawid Weiss, Marcin Miłkowski +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of Morfologik nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +The dictionary comes from Morfologik project. Morfologik uses data from +Polish ispell/myspell dictionary hosted at http://www.sjp.pl/slownik/en/ and +is licenced on the terms of (inter alia) LGPL and Creative Commons +ShareAlike. The part-of-speech tags were added in Morfologik project and +are not found in the data from sjp.pl. The tagset is similar to IPI PAN +tagset. + +--- + +The following license applies to the Morfeusz project, +used by org.apache.lucene.analysis.morfologik. + +BSD-licensed dictionary of Polish (SGJP) +http://sgjp.pl/morfeusz/ + +Copyright © 2011 Zygmunt Saloni, Włodzimierz Gruszczyński, +Marcin Woliński, Robert Wołosz + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDERS “AS IS” AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +core/src/java/org/apache/lucene/util/compress/LZ4.java is a Java +implementation of the LZ4 (https://github.com/lz4/lz4/tree/dev/lib) +compression format for Lucene's DataInput/DataOutput abstractions. + +LZ4 Library +Copyright (c) 2011-2016, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.