From a467a42278f864bd333bf4bc551f5222d3f91ad2 Mon Sep 17 00:00:00 2001 From: Jeff Shaw Date: Fri, 7 Feb 2025 16:35:57 -0500 Subject: [PATCH 1/3] Use JMH for benchmarks. --- .gitmodules | 2 +- README.md | 12 + benchmarks/pom.xml | 73 ++++++ .../java/com/maxmind/db/BenchmarkGet.java | 71 +++++ dev-bin/release.sh | 8 +- pom.xml | 247 +----------------- .../checkstyle-suppressions.xml | 0 reader/pom.xml | 237 +++++++++++++++++ .../java/com/maxmind/db/BufferHolder.java | 0 .../main/java/com/maxmind/db/CHMCache.java | 0 .../main/java/com/maxmind/db/CacheKey.java | 0 .../com/maxmind/db/CachedConstructor.java | 0 .../maxmind/db/ClosedDatabaseException.java | 0 .../db/ConstructorNotFoundException.java | 0 .../main/java/com/maxmind/db/CtrlData.java | 0 .../java/com/maxmind/db/DatabaseRecord.java | 0 .../java/com/maxmind/db/DecodedValue.java | 0 .../main/java/com/maxmind/db/Decoder.java | 0 .../maxmind/db/DeserializationException.java | 0 .../maxmind/db/InvalidDatabaseException.java | 0 .../maxmind/db/InvalidNetworkException.java | 0 .../com/maxmind/db/MaxMindDbConstructor.java | 0 .../com/maxmind/db/MaxMindDbParameter.java | 0 .../main/java/com/maxmind/db/Metadata.java | 0 .../main/java/com/maxmind/db/Network.java | 0 .../main/java/com/maxmind/db/Networks.java | 0 .../db/NetworksIterationException.java | 0 .../main/java/com/maxmind/db/NoCache.java | 0 .../main/java/com/maxmind/db/NodeCache.java | 0 .../db/ParameterNotFoundException.java | 0 .../src}/main/java/com/maxmind/db/Reader.java | 0 .../src}/main/java/com/maxmind/db/Type.java | 0 .../java/com/maxmind/db/package-info.java | 0 .../src}/main/java/module-info.java | 0 .../test/java/com/maxmind/db/DecoderTest.java | 0 .../com/maxmind/db/MultiThreadedTest.java | 0 .../test/java/com/maxmind/db/NetworkTest.java | 0 .../test/java/com/maxmind/db/PointerTest.java | 0 .../test/java/com/maxmind/db/ReaderTest.java | 0 {src => reader/src}/test/resources/maxmind-db | 0 sample/Benchmark.java | 62 ----- 41 files changed, 412 insertions(+), 300 deletions(-) create mode 100644 benchmarks/pom.xml create mode 100644 benchmarks/src/main/java/com/maxmind/db/BenchmarkGet.java rename checkstyle-suppressions.xml => reader/checkstyle-suppressions.xml (100%) create mode 100644 reader/pom.xml rename {src => reader/src}/main/java/com/maxmind/db/BufferHolder.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/CHMCache.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/CacheKey.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/CachedConstructor.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/ClosedDatabaseException.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/ConstructorNotFoundException.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/CtrlData.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/DatabaseRecord.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/DecodedValue.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/Decoder.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/DeserializationException.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/InvalidDatabaseException.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/InvalidNetworkException.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/MaxMindDbConstructor.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/MaxMindDbParameter.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/Metadata.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/Network.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/Networks.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/NetworksIterationException.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/NoCache.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/NodeCache.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/ParameterNotFoundException.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/Reader.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/Type.java (100%) rename {src => reader/src}/main/java/com/maxmind/db/package-info.java (100%) rename {src => reader/src}/main/java/module-info.java (100%) rename {src => reader/src}/test/java/com/maxmind/db/DecoderTest.java (100%) rename {src => reader/src}/test/java/com/maxmind/db/MultiThreadedTest.java (100%) rename {src => reader/src}/test/java/com/maxmind/db/NetworkTest.java (100%) rename {src => reader/src}/test/java/com/maxmind/db/PointerTest.java (100%) rename {src => reader/src}/test/java/com/maxmind/db/ReaderTest.java (100%) rename {src => reader/src}/test/resources/maxmind-db (100%) delete mode 100644 sample/Benchmark.java diff --git a/.gitmodules b/.gitmodules index 6660f3f7..2fb5f51a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "src/test/resources/maxmind-db"] - path = src/test/resources/maxmind-db + path = reader/src/test/resources/maxmind-db url = https://github.com/maxmind/MaxMind-DB diff --git a/README.md b/README.md index 8e8c4532..8008aa8a 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,18 @@ Maven, you must Failure to do so will result in `InvalidDatabaseException` exceptions being thrown when querying the database. +## Benchmarking ## + +Set an environment variable `GEO_LITE` with the path to `GeoLite2-City.mmdb`. + +```shell +mvn -Dcheckstyle.skip -DskipTests clean package +GEO_LITE=/.../GeoLite2-City.mmdb java -jar benchmarks/target/microbenchmarks.jar +``` + +For more, see [https://github.com/openjdk/jmh](https://github.com/openjdk/jmh) or +`java -jar benchmarks/target/microbenchmarks.jar -h`. + ## Format ## The MaxMind DB format is an open format for quickly mapping IP addresses to diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml new file mode 100644 index 00000000..5941ee97 --- /dev/null +++ b/benchmarks/pom.xml @@ -0,0 +1,73 @@ + + + 4.0.0 + com.maxmind.db + benchmarks + 3.1.2-SNAPSHOT + jar + + 1.37 + + + + com.maxmind.db + maxmind-db + 3.1.2-SNAPSHOT + + + org.openjdk.jmh + jmh-core + ${jmhVersion} + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 11 + 11 + 11 + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmhVersion} + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + package + + shade + + + microbenchmarks + + + org.openjdk.jmh.Main + + + + + *:* + + META-INF/services/javax.annotation.processing.Processor + + + + + + + + + + diff --git a/benchmarks/src/main/java/com/maxmind/db/BenchmarkGet.java b/benchmarks/src/main/java/com/maxmind/db/BenchmarkGet.java new file mode 100644 index 00000000..23db99d8 --- /dev/null +++ b/benchmarks/src/main/java/com/maxmind/db/BenchmarkGet.java @@ -0,0 +1,71 @@ +package com.maxmind.db; + +import com.maxmind.db.Reader.FileMode; +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 5) +@Measurement(iterations = 10, time = 5) +@Fork(1) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +public class BenchmarkGet { + private static final int COUNT = 1_000_000; + + private static InetAddress[] getInetAddresses(final int seed) throws UnknownHostException { + final InetAddress[] addresses = new InetAddress[COUNT]; + final Random random = new Random(seed); + final byte[] address = new byte[4]; + for (int addressIx = 0; addressIx < COUNT; addressIx++) { + random.nextBytes(address); + addresses[addressIx] = InetAddress.getByAddress(address); + } + return addresses; + } + + InetAddress[] addresses; + Reader reader; + Reader cachedReader; + + @Setup + public void setup() throws IOException { + addresses = getInetAddresses(0); + final File database = new File(System.getenv("GEO_LITE")); + reader = new Reader(database, FileMode.MEMORY_MAPPED, NoCache.getInstance()); + cachedReader = new Reader(database, FileMode.MEMORY_MAPPED, new CHMCache()); + } + + @Benchmark + @OperationsPerInvocation(COUNT) + public void withoutCaching(Blackhole bh) throws IOException { + for (InetAddress address: addresses) { + bh.consume(reader.get(address, Map.class)); + } + } + + @Benchmark + @OperationsPerInvocation(COUNT) + public void withCaching(Blackhole bh) throws IOException { + for (InetAddress address: addresses) { + bh.consume(cachedReader.get(address, Map.class)); + } + } +} diff --git a/dev-bin/release.sh b/dev-bin/release.sh index 497c346e..38ccd69f 100755 --- a/dev-bin/release.sh +++ b/dev-bin/release.sh @@ -97,11 +97,11 @@ fi # could be combined with the primary build -mvn release:clean -mvn release:prepare -DreleaseVersion="$version" -Dtag="$tag" -mvn release:perform +mvn -pl reader release:clean +mvn -pl reader release:prepare -DreleaseVersion="$version" -Dtag="$tag" +mvn -pl reader release:perform rm -fr ".gh-pages/doc/$tag" -cp -r target/checkout/target/reports/apidocs ".gh-pages/doc/$tag" +cp -r reader/target/checkout/target/reports/apidocs ".gh-pages/doc/$tag" rm .gh-pages/doc/latest ln -fs "$tag" .gh-pages/doc/latest diff --git a/pom.xml b/pom.xml index f71271c2..8b7d95bc 100644 --- a/pom.xml +++ b/pom.xml @@ -1,237 +1,18 @@ - 4.0.0 - com.maxmind.db - maxmind-db - 3.1.2-SNAPSHOT - jar - MaxMind DB Reader - Reader for MaxMind DB - http://dev.maxmind.com/ - - - Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0.html - repo - - - - MaxMind, Inc. - http://www.maxmind.com/ - - - https://github.com/maxmind/MaxMind-DB-Reader-java - scm:git:git://github.com:maxmind/MaxMind-DB-Reader-java.git - scm:git:git@github.com:maxmind/MaxMind-DB-Reader-java.git - HEAD + 4.0.0 + pom + com.maxmind.db + maxmind-parent + 3.1.2-SNAPSHOT + + https://github.com/maxmind/MaxMind-DB-Reader-java + scm:git:git://github.com:maxmind/MaxMind-DB-Reader-java.git + scm:git:git@github.com:maxmind/MaxMind-DB-Reader-java.git + HEAD - - https://github.com/maxmind/MaxMind-DB-Reader-java/issues - GitHub - - - - oschwald - Gregory J. Oschwald - goschwald@maxmind.com - - - - - org.junit.jupiter - junit-jupiter - 5.11.4 - test - - - org.hamcrest - java-hamcrest - 2.0.0.0 - test - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - 3.5.0 - - - enforce-maven - - enforce - - - - - 3.6.3 - - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - 3.6.0 - - true - checkstyle.xml - checkstyle-suppressions.xml - warning - - - - com.puppycrawl.tools - checkstyle - 10.21.2 - - - - - test - test - - check - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 3.2.7 - - - sign-artifacts - verify - - sign - - - - - - org.apache.maven.plugins - maven-source-plugin - 3.3.1 - - - attach-sources - package - - jar-no-fork - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 3.11.2 - - -missing - - - - - jar - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.13.0 - - 11 - 11 - 11 - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.5.2 - - - org.codehaus.mojo - versions-maven-plugin - 2.18.0 - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.7.0 - true - - sonatype-nexus-staging - https://oss.sonatype.org/ - true - - - - default-deploy - deploy - - deploy - - - - - - - - UTF-8 - - - - not-windows - - - !Windows - - - - - - org.codehaus.mojo - exec-maven-plugin - 3.5.0 - - - initialize - invoke build - - exec - - - - - git - submodule update --init --recursive - - - - - - - - - sonatype-nexus-staging - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - + + reader + benchmarks + diff --git a/checkstyle-suppressions.xml b/reader/checkstyle-suppressions.xml similarity index 100% rename from checkstyle-suppressions.xml rename to reader/checkstyle-suppressions.xml diff --git a/reader/pom.xml b/reader/pom.xml new file mode 100644 index 00000000..c05bcb75 --- /dev/null +++ b/reader/pom.xml @@ -0,0 +1,237 @@ + + + 4.0.0 + jar + com.maxmind.db + maxmind-db + 3.1.2-SNAPSHOT + MaxMind DB Reader + Reader for MaxMind DB + http://dev.maxmind.com/ + + + Apache License 2.0 + http://www.apache.org/licenses/LICENSE-2.0.html + repo + + + + MaxMind, Inc. + http://www.maxmind.com/ + + + https://github.com/maxmind/MaxMind-DB-Reader-java + scm:git:git://github.com:maxmind/MaxMind-DB-Reader-java.git + scm:git:git@github.com:maxmind/MaxMind-DB-Reader-java.git + HEAD + + + https://github.com/maxmind/MaxMind-DB-Reader-java/issues + GitHub + + + + oschwald + Gregory J. Oschwald + goschwald@maxmind.com + + + + + org.junit.jupiter + junit-jupiter + 5.11.4 + test + + + org.hamcrest + java-hamcrest + 2.0.0.0 + test + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.5.0 + + + enforce-maven + + enforce + + + + + 3.6.3 + + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.6.0 + + true + ../checkstyle.xml + checkstyle-suppressions.xml + warning + + + + com.puppycrawl.tools + checkstyle + 10.21.2 + + + + + test + test + + check + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 3.2.7 + + + sign-artifacts + verify + + sign + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.3.1 + + + attach-sources + package + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.11.2 + + -missing + + + + + jar + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 11 + 11 + 11 + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.2 + + + org.codehaus.mojo + versions-maven-plugin + 2.18.0 + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.7.0 + true + + sonatype-nexus-staging + https://oss.sonatype.org/ + true + + + + default-deploy + deploy + + deploy + + + + + + + + UTF-8 + + + + not-windows + + + !Windows + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.5.0 + + + initialize + invoke build + + exec + + + + + git + submodule update --init --recursive + + + + + + + + + sonatype-nexus-staging + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + diff --git a/src/main/java/com/maxmind/db/BufferHolder.java b/reader/src/main/java/com/maxmind/db/BufferHolder.java similarity index 100% rename from src/main/java/com/maxmind/db/BufferHolder.java rename to reader/src/main/java/com/maxmind/db/BufferHolder.java diff --git a/src/main/java/com/maxmind/db/CHMCache.java b/reader/src/main/java/com/maxmind/db/CHMCache.java similarity index 100% rename from src/main/java/com/maxmind/db/CHMCache.java rename to reader/src/main/java/com/maxmind/db/CHMCache.java diff --git a/src/main/java/com/maxmind/db/CacheKey.java b/reader/src/main/java/com/maxmind/db/CacheKey.java similarity index 100% rename from src/main/java/com/maxmind/db/CacheKey.java rename to reader/src/main/java/com/maxmind/db/CacheKey.java diff --git a/src/main/java/com/maxmind/db/CachedConstructor.java b/reader/src/main/java/com/maxmind/db/CachedConstructor.java similarity index 100% rename from src/main/java/com/maxmind/db/CachedConstructor.java rename to reader/src/main/java/com/maxmind/db/CachedConstructor.java diff --git a/src/main/java/com/maxmind/db/ClosedDatabaseException.java b/reader/src/main/java/com/maxmind/db/ClosedDatabaseException.java similarity index 100% rename from src/main/java/com/maxmind/db/ClosedDatabaseException.java rename to reader/src/main/java/com/maxmind/db/ClosedDatabaseException.java diff --git a/src/main/java/com/maxmind/db/ConstructorNotFoundException.java b/reader/src/main/java/com/maxmind/db/ConstructorNotFoundException.java similarity index 100% rename from src/main/java/com/maxmind/db/ConstructorNotFoundException.java rename to reader/src/main/java/com/maxmind/db/ConstructorNotFoundException.java diff --git a/src/main/java/com/maxmind/db/CtrlData.java b/reader/src/main/java/com/maxmind/db/CtrlData.java similarity index 100% rename from src/main/java/com/maxmind/db/CtrlData.java rename to reader/src/main/java/com/maxmind/db/CtrlData.java diff --git a/src/main/java/com/maxmind/db/DatabaseRecord.java b/reader/src/main/java/com/maxmind/db/DatabaseRecord.java similarity index 100% rename from src/main/java/com/maxmind/db/DatabaseRecord.java rename to reader/src/main/java/com/maxmind/db/DatabaseRecord.java diff --git a/src/main/java/com/maxmind/db/DecodedValue.java b/reader/src/main/java/com/maxmind/db/DecodedValue.java similarity index 100% rename from src/main/java/com/maxmind/db/DecodedValue.java rename to reader/src/main/java/com/maxmind/db/DecodedValue.java diff --git a/src/main/java/com/maxmind/db/Decoder.java b/reader/src/main/java/com/maxmind/db/Decoder.java similarity index 100% rename from src/main/java/com/maxmind/db/Decoder.java rename to reader/src/main/java/com/maxmind/db/Decoder.java diff --git a/src/main/java/com/maxmind/db/DeserializationException.java b/reader/src/main/java/com/maxmind/db/DeserializationException.java similarity index 100% rename from src/main/java/com/maxmind/db/DeserializationException.java rename to reader/src/main/java/com/maxmind/db/DeserializationException.java diff --git a/src/main/java/com/maxmind/db/InvalidDatabaseException.java b/reader/src/main/java/com/maxmind/db/InvalidDatabaseException.java similarity index 100% rename from src/main/java/com/maxmind/db/InvalidDatabaseException.java rename to reader/src/main/java/com/maxmind/db/InvalidDatabaseException.java diff --git a/src/main/java/com/maxmind/db/InvalidNetworkException.java b/reader/src/main/java/com/maxmind/db/InvalidNetworkException.java similarity index 100% rename from src/main/java/com/maxmind/db/InvalidNetworkException.java rename to reader/src/main/java/com/maxmind/db/InvalidNetworkException.java diff --git a/src/main/java/com/maxmind/db/MaxMindDbConstructor.java b/reader/src/main/java/com/maxmind/db/MaxMindDbConstructor.java similarity index 100% rename from src/main/java/com/maxmind/db/MaxMindDbConstructor.java rename to reader/src/main/java/com/maxmind/db/MaxMindDbConstructor.java diff --git a/src/main/java/com/maxmind/db/MaxMindDbParameter.java b/reader/src/main/java/com/maxmind/db/MaxMindDbParameter.java similarity index 100% rename from src/main/java/com/maxmind/db/MaxMindDbParameter.java rename to reader/src/main/java/com/maxmind/db/MaxMindDbParameter.java diff --git a/src/main/java/com/maxmind/db/Metadata.java b/reader/src/main/java/com/maxmind/db/Metadata.java similarity index 100% rename from src/main/java/com/maxmind/db/Metadata.java rename to reader/src/main/java/com/maxmind/db/Metadata.java diff --git a/src/main/java/com/maxmind/db/Network.java b/reader/src/main/java/com/maxmind/db/Network.java similarity index 100% rename from src/main/java/com/maxmind/db/Network.java rename to reader/src/main/java/com/maxmind/db/Network.java diff --git a/src/main/java/com/maxmind/db/Networks.java b/reader/src/main/java/com/maxmind/db/Networks.java similarity index 100% rename from src/main/java/com/maxmind/db/Networks.java rename to reader/src/main/java/com/maxmind/db/Networks.java diff --git a/src/main/java/com/maxmind/db/NetworksIterationException.java b/reader/src/main/java/com/maxmind/db/NetworksIterationException.java similarity index 100% rename from src/main/java/com/maxmind/db/NetworksIterationException.java rename to reader/src/main/java/com/maxmind/db/NetworksIterationException.java diff --git a/src/main/java/com/maxmind/db/NoCache.java b/reader/src/main/java/com/maxmind/db/NoCache.java similarity index 100% rename from src/main/java/com/maxmind/db/NoCache.java rename to reader/src/main/java/com/maxmind/db/NoCache.java diff --git a/src/main/java/com/maxmind/db/NodeCache.java b/reader/src/main/java/com/maxmind/db/NodeCache.java similarity index 100% rename from src/main/java/com/maxmind/db/NodeCache.java rename to reader/src/main/java/com/maxmind/db/NodeCache.java diff --git a/src/main/java/com/maxmind/db/ParameterNotFoundException.java b/reader/src/main/java/com/maxmind/db/ParameterNotFoundException.java similarity index 100% rename from src/main/java/com/maxmind/db/ParameterNotFoundException.java rename to reader/src/main/java/com/maxmind/db/ParameterNotFoundException.java diff --git a/src/main/java/com/maxmind/db/Reader.java b/reader/src/main/java/com/maxmind/db/Reader.java similarity index 100% rename from src/main/java/com/maxmind/db/Reader.java rename to reader/src/main/java/com/maxmind/db/Reader.java diff --git a/src/main/java/com/maxmind/db/Type.java b/reader/src/main/java/com/maxmind/db/Type.java similarity index 100% rename from src/main/java/com/maxmind/db/Type.java rename to reader/src/main/java/com/maxmind/db/Type.java diff --git a/src/main/java/com/maxmind/db/package-info.java b/reader/src/main/java/com/maxmind/db/package-info.java similarity index 100% rename from src/main/java/com/maxmind/db/package-info.java rename to reader/src/main/java/com/maxmind/db/package-info.java diff --git a/src/main/java/module-info.java b/reader/src/main/java/module-info.java similarity index 100% rename from src/main/java/module-info.java rename to reader/src/main/java/module-info.java diff --git a/src/test/java/com/maxmind/db/DecoderTest.java b/reader/src/test/java/com/maxmind/db/DecoderTest.java similarity index 100% rename from src/test/java/com/maxmind/db/DecoderTest.java rename to reader/src/test/java/com/maxmind/db/DecoderTest.java diff --git a/src/test/java/com/maxmind/db/MultiThreadedTest.java b/reader/src/test/java/com/maxmind/db/MultiThreadedTest.java similarity index 100% rename from src/test/java/com/maxmind/db/MultiThreadedTest.java rename to reader/src/test/java/com/maxmind/db/MultiThreadedTest.java diff --git a/src/test/java/com/maxmind/db/NetworkTest.java b/reader/src/test/java/com/maxmind/db/NetworkTest.java similarity index 100% rename from src/test/java/com/maxmind/db/NetworkTest.java rename to reader/src/test/java/com/maxmind/db/NetworkTest.java diff --git a/src/test/java/com/maxmind/db/PointerTest.java b/reader/src/test/java/com/maxmind/db/PointerTest.java similarity index 100% rename from src/test/java/com/maxmind/db/PointerTest.java rename to reader/src/test/java/com/maxmind/db/PointerTest.java diff --git a/src/test/java/com/maxmind/db/ReaderTest.java b/reader/src/test/java/com/maxmind/db/ReaderTest.java similarity index 100% rename from src/test/java/com/maxmind/db/ReaderTest.java rename to reader/src/test/java/com/maxmind/db/ReaderTest.java diff --git a/src/test/resources/maxmind-db b/reader/src/test/resources/maxmind-db similarity index 100% rename from src/test/resources/maxmind-db rename to reader/src/test/resources/maxmind-db diff --git a/sample/Benchmark.java b/sample/Benchmark.java deleted file mode 100644 index 3d73011b..00000000 --- a/sample/Benchmark.java +++ /dev/null @@ -1,62 +0,0 @@ -import java.io.File; -import java.io.IOException; -import java.net.InetAddress; -import java.util.Map; -import java.util.Random; - -import com.maxmind.db.CHMCache; -import com.maxmind.db.InvalidDatabaseException; -import com.maxmind.db.NoCache; -import com.maxmind.db.NodeCache; -import com.maxmind.db.Reader; -import com.maxmind.db.Reader.FileMode; - -public class Benchmark { - - private final static int COUNT = 1000000; - private final static int WARMUPS = 3; - private final static int BENCHMARKS = 5; - private final static boolean TRACE = false; - - public static void main(String[] args) throws IOException, InvalidDatabaseException { - File file = new File(args.length > 0 ? args[0] : "GeoLite2-City.mmdb"); - System.out.println("No caching"); - loop("Warming up", file, WARMUPS, NoCache.getInstance()); - loop("Benchmarking", file, BENCHMARKS, NoCache.getInstance()); - - System.out.println("With caching"); - loop("Warming up", file, WARMUPS, new CHMCache()); - loop("Benchmarking", file, BENCHMARKS, new CHMCache()); - } - - private static void loop(String msg, File file, int loops, NodeCache cache) throws IOException { - System.out.println(msg); - for (int i = 0; i < loops; i++) { - Reader r = new Reader(file, FileMode.MEMORY_MAPPED, cache); - bench(r, COUNT, i); - } - System.out.println(); - } - - private static void bench(Reader r, int count, int seed) throws IOException { - Random random = new Random(seed); - long startTime = System.nanoTime(); - byte[] address = new byte[4]; - for (int i = 0; i < count; i++) { - random.nextBytes(address); - InetAddress ip = InetAddress.getByAddress(address); - Map t = r.get(ip, Map.class); - if (TRACE) { - if (i % 50000 == 0) { - System.out.println(i + " " + ip); - System.out.println(t); - } - } - } - long endTime = System.nanoTime(); - - long duration = endTime - startTime; - long qps = count * 1000000000L / duration; - System.out.println("Requests per second: " + qps); - } -} From 529af2e9c13690bddbab201e5df19c80007d61ad Mon Sep 17 00:00:00 2001 From: Jeff Shaw Date: Fri, 7 Feb 2025 17:44:06 -0500 Subject: [PATCH 2/3] Allow Reader to accept any method of reading bytes. `bufferHolderReference` can be volatile instead of atomic, since operations like `getAndSet` aren't used. --- .../com/maxmind/db/ByteBufferByteReader.java | 60 ++++++++ .../main/java/com/maxmind/db/ByteReader.java | 57 +++++++ ...ufferHolder.java => ByteReaderHolder.java} | 29 ++-- .../main/java/com/maxmind/db/CacheKey.java | 8 +- .../main/java/com/maxmind/db/CtrlData.java | 10 +- .../src/main/java/com/maxmind/db/Decoder.java | 46 ++---- .../com/maxmind/db/IntLimitedByteReader.java | 34 ++++ .../main/java/com/maxmind/db/Networks.java | 4 +- .../com/maxmind/db/PositionedByteReader.java | 145 ++++++++++++++++++ .../db/PositionedIntLimitedByteReader.java | 109 +++++++++++++ .../src/main/java/com/maxmind/db/Reader.java | 75 +++++---- .../com/maxmind/db/ByteReaderHolderTest.java | 22 +++ .../test/java/com/maxmind/db/DecoderTest.java | 6 +- .../test/java/com/maxmind/db/PointerTest.java | 2 +- .../com/maxmind/db/PositionByteReader.java | 33 ++++ .../maxmind/db/PositionedByteReaderTest.java | 33 ++++ .../test/java/com/maxmind/db/ReaderTest.java | 16 +- 17 files changed, 594 insertions(+), 95 deletions(-) create mode 100644 reader/src/main/java/com/maxmind/db/ByteBufferByteReader.java create mode 100644 reader/src/main/java/com/maxmind/db/ByteReader.java rename reader/src/main/java/com/maxmind/db/{BufferHolder.java => ByteReaderHolder.java} (74%) create mode 100644 reader/src/main/java/com/maxmind/db/IntLimitedByteReader.java create mode 100644 reader/src/main/java/com/maxmind/db/PositionedByteReader.java create mode 100644 reader/src/main/java/com/maxmind/db/PositionedIntLimitedByteReader.java create mode 100644 reader/src/test/java/com/maxmind/db/ByteReaderHolderTest.java create mode 100644 reader/src/test/java/com/maxmind/db/PositionByteReader.java create mode 100644 reader/src/test/java/com/maxmind/db/PositionedByteReaderTest.java diff --git a/reader/src/main/java/com/maxmind/db/ByteBufferByteReader.java b/reader/src/main/java/com/maxmind/db/ByteBufferByteReader.java new file mode 100644 index 00000000..4d451bdc --- /dev/null +++ b/reader/src/main/java/com/maxmind/db/ByteBufferByteReader.java @@ -0,0 +1,60 @@ +package com.maxmind.db; + +import java.nio.ByteBuffer; + +/** + * A {@link ByteReader} implemented by a {@link ByteBuffer}. + */ +public class ByteBufferByteReader extends IntLimitedByteReader { + protected final ByteBuffer bytes; + + /** + * Create a {@link ByteReader} from a {@link ByteBuffer}. + */ + public ByteBufferByteReader(final ByteBuffer bytes) { + this.bytes = bytes; + } + + @Override + public byte[] getBytes(final int length) { + final byte[] dst = new byte[length]; + bytes.get(dst); + return dst; + } + + @Override + public byte get() { + return bytes.get(); + } + + @Override + public byte get(long index) { + return super.get(index); + } + + @Override + protected byte get(final int index) { + return bytes.get(index); + } + + @Override + public long capacity() { + return bytes.capacity(); + } + + @Override + protected ByteReader position(final int newPosition) { + bytes.position(newPosition); + return this; + } + + @Override + public long position() { + return bytes.position(); + } + + @Override + public ByteReader duplicate() { + return new ByteBufferByteReader(bytes.duplicate()); + } +} diff --git a/reader/src/main/java/com/maxmind/db/ByteReader.java b/reader/src/main/java/com/maxmind/db/ByteReader.java new file mode 100644 index 00000000..74984a0a --- /dev/null +++ b/reader/src/main/java/com/maxmind/db/ByteReader.java @@ -0,0 +1,57 @@ +package com.maxmind.db; + +import java.nio.ByteBuffer; + +/** + *

This provides functionality from {@link ByteBuffer} that is required for this library. + * The default implementation is {@link ByteBufferByteReader}, which just redirects method calls + * to a {@link ByteBuffer}.Another implementation might be preferable. In that case, you can pass + * it to {@link Reader#Reader(ByteReader, String, NodeCache)}.

+ * + *

Methods in implementations should throw the same exceptions as the methods with the same name + * in {@link ByteBuffer}.

+ * + *

If your implementation can only handle positions of up to Integer.MAX_VALUE, + * implement {@link IntLimitedByteReader}.

+ */ +public interface ByteReader { + + /** + * See {@link ByteBuffer#position()} + */ + long position(); + + /** + * See {@link ByteBuffer#position(int)} + */ + ByteReader position(final long newPosition); + + /** + * See {@link ByteBuffer#capacity()}. + */ + long capacity(); + + /** + * Get the byte at {@link ByteReader#position()}. See {@link ByteBuffer#get()} + */ + byte get(); + + /** + * Get the byte at the given index. See {@link ByteBuffer#get(int)} + */ + byte get(final long index); + + /** + * Read the requested number of bytes starting at {@link ByteReader#position()}. + * It moves {@link ByteReader#position()} forward by the same amount. + */ + byte[] getBytes(final int length); + + /** + * This is intended for creating a copy of this collection that is safe + * for use by a new thread.

+ * + * See {@link java.nio.ByteBuffer#duplicate()} + */ + ByteReader duplicate(); +} diff --git a/reader/src/main/java/com/maxmind/db/BufferHolder.java b/reader/src/main/java/com/maxmind/db/ByteReaderHolder.java similarity index 74% rename from reader/src/main/java/com/maxmind/db/BufferHolder.java rename to reader/src/main/java/com/maxmind/db/ByteReaderHolder.java index dbbe8aa5..1b461ba7 100644 --- a/reader/src/main/java/com/maxmind/db/BufferHolder.java +++ b/reader/src/main/java/com/maxmind/db/ByteReaderHolder.java @@ -10,11 +10,15 @@ import java.nio.channels.FileChannel; import java.nio.channels.FileChannel.MapMode; -final class BufferHolder { +final class ByteReaderHolder { // DO NOT PASS OUTSIDE THIS CLASS. Doing so will remove thread safety. - private final ByteBuffer buffer; + private final ByteReader reader; - BufferHolder(File database, FileMode mode) throws IOException { + ByteReaderHolder(ByteReader reader) { + this.reader = reader; + } + + ByteReaderHolder(File database, FileMode mode) throws IOException { try ( final RandomAccessFile file = new RandomAccessFile(database, "r"); final FileChannel channel = file.getChannel() @@ -26,21 +30,23 @@ final class BufferHolder { + database.getName() + " into memory. Unexpected end of stream."); } - this.buffer = buf.asReadOnlyBuffer(); + this.reader = new ByteBufferByteReader(buf.asReadOnlyBuffer()); } else { - this.buffer = channel.map(MapMode.READ_ONLY, 0, channel.size()).asReadOnlyBuffer(); + final ByteBuffer mappedBuffer = + channel.map(MapMode.READ_ONLY, 0, channel.size()).asReadOnlyBuffer(); + this.reader = new ByteBufferByteReader(mappedBuffer); } } } /** - * Construct a ThreadBuffer from the provided URL. + * Construct a {@link ByteReaderHolder} from the provided {@link InputStream}. * * @param stream the source of my bytes. * @throws IOException if unable to read from your source. - * @throws NullPointerException if you provide a NULL InputStream + * @throws NullPointerException if you provide a {@code null} InputStream */ - BufferHolder(InputStream stream) throws IOException { + ByteReaderHolder(InputStream stream) throws IOException { if (null == stream) { throw new NullPointerException("Unable to use a NULL InputStream"); } @@ -50,14 +56,15 @@ final class BufferHolder { while (-1 != (br = stream.read(bytes))) { baos.write(bytes, 0, br); } - this.buffer = ByteBuffer.wrap(baos.toByteArray()).asReadOnlyBuffer(); + final ByteBuffer readOnlyBuffer = ByteBuffer.wrap(baos.toByteArray()).asReadOnlyBuffer(); + this.reader = new ByteBufferByteReader(readOnlyBuffer); } /* * Returns a duplicate of the underlying ByteBuffer. The returned ByteBuffer * should not be shared between threads. */ - ByteBuffer get() { + public ByteReader get() { // The Java API docs for buffer state: // // Buffers are not safe for use by multiple concurrent threads. If a buffer is to be @@ -75,6 +82,6 @@ ByteBuffer get() { // operations on the original buffer object, the risk of not synchronizing this call seems // relatively low and worth taking for the performance benefit when lookups are being done // from many threads. - return this.buffer.duplicate(); + return reader.duplicate(); } } diff --git a/reader/src/main/java/com/maxmind/db/CacheKey.java b/reader/src/main/java/com/maxmind/db/CacheKey.java index d62c0843..81f79514 100644 --- a/reader/src/main/java/com/maxmind/db/CacheKey.java +++ b/reader/src/main/java/com/maxmind/db/CacheKey.java @@ -8,17 +8,17 @@ * @param the type of value */ public final class CacheKey { - private final int offset; + private final long offset; private final Class cls; private final java.lang.reflect.Type type; - CacheKey(int offset, Class cls, java.lang.reflect.Type type) { + CacheKey(long offset, Class cls, java.lang.reflect.Type type) { this.offset = offset; this.cls = cls; this.type = type; } - int getOffset() { + long getOffset() { return this.offset; } @@ -58,7 +58,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - int result = offset; + int result = Long.hashCode(offset); result = 31 * result + (cls == null ? 0 : cls.hashCode()); result = 31 * result + (type == null ? 0 : type.hashCode()); return result; diff --git a/reader/src/main/java/com/maxmind/db/CtrlData.java b/reader/src/main/java/com/maxmind/db/CtrlData.java index c9bf03f1..af7062a2 100644 --- a/reader/src/main/java/com/maxmind/db/CtrlData.java +++ b/reader/src/main/java/com/maxmind/db/CtrlData.java @@ -3,10 +3,10 @@ final class CtrlData { private final Type type; private final int ctrlByte; - private final int offset; - private final int size; + private final long offset; + private final long size; - CtrlData(Type type, int ctrlByte, int offset, int size) { + CtrlData(Type type, int ctrlByte, long offset, long size) { this.type = type; this.ctrlByte = ctrlByte; this.offset = offset; @@ -21,11 +21,11 @@ public int getCtrlByte() { return this.ctrlByte; } - public int getOffset() { + public long getOffset() { return this.offset; } - public int getSize() { + public long getSize() { return this.size; } } diff --git a/reader/src/main/java/com/maxmind/db/Decoder.java b/reader/src/main/java/com/maxmind/db/Decoder.java index ab1bc5e2..56ce9c01 100644 --- a/reader/src/main/java/com/maxmind/db/Decoder.java +++ b/reader/src/main/java/com/maxmind/db/Decoder.java @@ -6,8 +6,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.ParameterizedType; import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; import java.nio.charset.StandardCharsets; @@ -38,11 +36,11 @@ final class Decoder { private final CharsetDecoder utfDecoder = UTF_8.newDecoder(); - private final ByteBuffer buffer; + private final ByteReader buffer; private final ConcurrentHashMap constructors; - Decoder(NodeCache cache, ByteBuffer buffer, long pointerBase) { + Decoder(NodeCache cache, ByteReader buffer, long pointerBase) { this( cache, buffer, @@ -53,7 +51,7 @@ final class Decoder { Decoder( NodeCache cache, - ByteBuffer buffer, + ByteReader buffer, long pointerBase, ConcurrentHashMap constructors ) { @@ -65,7 +63,7 @@ final class Decoder { private final NodeCache.Loader cacheLoader = this::decode; - public T decode(int offset, Class cls) throws IOException { + public T decode(long offset, Class cls) throws IOException { if (offset >= this.buffer.capacity()) { throw new InvalidDatabaseException( "The MaxMind DB file's data section contains bad data: " @@ -77,7 +75,7 @@ public T decode(int offset, Class cls) throws IOException { } private DecodedValue decode(CacheKey key) throws IOException { - int offset = key.getOffset(); + long offset = key.getOffset(); if (offset >= this.buffer.capacity()) { throw new InvalidDatabaseException( "The MaxMind DB file's data section contains bad data: " @@ -110,7 +108,7 @@ private DecodedValue decode(Class cls, java.lang.reflect.Type genericType } int targetOffset = (int) pointer; - int position = buffer.position(); + long position = buffer.position(); CacheKey key = new CacheKey(targetOffset, cls, genericType); DecodedValue o = cache.get(key, cacheLoader); @@ -195,12 +193,8 @@ private Object decodeByType( } } - private String decodeString(int size) throws CharacterCodingException { - int oldLimit = buffer.limit(); - buffer.limit(buffer.position() + size); - String s = utfDecoder.decode(buffer).toString(); - buffer.limit(oldLimit); - return s; + private String decodeString(int size) { + return new String(buffer.getBytes(size), UTF_8); } private int decodeUint16(int size) { @@ -231,7 +225,7 @@ private int decodeInteger(int base, int size) { return Decoder.decodeInteger(this.buffer, base, size); } - static int decodeInteger(ByteBuffer buffer, int base, int size) { + static int decodeInteger(ByteReader buffer, int base, int size) { int integer = base; for (int i = 0; i < size; i++) { integer = (integer << 8) | (buffer.get() & 0xFF); @@ -250,7 +244,7 @@ private double decodeDouble(int size) throws InvalidDatabaseException { "The MaxMind DB file's data section contains bad data: " + "invalid size of double."); } - return this.buffer.getDouble(); + return Double.longBitsToDouble(decodeLong(size)); } private float decodeFloat(int size) throws InvalidDatabaseException { @@ -259,7 +253,7 @@ private float decodeFloat(int size) throws InvalidDatabaseException { "The MaxMind DB file's data section contains bad data: " + "invalid size of float."); } - return this.buffer.getFloat(); + return Float.intBitsToFloat(decodeInteger(size)); } private static boolean decodeBoolean(int size) @@ -376,7 +370,7 @@ private Map decodeMapIntoMap( map.put(key, valueClass.cast(value)); } catch (ClassCastException e) { throw new DeserializationException( - "Error creating map entry for '" + key + "': " + e.getMessage(), e); + "Error creating map entry for '" + key + "': " + e.getMessage(), e); } } @@ -426,7 +420,7 @@ private Object decodeMapIntoObject(int size, Class cls) Integer parameterIndex = parameterIndexes.get(key); if (parameterIndex == null) { - int offset = this.nextValueOffset(this.buffer.position(), 1); + long offset = this.nextValueOffset(this.buffer.position(), 1); this.buffer.position(offset); continue; } @@ -492,7 +486,7 @@ private static String getParameterName( + " is not annotated with MaxMindDbParameter."); } - private int nextValueOffset(int offset, int numberToSkip) + private long nextValueOffset(long offset, long numberToSkip) throws InvalidDatabaseException { if (numberToSkip == 0) { return offset; @@ -500,7 +494,7 @@ private int nextValueOffset(int offset, int numberToSkip) CtrlData ctrlData = this.getCtrlData(offset); int ctrlByte = ctrlData.getCtrlByte(); - int size = ctrlData.getSize(); + long size = ctrlData.getSize(); offset = ctrlData.getOffset(); Type type = ctrlData.getType(); @@ -525,7 +519,7 @@ private int nextValueOffset(int offset, int numberToSkip) return nextValueOffset(offset, numberToSkip - 1); } - private CtrlData getCtrlData(int offset) + private CtrlData getCtrlData(long offset) throws InvalidDatabaseException { if (offset >= this.buffer.capacity()) { throw new InvalidDatabaseException( @@ -575,12 +569,6 @@ private CtrlData getCtrlData(int offset) } private byte[] getByteArray(int length) { - return Decoder.getByteArray(this.buffer, length); - } - - private static byte[] getByteArray(ByteBuffer buffer, int length) { - byte[] bytes = new byte[length]; - buffer.get(bytes); - return bytes; + return buffer.getBytes(length); } } diff --git a/reader/src/main/java/com/maxmind/db/IntLimitedByteReader.java b/reader/src/main/java/com/maxmind/db/IntLimitedByteReader.java new file mode 100644 index 00000000..c910cda2 --- /dev/null +++ b/reader/src/main/java/com/maxmind/db/IntLimitedByteReader.java @@ -0,0 +1,34 @@ +package com.maxmind.db; + +/** + * This provides safe functionality for {@link ByteReader}s that only can + * handle up to Integer.MAX_VALUE positions. + */ +public abstract class IntLimitedByteReader implements ByteReader { + protected abstract ByteReader position(final int index); + + @Override + public ByteReader position(long newPosition) { + final boolean isLarge = newPosition > Integer.MAX_VALUE; + if (isLarge || newPosition < Integer.MIN_VALUE) { + throw PositionedByteReader.createPositionException(capacity(), newPosition, isLarge); + } + return position((int) newPosition); + } + + protected abstract byte get(final int index); + + @Override + public byte get(final long index) { + /* + bytes.get(int) will do a range check for [0,limit), + but before that, we need to make sure that casting won't + do anything unexpected, like making a negative long + cast to a positive int. + */ + if (index > Integer.MAX_VALUE || index < Integer.MIN_VALUE) { + throw new IndexOutOfBoundsException(); + } + return get((int) index); + } +} diff --git a/reader/src/main/java/com/maxmind/db/Networks.java b/reader/src/main/java/com/maxmind/db/Networks.java index b06dfffb..0439f2af 100644 --- a/reader/src/main/java/com/maxmind/db/Networks.java +++ b/reader/src/main/java/com/maxmind/db/Networks.java @@ -19,9 +19,9 @@ public final class Networks implements Iterator> { private final Stack nodes; private NetworkNode lastNode; private final boolean includeAliasedNetworks; - private final ByteBuffer buffer; /* Stores the buffer for Next() calls */ + private final ByteReader buffer; /* Stores the buffer for Next() calls */ private final Class typeParameterClass; - + /** * Constructs a Networks instance. * diff --git a/reader/src/main/java/com/maxmind/db/PositionedByteReader.java b/reader/src/main/java/com/maxmind/db/PositionedByteReader.java new file mode 100644 index 00000000..27f1fb87 --- /dev/null +++ b/reader/src/main/java/com/maxmind/db/PositionedByteReader.java @@ -0,0 +1,145 @@ +package com.maxmind.db; + +import java.nio.BufferUnderflowException; + +/** + * This is a {@link ByteReader} that tracks its position. For instance, + * {@link java.nio.ByteBuffer}s track their position, so when using that, this class isn't + * necessary. If implemented with something like an array, which doesn't have its own pointer, + * this class is helpful. + */ +public abstract class PositionedByteReader implements ByteReader { + // Copied from java.nio.Buffer + /** + * {@link java.nio.ByteBuffer#createCapacityException(int)} + */ + static IllegalArgumentException createCapacityException(long capacity) { + assert capacity < 0 : "capacity expected to be negative"; + return new IllegalArgumentException("capacity < 0: (" + + capacity + " < 0)"); + } + + // copied from Buffer.java + /** + * {@link java.nio.Buffer#createPositionException(int)} + */ + static IllegalArgumentException createPositionException( + final long capacity, + final long newPosition, + final boolean isLarge + ) { + String msg = null; + + if (isLarge) { + msg = "newPosition > limit: (" + newPosition + " > " + capacity + ")"; + } else { // assume negative + assert newPosition < 0 : "newPosition expected to be negative"; + msg = "newPosition < 0: (" + newPosition + " < 0)"; + } + + return new IllegalArgumentException(msg); + } + + protected long position; + protected final long capacity; + + protected PositionedByteReader( + final long position, + final long capacity + ) { + if (capacity < 0) { + throw createCapacityException(capacity); + } + this.position = position; + this.capacity = capacity; + } + + protected PositionedByteReader(final long capacity) { + this(0, capacity); + } + + // copied from Buffer.java + /** + * {@link java.nio.ByteBuffer#nextGetIndex(int)} + */ + protected long nextGetIndex(long nb) { // package-private + long p = position(); + if (capacity() - p < nb) { + throw new BufferUnderflowException(); + } + position = p + nb; + return p; + } + + // copied from Buffer.java + /** + * {@link java.nio.Buffer#checkIndex(int)} + */ + protected long checkIndex(long i) { // package-private + if ((i < 0) || (i >= capacity())) { + throw new IndexOutOfBoundsException(); + } + return i; + } + + /** + * See {@link java.nio.ByteBuffer#position(int)} + */ + @Override + public ByteReader position(final long newPosition) { + final boolean isLarge = newPosition >= capacity(); + if (isLarge || newPosition < 0) { + throw createPositionException(capacity, newPosition, isLarge); + } + this.position = newPosition; + return this; + } + + @Override + public long position() { + return position; + } + + @Override + public long capacity() { + return capacity; + } + + /** + * Get the byte with the assumption that the index is valid. + */ + protected abstract byte getUnsafe(long index); + + /** + * Get the byte at {@link ByteReader#position()}. See {@link java.nio.ByteBuffer#get()} + */ + @Override + public byte get() { + return getUnsafe(nextGetIndex(1)); + } + + /** + * Get the byte at the given index. See {@link java.nio.ByteBuffer#get(int)} + */ + @Override + public byte get(long index) { + return getUnsafe(checkIndex(index)); + } + + /** + * Get the bytes with the assumption that the bounds checks already passed. + */ + protected abstract byte[] getBytesUnsafe( + final long position, + final int length + ); + + /** + * Read the requested number of bytes starting at {@link ByteReader#position()}. + * It moves {@link ByteReader#position()} forward by the same amount. + */ + @Override + public byte[] getBytes(final int length) { + return getBytesUnsafe(nextGetIndex(length), length); + } +} diff --git a/reader/src/main/java/com/maxmind/db/PositionedIntLimitedByteReader.java b/reader/src/main/java/com/maxmind/db/PositionedIntLimitedByteReader.java new file mode 100644 index 00000000..b61ca90a --- /dev/null +++ b/reader/src/main/java/com/maxmind/db/PositionedIntLimitedByteReader.java @@ -0,0 +1,109 @@ +package com.maxmind.db; + +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; + +/** + * This is a combination of {@link PositionedByteReader} and {@link IntLimitedByteReader}. + */ +public abstract class PositionedIntLimitedByteReader extends IntLimitedByteReader { + protected int position; + protected final int capacity; + + protected PositionedIntLimitedByteReader( + final int position, + final int capacity + ) { + if (capacity < 0) { + throw PositionedByteReader.createCapacityException(capacity); + } + this.position = position; + this.capacity = capacity; + } + + protected PositionedIntLimitedByteReader(final int capacity) { + this(0, capacity); + } + + // copied from Buffer.java + /** + * {@link ByteBuffer#nextGetIndex(int)} + */ + protected int nextGetIndex(long nb) { // package-private + int p = position; + if (capacity - p < nb) { + throw new BufferUnderflowException(); + } + position = p + (int) nb; + return p; + } + + // copied from Buffer.java + /** + * {@link java.nio.Buffer#checkIndex(int)} + */ + protected int checkIndex(long i) { // package-private + if ((i < 0) || (i >= capacity)) { + throw new IndexOutOfBoundsException(); + } + return (int) i; + } + + /** + * See {@link ByteBuffer#position(int)} + */ + @Override + protected ByteReader position(final int newPosition) { + final boolean isLarge = newPosition >= capacity; + if (isLarge || newPosition < 0) { + throw PositionedByteReader.createPositionException(capacity, newPosition, isLarge); + } + this.position = newPosition; + return this; + } + + @Override + public long position() { + return position; + } + + @Override + public long capacity() { + return capacity; + } + + /** + * Get the byte with the assumption that the index is valid. + */ + protected abstract byte getUnsafe(int index); + + /** + * Get the byte at {@link ByteReader#position()}. See {@link ByteBuffer#get()}. + */ + public byte get() { + return getUnsafe(nextGetIndex(1)); + } + + /** + * Get the byte at the given index. See {@link ByteBuffer#get(int)} + */ + public byte get(long index) { + return getUnsafe(checkIndex(index)); + } + + /** + * Get the bytes with the assumption that the bounds checks already passed. + */ + protected abstract byte[] getBytesUnsafe( + final int position, + final int length + ); + + /** + * Read the requested number of bytes starting at {@link ByteReader#position()}. + * It moves {@link ByteReader#position()} forward by the same amount. + */ + public byte[] getBytes(final int length) { + return getBytesUnsafe(nextGetIndex(length), length); + } +} diff --git a/reader/src/main/java/com/maxmind/db/Reader.java b/reader/src/main/java/com/maxmind/db/Reader.java index 1b8b84b2..e0e71405 100644 --- a/reader/src/main/java/com/maxmind/db/Reader.java +++ b/reader/src/main/java/com/maxmind/db/Reader.java @@ -7,7 +7,6 @@ import java.net.Inet6Address; import java.net.InetAddress; import java.net.UnknownHostException; -import java.nio.ByteBuffer; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; @@ -24,7 +23,7 @@ public final class Reader implements Closeable { private final int ipV4Start; private final Metadata metadata; - private final AtomicReference bufferHolderReference; + private volatile ByteReaderHolder bufferHolderReference; private final NodeCache cache; private final ConcurrentHashMap constructors; @@ -44,6 +43,13 @@ public enum FileMode { MEMORY } + /** + * Create a Reader using your own implementation of {@link ByteReader}. + */ + public Reader(final ByteReader reader, String name, NodeCache cache) throws IOException { + this(new ByteReaderHolder(reader), name, cache); + } + /** * Constructs a Reader for the MaxMind DB format, with no caching. The file * passed to it must be a valid MaxMind DB file such as a GeoIP2 database @@ -89,7 +95,7 @@ public Reader(InputStream source) throws IOException { * @throws IOException if there is an error reading from the Stream. */ public Reader(InputStream source, NodeCache cache) throws IOException { - this(new BufferHolder(source), "", cache); + this(new ByteReaderHolder(source), "", cache); } /** @@ -116,20 +122,23 @@ public Reader(File database, FileMode fileMode) throws IOException { * @throws IOException if there is an error opening or reading from the file. */ public Reader(File database, FileMode fileMode, NodeCache cache) throws IOException { - this(new BufferHolder(database, fileMode), database.getName(), cache); + this(new ByteReaderHolder(database, fileMode), database.getName(), cache); } - private Reader(BufferHolder bufferHolder, String name, NodeCache cache) throws IOException { - this.bufferHolderReference = new AtomicReference<>( - bufferHolder); + private Reader( + ByteReaderHolder byteReaderHolder, + String name, + NodeCache cache + ) throws IOException { + this.bufferHolderReference = byteReaderHolder; if (cache == null) { throw new NullPointerException("Cache cannot be null"); } this.cache = cache; - ByteBuffer buffer = bufferHolder.get(); - int start = this.findMetadataStart(buffer, name); + ByteReader buffer = byteReaderHolder.get(); + long start = this.findMetadataStart(buffer, name); Decoder metadataDecoder = new Decoder(this.cache, buffer, start); this.metadata = metadataDecoder.decode(start, Metadata.class); @@ -176,8 +185,8 @@ public DatabaseRecord getRecord(InetAddress ipAddress, Class cls) int pl = traverseResult[1]; int record = traverseResult[0]; - int nodeCount = this.metadata.getNodeCount(); - ByteBuffer buffer = this.getBufferHolder().get(); + long nodeCount = this.metadata.getNodeCount(); + ByteReader buffer = this.getBufferHolder().get(); T dataRecord = null; if (record > nodeCount) { // record is a data pointer @@ -205,7 +214,7 @@ int record = traverseResult[0]; * @throws ClosedDatabaseException Exception for a closed databased. * @throws InvalidDatabaseException Exception for an invalid database. */ - public Networks networks(Class typeParameterClass) throws + public Networks networks(Class typeParameterClass) throws InvalidNetworkException, ClosedDatabaseException, InvalidDatabaseException { return this.networks(false, typeParameterClass); } @@ -225,8 +234,8 @@ public Networks networks(Class typeParameterClass) throws * @throws InvalidDatabaseException Exception for an invalid database. */ public Networks networks( - boolean includeAliasedNetworks, - Class typeParameterClass) throws + boolean includeAliasedNetworks, + Class typeParameterClass) throws InvalidNetworkException, ClosedDatabaseException, InvalidDatabaseException { try { if (this.getMetadata().getIpVersion() == 6) { @@ -245,12 +254,12 @@ public Networks networks( } } - BufferHolder getBufferHolder() throws ClosedDatabaseException { - BufferHolder bufferHolder = this.bufferHolderReference.get(); - if (bufferHolder == null) { + ByteReaderHolder getBufferHolder() throws ClosedDatabaseException { + ByteReaderHolder byteReaderHolder = this.bufferHolderReference; + if (byteReaderHolder == null) { throw new ClosedDatabaseException(); } - return bufferHolder; + return byteReaderHolder; } private int startNode(int bitLength) { @@ -264,7 +273,7 @@ private int startNode(int bitLength) { return 0; } - private int findIpV4StartNode(ByteBuffer buffer) + private int findIpV4StartNode(ByteReader buffer) throws InvalidDatabaseException { if (this.metadata.getIpVersion() == 4) { return 0; @@ -294,9 +303,9 @@ private int findIpV4StartNode(ByteBuffer buffer) * @throws InvalidDatabaseException Exception for an invalid database. */ public Networks networksWithin( - Network network, - boolean includeAliasedNetworks, - Class typeParameterClass) + Network network, + boolean includeAliasedNetworks, + Class typeParameterClass) throws InvalidNetworkException, ClosedDatabaseException, InvalidDatabaseException { InetAddress networkAddress = network.getNetworkAddress(); if (this.metadata.getIpVersion() == 4 && networkAddress instanceof Inet6Address) { @@ -337,10 +346,10 @@ public Networks networksWithin( */ private int[] traverseTree(byte[] ip, int bitCount) throws ClosedDatabaseException, InvalidDatabaseException { - ByteBuffer buffer = this.getBufferHolder().get(); + ByteReader buffer = this.getBufferHolder().get(); int bitLength = ip.length * 8; int record = this.startNode(bitLength); - int nodeCount = this.metadata.getNodeCount(); + long nodeCount = this.metadata.getNodeCount(); int i = 0; for (; i < bitCount && record < nodeCount; i++) { @@ -355,11 +364,11 @@ record = this.readNode(buffer, record, bit); return new int[]{record, i}; } - int readNode(ByteBuffer buffer, int nodeNumber, int index) - throws InvalidDatabaseException { + int readNode(ByteReader buffer, int nodeNumber, int index) + throws InvalidDatabaseException { // index is the index of the record within the node, which // can either be 0 or 1. - int baseOffset = nodeNumber * this.metadata.getNodeByteSize(); + long baseOffset = ((long) nodeNumber) * this.metadata.getNodeByteSize(); switch (this.metadata.getRecordSize()) { case 24: @@ -389,11 +398,11 @@ int readNode(ByteBuffer buffer, int nodeNumber, int index) } T resolveDataPointer( - ByteBuffer buffer, + ByteReader buffer, int pointer, Class cls ) throws IOException { - int resolved = (pointer - this.metadata.getNodeCount()) + long resolved = (pointer - this.metadata.getNodeCount()) + this.metadata.getSearchTreeSize(); if (resolved >= buffer.capacity()) { @@ -421,9 +430,9 @@ T resolveDataPointer( * are much faster algorithms (e.g., Boyer-Moore) for this if speed is ever * an issue, but I suspect it won't be. */ - private int findMetadataStart(ByteBuffer buffer, String databaseName) + private long findMetadataStart(ByteReader buffer, String databaseName) throws InvalidDatabaseException { - int fileSize = buffer.capacity(); + long fileSize = buffer.capacity(); FILE: for (int i = 0; i < fileSize - METADATA_START_MARKER.length + 1; i++) { @@ -455,7 +464,7 @@ public Metadata getMetadata() { *

* If you are using FileMode.MEMORY_MAPPED, this will * not unmap the underlying file due to a limitation in Java's - * MappedByteBuffer. It will however set the reference to + * MappedBigByteBuffer. It will however set the reference to * the buffer to null, allowing the garbage collector to * collect it. *

@@ -464,6 +473,6 @@ public Metadata getMetadata() { */ @Override public void close() throws IOException { - this.bufferHolderReference.set(null); + this.bufferHolderReference = null; } } diff --git a/reader/src/test/java/com/maxmind/db/ByteReaderHolderTest.java b/reader/src/test/java/com/maxmind/db/ByteReaderHolderTest.java new file mode 100644 index 00000000..b46b1a67 --- /dev/null +++ b/reader/src/test/java/com/maxmind/db/ByteReaderHolderTest.java @@ -0,0 +1,22 @@ +package com.maxmind.db; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ByteReaderHolderTest { + + @Test + public void testThreadSafe() throws IOException { + final ByteReaderHolder holder = + new ByteReaderHolder( + new ByteArrayInputStream(new byte[] {0, 1}) + ); + final ByteReader byteReader0 = holder.get(); + byteReader0.position(1); + final ByteReader byteReader1 = holder.get(); + Assertions.assertNotEquals(byteReader0.position(), byteReader1.position()); + } + +} diff --git a/reader/src/test/java/com/maxmind/db/DecoderTest.java b/reader/src/test/java/com/maxmind/db/DecoderTest.java index 2668bd58..6a3f9181 100644 --- a/reader/src/test/java/com/maxmind/db/DecoderTest.java +++ b/reader/src/test/java/com/maxmind/db/DecoderTest.java @@ -407,8 +407,9 @@ public void testArrays() throws IOException { public void testInvalidControlByte() throws IOException { try (FileChannel fc = DecoderTest.getFileChannel(new byte[] {0x0, 0xF})) { MappedByteBuffer mmap = fc.map(MapMode.READ_ONLY, 0, fc.size()); + ByteReader byteReader = new ByteBufferByteReader(mmap); - Decoder decoder = new Decoder(new CHMCache(), mmap, 0); + Decoder decoder = new Decoder(new CHMCache(), byteReader, 0); InvalidDatabaseException ex = assertThrows( InvalidDatabaseException.class, () -> decoder.decode(0, String.class)); @@ -428,8 +429,9 @@ private static void testTypeDecoding(Type type, Map tests) String desc = "decoded " + type.name() + " - " + expect; try (FileChannel fc = DecoderTest.getFileChannel(input)) { MappedByteBuffer mmap = fc.map(MapMode.READ_ONLY, 0, fc.size()); + ByteReader byteReader = new ByteBufferByteReader(mmap); - Decoder decoder = new Decoder(cache, mmap, 0); + Decoder decoder = new Decoder(cache, byteReader, 0); decoder.pointerTestHack = true; // XXX - this could be streamlined diff --git a/reader/src/test/java/com/maxmind/db/PointerTest.java b/reader/src/test/java/com/maxmind/db/PointerTest.java index 1aa755f6..3e50901e 100644 --- a/reader/src/test/java/com/maxmind/db/PointerTest.java +++ b/reader/src/test/java/com/maxmind/db/PointerTest.java @@ -14,7 +14,7 @@ public class PointerTest { @Test public void testWithPointers() throws IOException { File file = ReaderTest.getFile("maps-with-pointers.raw"); - BufferHolder ptf = new BufferHolder(file, FileMode.MEMORY); + ByteReaderHolder ptf = new ByteReaderHolder(file, FileMode.MEMORY); Decoder decoder = new Decoder(NoCache.getInstance(), ptf.get(), 0); Map map = new HashMap<>(); diff --git a/reader/src/test/java/com/maxmind/db/PositionByteReader.java b/reader/src/test/java/com/maxmind/db/PositionByteReader.java new file mode 100644 index 00000000..3a92f38f --- /dev/null +++ b/reader/src/test/java/com/maxmind/db/PositionByteReader.java @@ -0,0 +1,33 @@ +package com.maxmind.db; + +import java.util.Arrays; + +/** + * This is just for testing. It returns the position as the byte value. + */ +public class PositionByteReader extends PositionedByteReader { + public PositionByteReader(long position, long capacity) { + super(position, capacity); + } + + public PositionByteReader(long capacity) { + super(capacity); + } + + @Override + protected byte getUnsafe(long index) { + return (byte) index; + } + + @Override + protected byte[] getBytesUnsafe(final long position, final int length) { + final byte[] bytes = new byte[length]; + Arrays.fill(bytes, (byte) position); + return bytes; + } + + @Override + public ByteReader duplicate() { + return new PositionByteReader(position, capacity); + } +} diff --git a/reader/src/test/java/com/maxmind/db/PositionedByteReaderTest.java b/reader/src/test/java/com/maxmind/db/PositionedByteReaderTest.java new file mode 100644 index 00000000..488b4dca --- /dev/null +++ b/reader/src/test/java/com/maxmind/db/PositionedByteReaderTest.java @@ -0,0 +1,33 @@ +package com.maxmind.db; + +import java.util.Arrays; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class PositionedByteReaderTest { + + @Test + public void position() { + final PositionedByteReader r = new PositionByteReader(10); + Assertions.assertEquals(0L, r.position()); + Assertions.assertSame(r, r.position(1)); + Assertions.assertEquals(1L, r.position()); + Assertions.assertThrows(IllegalArgumentException.class, () -> r.position(r.capacity())); + Assertions.assertThrows(IllegalArgumentException.class, () -> r.position(-1)); + final byte[] expectedPositionBytes = new byte[3]; + Arrays.fill(expectedPositionBytes, (byte) 1); + Assertions.assertArrayEquals( + expectedPositionBytes, + r.getBytes(expectedPositionBytes.length) + ); + } + + @Test + public void capacity() { + final long capacity = 0; + final PositionedByteReader r = new PositionByteReader(capacity); + Assertions.assertEquals(capacity, r.capacity()); + Assertions.assertThrows(IllegalArgumentException.class, () -> new PositionByteReader(-1)); + } + +} diff --git a/reader/src/test/java/com/maxmind/db/ReaderTest.java b/reader/src/test/java/com/maxmind/db/ReaderTest.java index 004ecb24..5f1642ba 100644 --- a/reader/src/test/java/com/maxmind/db/ReaderTest.java +++ b/reader/src/test/java/com/maxmind/db/ReaderTest.java @@ -94,7 +94,7 @@ public void testNetworks() throws IOException, InvalidDatabaseException, Invalid InetAddress actualIPInData = InetAddress.getByName(data.get("ip")); assertEquals( - iteration.getNetwork().getNetworkAddress(), + iteration.getNetwork().getNetworkAddress(), actualIPInData, "expected ip address" ); @@ -172,7 +172,7 @@ public networkTest(String network, int prefix,String database, String[] expecte } ), new networkTest( - "255.255.255.0", + "255.255.255.0", 24, "ipv4", new String[]{} @@ -238,7 +238,7 @@ public networkTest(String network, int prefix,String database, String[] expecte "1.1.1.4/30", "1.1.1.8/29", "1.1.1.16/28", - "1.1.1.32/32", + "1.1.1.32/32", } ), new networkTest( @@ -251,7 +251,7 @@ public networkTest(String network, int prefix,String database, String[] expecte "1.1.1.4/30", "1.1.1.8/29", "1.1.1.16/28", - "1.1.1.32/32", + "1.1.1.32/32", }, true ), @@ -937,8 +937,8 @@ public void testDecodeWrongTypeWithWrongArguments() throws IOException { public void testDecodeWithDataTypeMismatchInModel() throws IOException { this.testReader = new Reader(getFile("GeoIP2-City-Test.mmdb")); DeserializationException ex = assertThrows(DeserializationException.class, - () -> this.testReader.get(InetAddress.getByName("2.125.160.216"), - TestDataTypeMismatchInModel.class)); + () -> this.testReader.get(InetAddress.getByName("2.125.160.216"), + TestDataTypeMismatchInModel.class)); assertThat(ex.getMessage(), containsString("Error getting record for IP")); assertThat(ex.getMessage(), containsString("Error creating map entry for")); assertThat(ex.getCause().getCause().getClass(), equalTo(ClassCastException.class)); @@ -998,8 +998,8 @@ static class TestDataTypeMismatchInModel { @MaxMindDbConstructor public TestDataTypeMismatchInModel( - @MaxMindDbParameter(name = "location") - Map location + @MaxMindDbParameter(name = "location") + Map location ) { this.location = location; } From 9c99f862ad0762d9aad512ffc93c80e651bd681b Mon Sep 17 00:00:00 2001 From: Jeff Shaw Date: Tue, 11 Feb 2025 09:37:55 -0500 Subject: [PATCH 3/3] Remove `utfDecoder`, which was unused. --- reader/src/main/java/com/maxmind/db/Decoder.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/reader/src/main/java/com/maxmind/db/Decoder.java b/reader/src/main/java/com/maxmind/db/Decoder.java index 56ce9c01..83a3a35c 100644 --- a/reader/src/main/java/com/maxmind/db/Decoder.java +++ b/reader/src/main/java/com/maxmind/db/Decoder.java @@ -6,8 +6,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.ParameterizedType; import java.math.BigInteger; -import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; @@ -22,8 +20,6 @@ */ final class Decoder { - private static final Charset UTF_8 = StandardCharsets.UTF_8; - private static final int[] POINTER_VALUE_OFFSETS = {0, 0, 1 << 11, (1 << 19) + (1 << 11), 0}; // XXX - This is only for unit testings. We should possibly make a @@ -34,8 +30,6 @@ final class Decoder { private final long pointerBase; - private final CharsetDecoder utfDecoder = UTF_8.newDecoder(); - private final ByteReader buffer; private final ConcurrentHashMap constructors; @@ -194,7 +188,7 @@ private Object decodeByType( } private String decodeString(int size) { - return new String(buffer.getBytes(size), UTF_8); + return new String(buffer.getBytes(size), StandardCharsets.UTF_8); } private int decodeUint16(int size) {