diff --git a/src/main/java/com/maxmind/db/BadVersionException.java b/src/main/java/com/maxmind/db/BadVersionException.java new file mode 100644 index 00000000..f9e37379 --- /dev/null +++ b/src/main/java/com/maxmind/db/BadVersionException.java @@ -0,0 +1,9 @@ +package com.maxmind.db; + +import java.net.InetAddress; + +public class BadVersionException extends Exception { + public BadVersionException(InetAddress ip) { + super("you attempted to use an IPv6 network in an IPv4-only database: " + ip.toString()); + } +} diff --git a/src/main/java/com/maxmind/db/Networks.java b/src/main/java/com/maxmind/db/Networks.java new file mode 100644 index 00000000..fd87d627 --- /dev/null +++ b/src/main/java/com/maxmind/db/Networks.java @@ -0,0 +1,196 @@ +package com.maxmind.db; + +import java.io.IOException; +import java.net.Inet4Address; +import java.net.InetAddress; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; + +public class Networks implements Iterator> { + private final Reader reader; + private ArrayList nodes; + private NetworkNode lastNode; + private boolean skipAliasedNetworks; + private Exception err; + private ByteBuffer buffer; /* Stores the buffer for Next() calls */ + private Class typeParameterClass; + + /** + * Constructs a Networks instance. + * @param reader The reader object. + * @param skipAliasedNetworks The boolean to skip aliased networks. + * @throws ClosedDatabaseException Exception for a closed database. + */ + Networks(Reader reader, boolean skipAliasedNetworks) + throws ClosedDatabaseException { + this(reader, skipAliasedNetworks, new NetworkNode[]{}); + } + + /** + * Constructs a Networks instance. + * @param reader The reader object. + * @param skipAliasedNetworks The boolean to skip aliased networks. + * @param nodes The initial nodes array to start Networks iterator with. + * @throws ClosedDatabaseException Exception for a closed database. + */ + Networks(Reader reader, boolean skipAliasedNetworks, NetworkNode[] nodes) + throws ClosedDatabaseException { + this.reader = reader; + this.skipAliasedNetworks = skipAliasedNetworks; + this.nodes = new ArrayList(Arrays.asList(nodes)); + this.buffer = reader.getBufferHolder().get(); + } + + /** + * Constructs a Networks instance with skipAliasedNetworks set to false by default. + * @param reader The reader object. + */ + Networks(Reader reader) throws ClosedDatabaseException { + this(reader, false); + } + + /** + * Returns if Networks had any errors. + * @return Exception The exception to the Networks iteration. + */ + public Exception getErr() { + return this.err; + } + + /** + * Sets the Class for the data type in DataRecord. + * @param cls The class object. ( For example, Map.class ) + */ + public void setDataClass(Class cls) { + this.typeParameterClass = cls; + } + + /** + * Returns the next NetworksItem. You need to set the class using + * prepareForClass before calling next. + * For example, + * networks.prepareForClass(Map.Class); + * Map test = networks.next(); + */ + @Override + public DatabaseRecord next() { + if (this.err != null) { + return null; + } + + try { + T data = this.reader.resolveDataPointer( + this.buffer, this.lastNode.pointer, this.typeParameterClass); + + byte[] ip = this.lastNode.ip; + int prefixLength = this.lastNode.prefix; + + // We do this because uses of SkipAliasedNetworks expect the IPv4 networks + // to be returned as IPv4 networks. If we are not skipping aliased + // networks, then the user will get IPv4 networks from the ::FFFF:0:0/96 + // network. + if (this.skipAliasedNetworks && isInIpv4Subtree(ip)) { + ip = Arrays.copyOfRange(ip, 12, ip.length); + prefixLength -= 96; + } + + // If the ip is in ipv6 form, drop the prefix manually + // as InetAddress converts it to ipv4. + InetAddress ipAddr = InetAddress.getByAddress(ip); + if (ipAddr instanceof Inet4Address && ip.length > 4 + && ip[10] == -1 && ip[11] == -1 && prefixLength > 32) { + prefixLength -= 96; + } + + return new DatabaseRecord(data, InetAddress.getByAddress(ip), prefixLength); + } catch (IOException e) { + this.err = e; + return null; + } + } + + public boolean isInIpv4Subtree(byte[] ip) { + if (ip.length != 16) { + return false; + } + for (int i = 0; i < 12; i++) { + if (ip[i] != 0) { + return false; + } + } + return true; + } + + /* + * Next prepares the next network for reading with the Network method. It + * returns true if there is another network to be processed and false if there + * are no more networks or if there is an error. + */ + @Override + public boolean hasNext() { + if (this.err != null) { + return false; + } + while (!this.nodes.isEmpty()) { + // Pop the last one. + NetworkNode node = this.nodes.remove(this.nodes.size() - 1); + + // Next until we don't have data. + while (node.pointer != this.reader.getMetadata().getNodeCount()) { + // This skips IPv4 aliases without hardcoding the networks that the writer + // currently aliases. + if (this.skipAliasedNetworks && this.reader.getIpv4Start() != 0 + && node.pointer == this.reader.getIpv4Start() + && !isInIpv4Subtree(node.ip)) { + break; + } + + if (node.pointer > this.reader.getMetadata().getNodeCount()) { + this.lastNode = node; + return true; + } + + byte[] ipRight = Arrays.copyOf(node.ip, node.ip.length); + if (ipRight.length <= (node.prefix >> 3)) { + this.err = new InvalidDatabaseException("Invalid search tree"); + return false; + } + + ipRight[node.prefix >> 3] |= 1 << (7 - (node.prefix % 8)); + + try { + int rightPointer = this.reader.readNode(this.buffer, node.pointer, 1); + node.prefix++; + + this.nodes.add(new NetworkNode(ipRight, node.prefix, rightPointer)); + node.pointer = this.reader.readNode(this.buffer, node.pointer, 0); + } catch (InvalidDatabaseException e) { + this.err = e; + return false; + } + } + } + return false; + } + + protected static class NetworkNode { + public byte[] ip; + public int prefix; + public int pointer; + + /** + * Constructs a network node for internal use. + * @param ip The ip address of the node. + * @param prefix The prefix of the node. + * @param pointer The node number + */ + public NetworkNode(byte[] ip, int prefix, int pointer) { + this.ip = ip; + this.prefix = prefix; + this.pointer = pointer; + } + } + +} diff --git a/src/main/java/com/maxmind/db/Reader.java b/src/main/java/com/maxmind/db/Reader.java index aa3330f6..86836c20 100644 --- a/src/main/java/com/maxmind/db/Reader.java +++ b/src/main/java/com/maxmind/db/Reader.java @@ -4,7 +4,9 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.net.Inet6Address; import java.net.InetAddress; +import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; @@ -14,6 +16,8 @@ * addresses can be looked up using the get method. */ public final class Reader implements Closeable { + private static final int IPV4_LEN = 4; + private static final int IPV6_LEN = 6; private static final int DATA_SECTION_SEPARATOR_SIZE = 16; private static final byte[] METADATA_START_MARKER = {(byte) 0xAB, (byte) 0xCD, (byte) 0xEF, 'M', 'a', 'x', 'M', 'i', 'n', 'd', '.', @@ -149,6 +153,10 @@ public T get(InetAddress ipAddress, Class cls) throws IOException { return getRecord(ipAddress, cls).getData(); } + protected int getIpv4Start() { + return this.ipV4Start; + } + /** * Looks up ipAddress in the MaxMind DB. * @@ -190,7 +198,56 @@ record = this.readNode(buffer, record, bit); return new DatabaseRecord<>(dataRecord, ipAddress, pl); } - private BufferHolder getBufferHolder() throws ClosedDatabaseException { + /** + * Creates a Networks iterator and skips aliased networks. + * Please note that a MaxMind DB may map IPv4 networks into several locations + * in an IPv6 database. This iterator will iterate over all of these locations + * separately. To only iterate over the IPv4 networks once, use the + * SkipAliasedNetworks option. + * + * @return Networks The Networks iterator. + * @throws BadVersionException Exception for using an IPv6 network in ipv4-only database. + * @throws ClosedDatabaseException Exception for a closed databased. + * @throws InvalidDatabaseException Exception for an invalid database. + */ + public Networks networks() throws + BadVersionException, ClosedDatabaseException, InvalidDatabaseException { + return this.networks(true); + } + + /** + * Creates a Networks iterator. + * Please note that a MaxMind DB may map IPv4 networks into several locations + * in an IPv6 database. This iterator will iterate over all of these locations + * separately. To only iterate over the IPv4 networks once, use the + * SkipAliasedNetworks option. + * + * @param skipAliasedNetworks Enable skipping aliased networks. + * @return Networks The Networks iterator. + * @throws BadVersionException Exception for using an IPv6 network in ipv4-only database. + * @throws ClosedDatabaseException Exception for a closed databased. + * @throws InvalidDatabaseException Exception for an invalid database. + */ + public Networks networks(boolean skipAliasedNetworks) throws + BadVersionException, ClosedDatabaseException, InvalidDatabaseException { + try { + InetAddress ipv4 = InetAddress.getByAddress(new byte[4]); + InetAddress ipv6 = InetAddress.getByAddress(new byte[16]); + Network ipAllV4 = new Network(ipv4, 0); // Mask 32. + Network ipAllV6 = new Network(ipv6, 0); // Mask 128. + + if (this.getMetadata().getIpVersion() == 6) { + return this.networksWithIn(ipAllV6, skipAliasedNetworks); + } + return this.networksWithIn(ipAllV4, skipAliasedNetworks); + } catch (UnknownHostException e) { + /* This is returned by getByAddress. This should never happen + as the ipv4 and ipv6 are constants set by us. */ + return null; + } + } + + protected BufferHolder getBufferHolder() throws ClosedDatabaseException { BufferHolder bufferHolder = this.bufferHolderReference.get(); if (bufferHolder == null) { throw new ClosedDatabaseException(); @@ -222,20 +279,98 @@ private int findIpV4StartNode(ByteBuffer buffer) return node; } - private int readNode(ByteBuffer buffer, int nodeNumber, int index) - throws InvalidDatabaseException { + /** + * Returns an iterator within the specified network. + * Please note that a MaxMind DB may map IPv4 networks into several locations + * in an IPv6 database. This iterator will iterate over all of these locations + * separately. To only iterate over the IPv4 networks once, use the + * SkipAliasedNetworks option. + * @param Represents the data type(e.g., Map, HastMap, etc.). + * @param network Specifies the network to be iterated. + * @param skipAliasedNetworks Boolean for skipping aliased networks. + * @return Networks + * @throws BadVersionException Exception for using an IPv6 network in ipv4-only database. + * @throws ClosedDatabaseException Exception for a closed databased. + * @throws InvalidDatabaseException Exception for an invalid database. + */ + public Networks networksWithIn(Network network, boolean skipAliasedNetworks) + throws BadVersionException, ClosedDatabaseException, InvalidDatabaseException { + InetAddress networkAddress = network.getNetworkAddress(); + if (this.metadata.getIpVersion() == 4 && networkAddress instanceof Inet6Address) { + throw new BadVersionException(networkAddress); + } + + byte[] ipBytes = networkAddress.getAddress(); + int prefixLength = network.getPrefixLength(); + + if (this.metadata.getIpVersion() == 6 && ipBytes.length == IPV4_LEN) { + if (skipAliasedNetworks) { + ipBytes = new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ipBytes[0], ipBytes[1], ipBytes[2], ipBytes[3] }; + } else { + // Convert it to the IP address (in 16-byte from) of the IPv4 address. + ipBytes = new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + -1, -1, // -1 is for 0xff. + ipBytes[0], ipBytes[1], ipBytes[2], ipBytes[3]}; + } + prefixLength += 96; + } + + int[] traverseResult = this.traverseTree(ipBytes, 0, prefixLength); + int node = traverseResult[0]; + int prefix = traverseResult[1]; + + Networks networks = new Networks(this, skipAliasedNetworks, + new Networks.NetworkNode[]{ new Networks.NetworkNode(ipBytes, prefix, node) }); + + return networks; + } + + /** + * Returns the node number and the prefix for the network. + * @param ip The ip address to travese. + * @param node The number of the node. + * @param bitCount The prefix. + * @return int[] + */ + public int[] traverseTree(byte[] ip, int node, int bitCount) + throws ClosedDatabaseException, InvalidDatabaseException { + int nodeCount = this.metadata.getNodeCount(); + int i = 0; + + ByteBuffer buffer = this.getBufferHolder().get(); + + for (; i < bitCount && node < nodeCount; i++) { + int bit = 1 & (ip[i >> 3] >> (7 - (i % 8))); + + // bit:0 -> left record. + // bit:1 -> right record. + node = this.readNode(buffer, node, bit); + } + + return new int[]{node, i}; + } + + protected int readNode(ByteBuffer buffer, int nodeNumber, int index) + throws InvalidDatabaseException { + // index is the index of the record within the node, which + // can either be 0 or 1. int baseOffset = nodeNumber * this.metadata.getNodeByteSize(); switch (this.metadata.getRecordSize()) { case 24: + // For a 24 bit record, each record is 3 bytes. buffer.position(baseOffset + index * 3); return Decoder.decodeInteger(buffer, 0, 3); case 28: int middle = buffer.get(baseOffset + 3); if (index == 0) { + // We get the most significant from the first half + // of the byte. It belongs to the first record. middle = (0xF0 & middle) >>> 4; } else { + // We get the most significant byte of the second record. middle = 0x0F & middle; } buffer.position(baseOffset + index * 4); @@ -249,7 +384,7 @@ private int readNode(ByteBuffer buffer, int nodeNumber, int index) } } - private T resolveDataPointer( + protected T resolveDataPointer( ByteBuffer buffer, int pointer, Class cls diff --git a/src/test/java/com/maxmind/db/ReaderTest.java b/src/test/java/com/maxmind/db/ReaderTest.java index a3aae9b5..52a0823d 100644 --- a/src/test/java/com/maxmind/db/ReaderTest.java +++ b/src/test/java/com/maxmind/db/ReaderTest.java @@ -78,6 +78,320 @@ static class GetRecordTest { } } + @Test + public void testNetworks() throws IOException, InvalidDatabaseException, BadVersionException { + for (long recordSize : new long[] {24, 28, 32}) { + for (int ipVersion : new int[] {4, 6}) { + File file = getFile("MaxMind-DB-test-ipv" + ipVersion + "-" + recordSize + ".mmdb"); + + Reader reader = new Reader(file); + Networks networks = reader.networks(false); + networks.setDataClass(Map.class); // This is needed before running any next(). + + while(networks.hasNext()) { + DatabaseRecord> iteration = networks.next(); + Map data = iteration.getData(); + assertNull(networks.getErr()); + + InetAddress actualIPInData = InetAddress.getByName(data.get("ip")); + + assertEquals("expected ip address", + iteration.getNetwork().getNetworkAddress(), + actualIPInData); + } + + assertNull(networks.getErr()); + reader.close(); + } + } + } + + @Test + public void testNetworksWithInvalidSearchTree() throws IOException, BadVersionException{ + File file = getFile("MaxMind-DB-test-broken-search-tree-24.mmdb"); + Reader reader = new Reader(file); + + Networks networks = reader.networks(false); + networks.setDataClass(Map.class); + while(networks.hasNext()){ + DatabaseRecord iteration = networks.next(); + } + + assertTrue(networks.getErr() instanceof InvalidDatabaseException); + reader.close(); + } + + private class networkTest { + String network; + String database; + int prefix; + String[] expected; + boolean skipAliasedNetworks; + public networkTest(String network, int prefix,String database, String[] expected, boolean skipAliasedNetworks){ + this(network, prefix, database, expected); + this.skipAliasedNetworks = skipAliasedNetworks; + } + public networkTest(String network, int prefix,String database, String[] expected){ + this.network = network; + this.prefix = prefix; + this.database = database; + this.expected = expected; + } + } + + private networkTest[] tests = new networkTest[]{ + new networkTest( + "0.0.0.0", + 0, + "ipv4", + new String[]{ + "1.1.1.1/32", + "1.1.1.2/31", + "1.1.1.4/30", + "1.1.1.8/29", + "1.1.1.16/28", + "1.1.1.32/32", + } + ), + new networkTest( + "1.1.1.1", + 30, + "ipv4", + new String[]{ + "1.1.1.1/32", + "1.1.1.2/31", + } + ), + new networkTest( + "1.1.1.1", + 32, + "ipv4", + new String[]{ + "1.1.1.1/32", + } + ), + new networkTest( + "255.255.255.0", + 24, + "ipv4", + new String[]{} + ), + new networkTest( + "1.1.1.1", + 32, + "mixed", + new String[]{ + "1.1.1.1/32", + } + ), + new networkTest( + "255.255.255.0", + 24, + "mixed", + new String[]{} + ), + new networkTest( + "::1:ffff:ffff", + 128, + "ipv6", + new String[]{ + "0:0:0:0:0:1:ffff:ffff/128", + }, + true + ), + new networkTest( + "::", + 0, + "ipv6", + new String[]{ + "0:0:0:0:0:1:ffff:ffff/128", + "0:0:0:0:0:2:0:0/122", + "0:0:0:0:0:2:0:40/124", + "0:0:0:0:0:2:0:50/125", + "0:0:0:0:0:2:0:58/127", + } + ), + new networkTest( + "::2:0:40", + 123, + "ipv6", + new String[]{ + "0:0:0:0:0:2:0:40/124", + "0:0:0:0:0:2:0:50/125", + "0:0:0:0:0:2:0:58/127", + } + ), + new networkTest( + "0:0:0:0:0:ffff:ffff:ff00", + 120, + "ipv6", + new String[]{} + ), + new networkTest( + "0.0.0.0", + 0, + "mixed", + new String[]{ + "1.1.1.1/32", + "1.1.1.2/31", + "1.1.1.4/30", + "1.1.1.8/29", + "1.1.1.16/28", + "1.1.1.32/32", + } + ), + new networkTest( + "0.0.0.0", + 0, + "mixed", + new String[]{ + "1.1.1.1/32", + "1.1.1.2/31", + "1.1.1.4/30", + "1.1.1.8/29", + "1.1.1.16/28", + "1.1.1.32/32", + }, + true + ), + new networkTest( + "::", + 0, + "mixed", + new String[]{ + "0:0:0:0:0:0:101:101/128", + "0:0:0:0:0:0:101:102/127", + "0:0:0:0:0:0:101:104/126", + "0:0:0:0:0:0:101:108/125", + "0:0:0:0:0:0:101:110/124", + "0:0:0:0:0:0:101:120/128", + "0:0:0:0:0:1:ffff:ffff/128", + "0:0:0:0:0:2:0:0/122", + "0:0:0:0:0:2:0:40/124", + "0:0:0:0:0:2:0:50/125", + "0:0:0:0:0:2:0:58/127", + "1.1.1.1/32", + "1.1.1.2/31", + "1.1.1.4/30", + "1.1.1.8/29", + "1.1.1.16/28", + "1.1.1.32/32", + "2001:0:101:101:0:0:0:0/64", + "2001:0:101:102:0:0:0:0/63", + "2001:0:101:104:0:0:0:0/62", + "2001:0:101:108:0:0:0:0/61", + "2001:0:101:110:0:0:0:0/60", + "2001:0:101:120:0:0:0:0/64", + "2002:101:101:0:0:0:0:0/48", + "2002:101:102:0:0:0:0:0/47", + "2002:101:104:0:0:0:0:0/46", + "2002:101:108:0:0:0:0:0/45", + "2002:101:110:0:0:0:0:0/44", + "2002:101:120:0:0:0:0:0/48", + } + ), + new networkTest( + "::", + 0, + "mixed", + new String[]{ + "1.1.1.1/32", + "1.1.1.2/31", + "1.1.1.4/30", + "1.1.1.8/29", + "1.1.1.16/28", + "1.1.1.32/32", + "0:0:0:0:0:1:ffff:ffff/128", + "0:0:0:0:0:2:0:0/122", + "0:0:0:0:0:2:0:40/124", + "0:0:0:0:0:2:0:50/125", + "0:0:0:0:0:2:0:58/127", + }, + true + ), + new networkTest( + "1.1.1.16", + 28, + "mixed", + new String[]{ + "1.1.1.16/28" + } + ), + new networkTest( + "1.1.1.4", + 30, + "ipv4", + new String[]{ + "1.1.1.4/30" + } + ) + }; + + @Test + public void testNetworksWithin() throws IOException, BadVersionException{ + for(networkTest test : tests){ + for(int recordSize : new int[]{24, 28, 32}){ + File file = getFile("MaxMind-DB-test-"+test.database+"-"+recordSize+".mmdb"); + Reader reader = new Reader(file); + + InetAddress address = InetAddress.getByName(test.network); + Network network = new Network(address, test.prefix); + + Networks networks = reader.networksWithIn(network, test.skipAliasedNetworks); + networks.setDataClass(Map.class); + + ArrayList innerIPs = new ArrayList(); + while(networks.hasNext()){ + DatabaseRecord> iteration = networks.next(); + innerIPs.add(iteration.getNetwork().toString()); + } + + assertNull(networks.getErr()); + assertArrayEquals(test.expected, innerIPs.toArray()); + + reader.close(); + } + } + } + + private networkTest[] geoipTests = new networkTest[]{ + new networkTest( + "81.2.69.128", + 26, + "GeoIP2-Country-Test.mmdb", + new String[]{ + "81.2.69.142/31", + "81.2.69.144/28", + "81.2.69.160/27", + } + ) + }; + + @Test + public void testGeoIPNetworksWithin() throws IOException, BadVersionException{ + for (networkTest test : geoipTests){ + File file = getFile(test.database); + Reader reader = new Reader(file); + + InetAddress address = InetAddress.getByName(test.network); + Network network = new Network(address, test.prefix); + + Networks networks = reader.networksWithIn(network, test.skipAliasedNetworks); + networks.setDataClass(Map.class); + + ArrayList innerIPs = new ArrayList(); + while(networks.hasNext()){ + DatabaseRecord> iteration = networks.next(); + innerIPs.add(iteration.getNetwork().toString()); + } + + assertNull(networks.getErr()); + assertArrayEquals(test.expected, innerIPs.toArray()); + + reader.close(); + } + } + @Test public void testGetRecord() throws IOException { GetRecordTest[] mapTests = {