Add iteration mechanism

faktas2 · faktas2 · commit 9d31fc8f6691 · 2023-11-02T10:36:58.000-04:00
diff --git a/src/main/java/com/maxmind/db/BadVersionException.java b/src/main/java/com/maxmind/db/BadVersionException.java
@@ -0,0 +1,9 @@
+package com.maxmind.db;
+
+import java.net.InetAddress;
+
+public class BadVersionException extends Exception {
+    public BadVersionException(InetAddress ip) {
+        super("you attempted to use an IPv6 network in an IPv4-only database: " + ip.toString());
+    }
+}
diff --git a/src/main/java/com/maxmind/db/Networks.java b/src/main/java/com/maxmind/db/Networks.java
@@ -0,0 +1,196 @@
+package com.maxmind.db;
+
+import java.io.IOException;
+import java.net.Inet4Address;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+
+public class Networks<T> implements Iterator<DatabaseRecord<T>> {
+    private final Reader reader;
+    private ArrayList<NetworkNode> nodes;
+    private NetworkNode lastNode;
+    private boolean skipAliasedNetworks;
+    private Exception err;
+    private ByteBuffer buffer; /* Stores the buffer for Next() calls */
+    private Class<T> typeParameterClass;
+    
+    /**
+     * Constructs a Networks instance.
+     * @param reader The reader object.
+     * @param skipAliasedNetworks The boolean to skip aliased networks.
+     * @throws ClosedDatabaseException Exception for a closed database.
+     */
+    Networks(Reader reader, boolean skipAliasedNetworks) 
+        throws ClosedDatabaseException {
+        this(reader, skipAliasedNetworks, new NetworkNode[]{});
+    }
+
+    /**
+     * Constructs a Networks instance.
+     * @param reader The reader object.
+     * @param skipAliasedNetworks The boolean to skip aliased networks.
+     * @param nodes The initial nodes array to start Networks iterator with.
+     * @throws ClosedDatabaseException Exception for a closed database.
+     */
+    Networks(Reader reader, boolean skipAliasedNetworks, NetworkNode[] nodes) 
+        throws ClosedDatabaseException {
+        this.reader = reader;
+        this.skipAliasedNetworks = skipAliasedNetworks;
+        this.nodes = new ArrayList<NetworkNode>(Arrays.asList(nodes));
+        this.buffer = reader.getBufferHolder().get();
+    }
+
+    /**
+     * Constructs a Networks instance with skipAliasedNetworks set to false by default.
+     * @param reader The reader object.
+     */
+    Networks(Reader reader) throws ClosedDatabaseException {
+        this(reader, false);
+    }
+
+    /**
+     * Returns if Networks had any errors.
+     * @return Exception The exception to the Networks iteration.
+     */
+    public Exception getErr() {
+        return this.err;
+    }
+
+    /**
+     * Sets the Class for the data type in DataRecord.
+     * @param cls The class object. ( For example, Map.class )
+     */
+    public void setDataClass(Class<T> cls) {
+        this.typeParameterClass = cls;
+    }
+
+    /**
+     * Returns the next NetworksItem. You need to set the class using
+     * prepareForClass before calling next. 
+     * For example,
+     *  networks.prepareForClass(Map.Class);
+     *  Map test = networks.next();
+     */
+    @Override
+    public DatabaseRecord<T> next() {
+        if (this.err != null) {
+            return null;
+        } 
+
+        try {
+            T data = this.reader.resolveDataPointer(
+                this.buffer, this.lastNode.pointer, this.typeParameterClass);
+
+            byte[] ip = this.lastNode.ip;
+            int prefixLength = this.lastNode.prefix;
+
+            // We do this because uses of SkipAliasedNetworks expect the IPv4 networks
+            // to be returned as IPv4 networks. If we are not skipping aliased
+            // networks, then the user will get IPv4 networks from the ::FFFF:0:0/96
+            // network.
+            if (this.skipAliasedNetworks && isInIpv4Subtree(ip)) {
+                ip = Arrays.copyOfRange(ip, 12, ip.length);
+                prefixLength -= 96;
+            }
+
+            // If the ip is in ipv6 form, drop the prefix manually
+            // as InetAddress converts it to ipv4.
+            InetAddress ipAddr = InetAddress.getByAddress(ip);
+            if (ipAddr instanceof Inet4Address && ip.length > 4
+                    && ip[10] == -1 && ip[11] == -1 && prefixLength > 32) {
+                prefixLength -= 96;
+            }
+
+            return new DatabaseRecord<T>(data, InetAddress.getByAddress(ip), prefixLength);
+        } catch (IOException e) {
+            this.err = e;
+            return null;
+        }
+    }
+
+    public boolean isInIpv4Subtree(byte[] ip) {
+        if (ip.length != 16) {
+            return false;
+        }
+        for (int i = 0; i < 12; i++) {
+            if (ip[i] != 0) {
+                return false;
+            }
+        }
+        return true;
+    }
+    
+    /*
+    * Next prepares the next network for reading with the Network method. It
+    * returns true if there is another network to be processed and false if there
+    * are no more networks or if there is an error.
+    */
+    @Override
+    public boolean hasNext()  {
+        if (this.err != null) {
+            return false;
+        }
+        while (!this.nodes.isEmpty()) {
+            // Pop the last one.
+            NetworkNode node = this.nodes.remove(this.nodes.size() - 1);
+
+            // Next until we don't have data.
+            while (node.pointer != this.reader.getMetadata().getNodeCount()) {
+                // This skips IPv4 aliases without hardcoding the networks that the writer
+                // currently aliases.
+                if (this.skipAliasedNetworks && this.reader.getIpv4Start() != 0
+                        && node.pointer == this.reader.getIpv4Start()
+                        && !isInIpv4Subtree(node.ip)) {
+                    break;
+                }
+
+                if (node.pointer > this.reader.getMetadata().getNodeCount()) {
+                    this.lastNode = node;
+                    return true;
+                }
+
+                byte[] ipRight = Arrays.copyOf(node.ip, node.ip.length);
+                if (ipRight.length <= (node.prefix >> 3)) {
+                    this.err = new InvalidDatabaseException("Invalid search tree");
+                    return false;
+                }
+
+                ipRight[node.prefix >> 3] |= 1 << (7 - (node.prefix % 8));
+
+                try {
+                    int rightPointer = this.reader.readNode(this.buffer, node.pointer, 1);
+                    node.prefix++;
+
+                    this.nodes.add(new NetworkNode(ipRight, node.prefix, rightPointer));
+                    node.pointer = this.reader.readNode(this.buffer, node.pointer, 0);
+                } catch (InvalidDatabaseException e) {
+                    this.err = e;
+                    return false;
+                }
+            }
+        }
+        return false;
+    }
+
+    protected static class NetworkNode {
+        public byte[] ip;
+        public int prefix;
+        public int pointer;
+
+        /**
+         * Constructs a network node for internal use.
+         * @param ip The ip address of the node.
+         * @param prefix The prefix of the node.
+         * @param pointer The node number
+         */
+        public NetworkNode(byte[] ip, int prefix, int pointer) {
+            this.ip = ip;
+            this.prefix = prefix;
+            this.pointer = pointer;
+        }
+    }
+
+}
diff --git a/src/main/java/com/maxmind/db/Reader.java b/src/main/java/com/maxmind/db/Reader.java
@@ -4,7 +4,9 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.Inet6Address;
 import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicReference;
@@ -14,6 +16,8 @@
  * addresses can be looked up using the <code>get</code> method.
  */
 public final class Reader implements Closeable {
+    private static final int IPV4_LEN = 4;
+    private static final int IPV6_LEN = 6;
     private static final int DATA_SECTION_SEPARATOR_SIZE = 16;
     private static final byte[] METADATA_START_MARKER = {(byte) 0xAB,
         (byte) 0xCD, (byte) 0xEF, 'M', 'a', 'x', 'M', 'i', 'n', 'd', '.',
@@ -149,6 +153,10 @@ public <T> T get(InetAddress ipAddress, Class<T> cls) throws IOException {
         return getRecord(ipAddress, cls).getData();
     }
 
+    protected int getIpv4Start() {
+        return this.ipV4Start;
+    }
+
     /**
      * Looks up <code>ipAddress</code> in the MaxMind DB.
      *
@@ -190,7 +198,40 @@ record = this.readNode(buffer, record, bit);
         return new DatabaseRecord<>(dataRecord, ipAddress, pl);
     }
 
-    private BufferHolder getBufferHolder() throws ClosedDatabaseException {
+    /**
+     * Creates a Networks iterator.
+     * Please note that a MaxMind DB may map IPv4 networks into several locations
+     * in an IPv6 database. This iterator will iterate over all of these locations
+     * separately. To only iterate over the IPv4 networks once, use the
+     * SkipAliasedNetworks option.
+     * 
+     * @param <T> the generics class for the type of the data in DatabaseRecord.
+     * @param skipAliasedNetworks Enable skipping aliased networks.
+     * @return Networks The Networks iterator.
+     * @throws BadVersionException Exception for using an IPv6 network in ipv4-only database.
+     * @throws ClosedDatabaseException Exception for a closed databased.
+     * @throws InvalidDatabaseException Exception for an invalid database.
+     */
+    public <T> Networks<T> networks(boolean skipAliasedNetworks) throws 
+        BadVersionException, ClosedDatabaseException, InvalidDatabaseException {
+        try {
+            InetAddress ipv4 = InetAddress.getByAddress(new byte[4]);
+            InetAddress ipv6 = InetAddress.getByAddress(new byte[16]);
+            Network ipAllV4 = new Network(ipv4, 0); // Mask 32.
+            Network ipAllV6 = new Network(ipv6, 0); // Mask 128.
+
+            if (this.getMetadata().getIpVersion() == 6) {
+                return this.networksWithIn(ipAllV6, skipAliasedNetworks);
+            }
+            return this.networksWithIn(ipAllV4, skipAliasedNetworks);
+        } catch (UnknownHostException e) {
+            /* This is returned by getByAddress. This should never happen
+            as the ipv4 and ipv6 are constants set by us. */
+            return null;
+        }
+    }
+
+    protected BufferHolder getBufferHolder() throws ClosedDatabaseException {
         BufferHolder bufferHolder = this.bufferHolderReference.get();
         if (bufferHolder == null) {
             throw new ClosedDatabaseException();
@@ -222,20 +263,98 @@ private int findIpV4StartNode(ByteBuffer buffer)
         return node;
     }
 
-    private int readNode(ByteBuffer buffer, int nodeNumber, int index)
-        throws InvalidDatabaseException {
+    /**
+     * Returns an iterator within the specified network.
+     * Please note that a MaxMind DB may map IPv4 networks into several locations
+     * in an IPv6 database. This iterator will iterate over all of these locations
+     * separately. To only iterate over the IPv4 networks once, use the
+     * SkipAliasedNetworks option.
+     * @param <T> Represents the data type(e.g., Map, HastMap, etc.).
+     * @param network Specifies the network to be iterated.
+     * @param skipAliasedNetworks Boolean for skipping aliased networks.
+     * @return Networks
+     * @throws BadVersionException Exception for using an IPv6 network in ipv4-only database.
+     * @throws ClosedDatabaseException Exception for a closed databased.
+     * @throws InvalidDatabaseException Exception for an invalid database.
+     */
+    public <T> Networks<T> networksWithIn(Network network, boolean skipAliasedNetworks) 
+        throws BadVersionException, ClosedDatabaseException, InvalidDatabaseException {
+        InetAddress networkAddress = network.getNetworkAddress();
+        if (this.metadata.getIpVersion() == 4 && networkAddress instanceof Inet6Address) {
+            throw new BadVersionException(networkAddress);
+        }
+
+        byte[] ipBytes = networkAddress.getAddress();
+        int prefixLength = network.getPrefixLength();
+
+        if (this.metadata.getIpVersion() == 6 && ipBytes.length == IPV4_LEN) {
+            if (skipAliasedNetworks) {
+                ipBytes = new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                        ipBytes[0], ipBytes[1], ipBytes[2], ipBytes[3] };
+            } else {
+                // Convert it to the IP address (in 16-byte from) of the IPv4 address.
+                ipBytes = new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                    -1, -1, // -1 is for 0xff.
+                    ipBytes[0], ipBytes[1], ipBytes[2], ipBytes[3]};
+            }
+            prefixLength += 96;
+        }
+
+        int[] traverseResult = this.traverseTree(ipBytes, 0, prefixLength);
+        int node = traverseResult[0];
+        int prefix = traverseResult[1];
+
+        Networks<T> networks = new Networks<T>(this, skipAliasedNetworks,
+            new Networks.NetworkNode[]{ new Networks.NetworkNode(ipBytes, prefix, node) });
+
+        return networks;
+    }
+
+    /**
+     * Returns the node number and the prefix for the network.
+     * @param ip The ip address to travese.
+     * @param node The number of the node.
+     * @param bitCount The prefix.
+     * @return int[]
+     */
+    public int[] traverseTree(byte[] ip, int node, int bitCount)
+        throws ClosedDatabaseException, InvalidDatabaseException {
+        int nodeCount = this.metadata.getNodeCount();
+        int i = 0;
+
+        ByteBuffer buffer = this.getBufferHolder().get();
+
+        for (; i < bitCount && node < nodeCount; i++) {
+            int bit = 1 & (ip[i >> 3] >> (7 - (i % 8)));
+
+            // bit:0 -> left record.
+            // bit:1 -> right record.
+            node = this.readNode(buffer, node, bit);
+        }
+
+        return new int[]{node, i};
+    }
+
+    protected int readNode(ByteBuffer buffer, int nodeNumber, int index)
+            throws InvalidDatabaseException {
+        // index is the index of the record within the node, which
+        // can either be 0 or 1.
         int baseOffset = nodeNumber * this.metadata.getNodeByteSize();
 
         switch (this.metadata.getRecordSize()) {
             case 24:
+                // For a 24 bit record, each record is 3 bytes.
                 buffer.position(baseOffset + index * 3);
                 return Decoder.decodeInteger(buffer, 0, 3);
             case 28:
                 int middle = buffer.get(baseOffset + 3);
 
                 if (index == 0) {
+                    // We get the most significant from the first half
+                    // of the byte. It belongs to the first record.
                     middle = (0xF0 & middle) >>> 4;
                 } else {
+                    // We get the most significant byte of the second record.
                     middle = 0x0F & middle;
                 }
                 buffer.position(baseOffset + index * 4);
@@ -249,7 +368,7 @@ private int readNode(ByteBuffer buffer, int nodeNumber, int index)
         }
     }
 
-    private <T> T resolveDataPointer(
+    protected <T> T resolveDataPointer(
         ByteBuffer buffer,
         int pointer,
         Class<T> cls
diff --git a/src/test/java/com/maxmind/db/ReaderTest.java b/src/test/java/com/maxmind/db/ReaderTest.java
