Skip to content

Commit

Permalink
Add support for defaults
Browse files Browse the repository at this point in the history
  • Loading branch information
@dhaavi
dhaavi committed Dec 6, 2023
1 parent fec80ff commit fc02e0b
Show file tree
Hide file tree
Showing 6 changed files with 211 additions and 86 deletions.
144 changes: 91 additions & 53 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,34 +28,38 @@ __Step 3: Build your MMDBs__

==========
building My IPv4 GeoIP DB
database options set: IncludeReservedNetworks=true IPVersion=4 RecordSize=24
database options set: IPVersion=4 RecordSize=24 (IncludeReservedNetworks=true DisableIPv4Aliasing=true)
database types: autonomous_system_number, autonomous_system_organization, country.iso_code
optimizations set: FloatDecimals=2 ForceIPVersion=true MaxPrefix=0
conditional resets: [{IfChanged:[country] Reset:[location]}]
---
processing input/asn-ipv4.csv...
inserted 100000 entries - batch in 314ms (3µs/op)
inserted 200000 entries - batch in 287ms (3µs/op)
inserted 300000 entries - batch in 281ms (3µs/op)
inserted 368765 entries - batch in 206ms (2µs/op)
processing input/iptoasn-asn-ipv4.csv...
inserted 100000 entries - batch in 356ms (4µs/op)
inserted 200000 entries - batch in 327ms (3µs/op)
inserted 300000 entries - batch in 322ms (3µs/op)
inserted 360970 entries - batch in 224ms (2µs/op)
---
processing input/geo-whois-asn-country-ipv4.csv...
inserted 100000 entries - batch in 931ms (9µs/op)
inserted 200000 entries - batch in 684ms (7µs/op)
inserted 238714 entries - batch in 353ms (4µs/op)
inserted 100000 entries - batch in 1.012s (10µs/op)
inserted 200000 entries - batch in 889ms (9µs/op)
inserted 238756 entries - batch in 418ms (4µs/op)
---
My IPv4 GeoIP DB finished: inserted 607479 entries in 3s, resulting in 7MB
My IPv4 GeoIP DB finished: inserted 599726 entries in 4s, resulting in 7.67 MB written to output/geoip-v4.mmdb

==========
building My IPv6 GeoIP DB
database options set: IncludeReservedNetworks=true IPVersion=6 RecordSize=24
database options set: IPVersion=6 RecordSize=24 (IncludeReservedNetworks=true DisableIPv4Aliasing=true)
database types: autonomous_system_number, autonomous_system_organization, country.iso_code
optimizations set: FloatDecimals=2 ForceIPVersion=true MaxPrefix=0
conditional resets: [{IfChanged:[country] Reset:[location]}]
---
processing input/asn-ipv6.csv...
inserted 86473 entries - batch in 382ms (4µs/op)
processing input/iptoasn-asn-ipv6.csv...
inserted 81939 entries - batch in 352ms (4µs/op)
---
processing input/geo-whois-asn-country-ipv6.csv...
inserted 82381 entries - batch in 1.389s (14µs/op)
inserted 82321 entries - batch in 1.609s (16µs/op)
---
My IPv6 GeoIP DB finished: inserted 168854 entries in 2s, resulting in 5MB
My IPv6 GeoIP DB finished: inserted 164260 entries in 2s, resulting in 5.42 MB written to output/geoip-v6.mmdb

__Step 4: Check your MMDBs__

Expand All @@ -64,43 +68,43 @@ __Step 4: Check your MMDBs__

[...]

loading output/geoip-v4.mmdb with 7.85 MB
loading output/geoip-v4.mmdb with 7.67 MB

Running all checks:

Probing:
..............................................................................................................................................................................................................................................................::
analyzed with 0 lookup errors
Total= 14462461 Country=99.57% Coords=0.00% ASN=87.04% ASOrg=87.04% AC=0.00% SP=0.00% AP=0.00%
Total= 14462461 Country=99.57% Coords=0.00% ASN=82.40% ASOrg=82.40% AC=0.00% SP=0.00% AP=0.00%

Network Mask Stats:
Total= 770040 Country=99.94% Coords=0.00% ASN=89.10% ASOrg=89.10% AC=0.00% SP=0.00% AP=0.00%
Total= 800822 Country=99.93% Coords=0.00% ASN=85.24% ASOrg=85.24% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 7 Total= 1 Country=100.00% Coords=0.00% ASN=100.00% ASOrg=100.00% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 8 Total= 6 Country=100.00% Coords=0.00% ASN=100.00% ASOrg=100.00% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 9 Total= 9 Country=100.00% Coords=0.00% ASN=88.89% ASOrg=88.89% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 10 Total= 35 Country=100.00% Coords=0.00% ASN=88.57% ASOrg=88.57% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 11 Total= 90 Country=100.00% Coords=0.00% ASN=88.89% ASOrg=88.89% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 12 Total= 273 Country=100.00% Coords=0.00% ASN=89.74% ASOrg=89.74% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 13 Total= 561 Country=100.00% Coords=0.00% ASN=90.02% ASOrg=90.02% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 14 Total= 1321 Country=100.00% Coords=0.00% ASN=87.06% ASOrg=87.06% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 15 Total= 2743 Country=100.00% Coords=0.00% ASN=83.81% ASOrg=83.81% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 16 Total= 10414 Country=99.96% Coords=0.00% ASN=83.52% ASOrg=83.52% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 17 Total= 7469 Country=100.00% Coords=0.00% ASN=87.36% ASOrg=87.36% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 18 Total= 13200 Country=99.97% Coords=0.00% ASN=87.29% ASOrg=87.29% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 19 Total= 24993 Country=99.98% Coords=0.00% ASN=88.65% ASOrg=88.65% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 20 Total= 38550 Country=99.96% Coords=0.00% ASN=88.21% ASOrg=88.21% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 21 Total= 53388 Country=99.95% Coords=0.00% ASN=86.92% ASOrg=86.92% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 22 Total= 115769 Country=99.92% Coords=0.00% ASN=88.42% ASOrg=88.42% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 23 Total= 118221 Country=99.93% Coords=0.00% ASN=84.43% ASOrg=84.43% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 24 Total= 228493 Country=99.90% Coords=0.00% ASN=86.77% ASOrg=86.77% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 25 Total= 10503 Country=100.00% Coords=0.00% ASN=96.15% ASOrg=96.15% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 26 Total= 15323 Country=100.00% Coords=0.00% ASN=96.80% ASOrg=96.80% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 27 Total= 21747 Country=100.00% Coords=0.00% ASN=97.20% ASOrg=97.20% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 28 Total= 28184 Country=100.00% Coords=0.00% ASN=98.64% ASOrg=98.64% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 29 Total= 30933 Country=100.00% Coords=0.00% ASN=98.97% ASOrg=98.97% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 30 Total= 23054 Country=100.00% Coords=0.00% ASN=99.51% ASOrg=99.51% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 31 Total= 7488 Country=100.00% Coords=0.00% ASN=99.49% ASOrg=99.49% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 32 Total= 17272 Country=100.00% Coords=0.00% ASN=99.58% ASOrg=99.58% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 9 Total= 8 Country=100.00% Coords=0.00% ASN=62.50% ASOrg=62.50% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 10 Total= 35 Country=100.00% Coords=0.00% ASN=71.43% ASOrg=71.43% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 11 Total= 85 Country=100.00% Coords=0.00% ASN=78.82% ASOrg=78.82% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 12 Total= 278 Country=100.00% Coords=0.00% ASN=83.45% ASOrg=83.45% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 13 Total= 569 Country=100.00% Coords=0.00% ASN=87.17% ASOrg=87.17% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 14 Total= 1327 Country=100.00% Coords=0.00% ASN=84.78% ASOrg=84.78% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 15 Total= 2760 Country=100.00% Coords=0.00% ASN=82.14% ASOrg=82.14% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 16 Total= 10167 Country=99.95% Coords=0.00% ASN=80.14% ASOrg=80.14% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 17 Total= 7563 Country=100.00% Coords=0.00% ASN=83.49% ASOrg=83.49% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 18 Total= 13354 Country=99.97% Coords=0.00% ASN=83.91% ASOrg=83.91% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 19 Total= 25354 Country=99.98% Coords=0.00% ASN=84.98% ASOrg=84.98% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 20 Total= 39116 Country=99.96% Coords=0.00% ASN=84.53% ASOrg=84.53% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 21 Total= 54874 Country=99.95% Coords=0.00% ASN=82.60% ASOrg=82.60% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 22 Total= 117368 Country=99.94% Coords=0.00% ASN=83.46% ASOrg=83.46% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 23 Total= 124541 Country=99.94% Coords=0.00% ASN=79.46% ASOrg=79.46% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 24 Total= 240113 Country=99.92% Coords=0.00% ASN=81.46% ASOrg=81.46% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 25 Total= 10581 Country=99.94% Coords=0.00% ASN=95.53% ASOrg=95.53% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 26 Total= 15500 Country=99.95% Coords=0.00% ASN=96.35% ASOrg=96.35% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 27 Total= 22059 Country=99.91% Coords=0.00% ASN=96.78% ASOrg=96.78% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 28 Total= 30393 Country=99.91% Coords=0.00% ASN=98.38% ASOrg=98.38% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 29 Total= 34128 Country=99.89% Coords=0.00% ASN=98.76% ASOrg=98.76% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 30 Total= 24836 Country=99.81% Coords=0.00% ASN=99.44% ASOrg=99.44% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 31 Total= 7847 Country=99.86% Coords=0.00% ASN=98.89% ASOrg=98.89% AC=0.00% SP=0.00% AP=0.00%
CIDR=/ 32 Total= 17959 Country=99.96% Coords=0.00% ASN=99.20% ASOrg=99.20% AC=0.00% SP=0.00% AP=0.00%

This check queries the following fields:

Expand Down Expand Up @@ -159,26 +163,60 @@ There are three special fields which are not defined in the types:

These are used to derive the IP ranges the data (row, entry) is applicable for.

### CSV
##### CSV

Define columns with `fields`, which must match a field defined in the `types`:

fields: ["from", "to", "autonomous_system_number", "autonomous_system_organization"]
databases:
...
fields: ["from", "to", "autonomous_system_number", "autonomous_system_organization"]

All rows must have exactly the specified amount of columns. Use `-` to define a column you are not using, eg.:

fields: ["from", "to", "country.iso_code", "-", "-", "-", "-", "location.latitude", "location.longitude", "-"]
databases:
...
fields: ["from", "to", "country.iso_code", "-", "-", "-", "-", "location.latitude", "location.longitude", "-"]

### IPFire
##### IPFire

The [IPFire Firewall](https://www.ipfire.org/) maintains a [geoip database in a custom format](https://git.ipfire.org/?p=location/location-database.git;a=summary), which notably includes IP categorization, such as `is-anycast`.

Define fields with `fieldMap`, mapping IPFire database keys to `types`:

fieldMap:
"aut-num": "autonomous_system_number"
"name": "autonomous_system_organization"
"country": "country.iso_code"
"is-anycast": "is_anycast"
"is-satellite-provider": "is_satellite_provider"
"is-anonymous-proxy": "is_anonymous_proxy"
databases:
...
fieldMap:
"aut-num": "autonomous_system_number"
"name": "autonomous_system_organization"
"country": "country.iso_code"
"is-anycast": "is_anycast"
"is-satellite-provider": "is_satellite_provider"
"is-anonymous-proxy": "is_anonymous_proxy"

### Defaults

If you are building more than one mmdb file, you can use defaults to apply certain configuration to all databases.
A common case is when you build an IPv4 and IPv6 database separately.

When using defaults, always check if they are applied correctly by checking the logs when building the databases.

defaults:
types: # Entries are merged.
# Databases inherit defaults types, if not yet set.
# You can define a type with a "-" type string to ignore a default in a database config.
"country.iso_code": string
"location.latitude": float32
"location.longitude": float32
"autonomous_system_organization": string
"autonomous_system_number": uint32
"is_anycast": bool
"is_satellite_provider": bool
"is_anonymous_proxy": bool
optimize: # Entries are used as default separately.
floatDecimals: 2 # Default is used when database value is 0.
forceIPVersion: true # Default is used when database value is not defined.
maxPrefix: 24 # Default is used when database value is 0.
merge: # Entries are used as default separately.
conditionalResets: # Default is used when not defined or empty in database config.
- ifChanged: ["country"]
reset: ["location"]
6 changes: 6 additions & 0 deletions cmd/mmdbmeld/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ func main() {
for _, db := range c.Databases {
fmt.Printf("\n==========\nbuilding %s\n", db.Name)

// Apply defaults.
dbP := &db //nolint:gosec,scopelint // Only used within loop.
c.Defaults.ApplyTo(dbP)

// Load sources for database.
sources, err := mmdbmeld.LoadSources(db)
if err != nil {
fmt.Println(err)
Expand All @@ -42,6 +47,7 @@ func main() {
}
}()

// Read all sources and write to mmdb.
err = mmdbmeld.WriteMMDB(db, sources, updates)
if err != nil {
fmt.Println(err)
Expand Down
49 changes: 46 additions & 3 deletions config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
// Config is the geoip build config.
type Config struct {
Databases []DatabaseConfig `yaml:"databases"`
Defaults DefaultConfig `yaml:"defaults"`
}

// DatabaseConfig holds the config for building one database.
Expand All @@ -22,6 +23,13 @@ type DatabaseConfig struct {
Merge MergeConfig `yaml:"merge"`
}

// DefaultConfig holds a subset of DatabaseConfig fields to be used as a default config.
type DefaultConfig struct {
Types map[string]string `yaml:"types"`
Optimize Optimizations `yaml:"optimize"`
Merge MergeConfig `yaml:"merge"`
}

// MMDBConfig holds mmdb specific config.
type MMDBConfig struct {
IPVersion int `yaml:"ipVersion"`
Expand All @@ -37,9 +45,14 @@ type DatabaseInput struct {

// Optimizations holds optimization config.
type Optimizations struct {
FloatDecimals int `yaml:"floatDecimals"`
ForceIPVersion bool `yaml:"forceIPVersion"`
MaxPrefix int `yaml:"maxPrefix"`
FloatDecimals int `yaml:"floatDecimals"`
ForceIPVersion *bool `yaml:"forceIPVersion"`
MaxPrefix int `yaml:"maxPrefix"`
}

// ForceIPVersionEnabled reports whether ForceIPVersion is set and true.
func (o Optimizations) ForceIPVersionEnabled() bool {
return o.ForceIPVersion != nil && *o.ForceIPVersion
}

// MergeConfig holds merge configuration.
Expand All @@ -62,3 +75,33 @@ func LoadConfig(filePath string) (*Config, error) {
config := &Config{}
return config, yaml.Unmarshal(data, config)
}

// ApplyTo applies the default config to the given database config.
func (d DefaultConfig) ApplyTo(c *DatabaseConfig) {
// Add all missing default types.
if c.Types == nil {
c.Types = make(map[string]string)
}
for k, v := range d.Types {
_, ok := c.Types[k]
if !ok {
c.Types[k] = v
}
}

// Apply Optimizations.
if c.Optimize.FloatDecimals == 0 && d.Optimize.FloatDecimals != 0 {
c.Optimize.FloatDecimals = d.Optimize.FloatDecimals
}
if c.Optimize.ForceIPVersion == nil && d.Optimize.ForceIPVersion != nil {
c.Optimize.ForceIPVersion = d.Optimize.ForceIPVersion
}
if c.Optimize.MaxPrefix == 0 && d.Optimize.MaxPrefix != 0 {
c.Optimize.MaxPrefix = d.Optimize.MaxPrefix
}

// Apply Merge Config.
if len(c.Merge.ConditionalResets) == 0 && len(d.Merge.ConditionalResets) != 0 {
c.Merge.ConditionalResets = d.Merge.ConditionalResets
}
}
11 changes: 7 additions & 4 deletions source_csv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,15 @@ func (csv *CSVSource) NextEntry() (*SourceEntry, error) {
if v4 := se.To.To4(); v4 != nil {
se.To = v4
}
case "-":
case "", "-":
// Ignore
default:
se.Values[fieldName] = SourceValue{
Type: csv.types[fieldName],
Value: row[i],
fieldType, ok := csv.types[fieldName]
if ok && fieldType != "" && fieldType != "-" {
se.Values[fieldName] = SourceValue{
Type: csv.types[fieldName],
Value: row[i],
}
}
}
}
Expand Down
Loading

0 comments on commit fc02e0b

Please sign in to comment.