Skip to content

Commit

Permalink
Add mmdb auto creation
Browse files Browse the repository at this point in the history
sapics committed Aug 15, 2024
1 parent 7e23969 commit 301f894
Showing 17 changed files with 648 additions and 0 deletions.
42 changes: 42 additions & 0 deletions .github/workflows/mmdb-create.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
name: Create mmdb files

on: push:
branches:
- main
- main-test

jobs:
build:
name: Create mmdb files
runs-on: ubuntu-latest
if: contains(github.event.head_commit.message, 'Daily Update')
timeout-minutes: 60
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup golang
uses: actions/setup-go@v5
with:
go-version: '1.22.x'
- name: Copy ip-location-to-mmdb
shell: bash
run: |
wget https://github.com/sapics/ip-location-to-mmdb/releases/download/v0.0.3/ip-location-to-mmdb-linux-x64.bin
chmod +x ip-location-to-mmdb-linux-x64.bin
- name: Copy package.json and README.md
shell: bash
run: node tools/copy-package-readme.js
- name: Update mmdb files
shell: bash
run: node tools/create-mmdb.js
- name: Publish to npm
run: tools/publish-mmdb.sh
env:
NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
GITHUB_TOKEN: ${{secrets.GTOKEN}}
- name: push changes
uses: ad-m/github-push-action@master
with:
github_token: ${{secrets.GTOKEN}}
repository: sapics/ip-location-db
branch: main
43 changes: 43 additions & 0 deletions tools/conf/asn-country.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
databases:
- name: "ASN-COUNTRY ipv4"
mmdb:
ipVersion: 4 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"country_code": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "asn-country/asn-country-ipv4.csv"
fields: ["from", "to", "country_code"]
output: asn-country-mmdb/asn-country-ipv4.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["country_code"]
reset: ["location"]
- name: "ASN-COUNTRY ipv6"
mmdb:
ipVersion: 6 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"country_code": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "asn-country/asn-country-ipv6.csv"
fields: ["from", "to", "country_code"]
output: asn-country-mmdb/asn-country-ipv6.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["country_code"]
reset: ["location"]
45 changes: 45 additions & 0 deletions tools/conf/asn.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
databases:
- name: "ASN ipv4"
mmdb:
ipVersion: 4 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"autonomous_system_number": uint32
"autonomous_system_organization": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "asn/asn-ipv4.csv"
fields: ["from", "to", "autonomous_system_number", "autonomous_system_organization"]
output: asn-mmdb/asn-ipv4.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["autonomous_system_number"]
reset: ["location"]
- name: "ASN ipv6"
mmdb:
ipVersion: 6 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"autonomous_system_number": uint32
"autonomous_system_organization": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "asn/asn-ipv6.csv"
fields: ["from", "to", "autonomous_system_number", "autonomous_system_organization"]
output: asn-mmdb/asn-ipv6.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["autonomous_system_number"]
reset: ["location"]
45 changes: 45 additions & 0 deletions tools/conf/dbip-asn.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
databases:
- name: "DBIP-ASN ipv4"
mmdb:
ipVersion: 4 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"autonomous_system_number": uint32
"autonomous_system_organization": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "dbip-asn/dbip-asn-ipv4.csv"
fields: ["from", "to", "autonomous_system_number", "autonomous_system_organization"]
output: dbip-asn-mmdb/dbip-asn-ipv4.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["autonomous_system_number"]
reset: ["location"]
- name: "DBIP-ASN ipv6"
mmdb:
ipVersion: 6 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"autonomous_system_number": uint32
"autonomous_system_organization": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "dbip-asn/dbip-asn-ipv6.csv"
fields: ["from", "to", "autonomous_system_number", "autonomous_system_organization"]
output: dbip-asn-mmdb/dbip-asn-ipv6.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["autonomous_system_number"]
reset: ["location"]
43 changes: 43 additions & 0 deletions tools/conf/dbip-country.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
databases:
- name: "DBIP-COUNTRY ipv4"
mmdb:
ipVersion: 4 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"country_code": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "dbip-country/dbip-country-ipv4.csv"
fields: ["from", "to", "country_code"]
output: dbip-country-mmdb/dbip-country-ipv4.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["country_code"]
reset: ["location"]
- name: "DBIP-COUNTRY ipv6"
mmdb:
ipVersion: 6 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"country_code": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "dbip-country/dbip-country-ipv6.csv"
fields: ["from", "to", "country_code"]
output: dbip-country-mmdb/dbip-country-ipv6.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["country_code"]
reset: ["location"]
43 changes: 43 additions & 0 deletions tools/conf/geo-asn-country.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
databases:
- name: "GEO-ASN-COUNTRY ipv4"
mmdb:
ipVersion: 4 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"country_code": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "geo-asn-country/geo-asn-country-ipv4.csv"
fields: ["from", "to", "country_code"]
output: geo-asn-country-mmdb/geo-asn-country-ipv4.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["country_code"]
reset: ["location"]
- name: "GEO-ASN-COUNTRY ipv6"
mmdb:
ipVersion: 6 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"country_code": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "geo-asn-country/geo-asn-country-ipv6.csv"
fields: ["from", "to", "country_code"]
output: geo-asn-country-mmdb/geo-asn-country-ipv6.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["country_code"]
reset: ["location"]
43 changes: 43 additions & 0 deletions tools/conf/geo-whois-asn-country.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
databases:
- name: "GEO-ASN-COUNTRY ipv4"
mmdb:
ipVersion: 4 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"country_code": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "geo-whois-asn-country/geo-whois-asn-country-ipv4.csv"
fields: ["from", "to", "country_code"]
output: geo-whois-asn-country-mmdb/geo-whois-asn-country-ipv4.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["country_code"]
reset: ["location"]
- name: "GEO-ASN-COUNTRY ipv6"
mmdb:
ipVersion: 6 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"country_code": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "geo-whois-asn-country/geo-whois-asn-country-ipv6.csv"
fields: ["from", "to", "country_code"]
output: geo-whois-asn-country-mmdb/geo-whois-asn-country-ipv6.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["country_code"]
reset: ["location"]
45 changes: 45 additions & 0 deletions tools/conf/geolite2-asn.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
databases:
- name: "GEOLITE2-ASN ipv4"
mmdb:
ipVersion: 4 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"autonomous_system_number": uint32
"autonomous_system_organization": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "asn/asn-ipv4.csv"
fields: ["from", "to", "autonomous_system_number", "autonomous_system_organization"]
output: geolite2-asn-mmdb/geolite2-asn-ipv4.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["autonomous_system_number"]
reset: ["location"]
- name: "GEOLITE2-ASN ipv6"
mmdb:
ipVersion: 6 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"autonomous_system_number": uint32
"autonomous_system_organization": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "geolite2-asn/geolite2-asn-ipv6.csv"
fields: ["from", "to", "autonomous_system_number", "autonomous_system_organization"]
output: geolite2-asn-mmdb/geolite2-asn-ipv6.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["autonomous_system_number"]
reset: ["location"]
57 changes: 57 additions & 0 deletions tools/conf/geolite2-city.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
databases:
- name: "GEOLITE2-CITY ipv4"
mmdb:
ipVersion: 4 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 28 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"country_code": string
"state1": string
"state2": string
"city": string
"postcode": string
"latitude": string
"longitude": string
"timezone": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "geolite2-city/geolite2-city-ipv4.csv"
fields: ["from", "to", "country_code", "state1", "state2", "city", "postcode", "latitude", "longitude", "timezone"]
output: geolite2-city-mmdb/geolite2-city-ipv4.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["country_code", "state1", "state2", "city", "postcode", "latitude", "longitude", "timezone"]
reset: ["location"]
- name: "GEOLITE2-CITY ipv6"
mmdb:
ipVersion: 6 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 28 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"country_code": string
"state1": string
"state2": string
"city": string
"postcode": string
"latitude": string
"longitude": string
"timezone": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "geolite2-city/geolite2-city-ipv6.csv"
fields: ["from", "to", "country_code", "state1", "state2", "city", "postcode", "latitude", "longitude", "timezone"]
output: geolite2-city-mmdb/geolite2-city-ipv6.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["country_code", "state1", "state2", "city", "postcode", "latitude", "longitude", "timezone"]
reset: ["location"]
43 changes: 43 additions & 0 deletions tools/conf/geolite2-country.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
databases:
- name: "GEOLITE2-COUNTRY ipv4"
mmdb:
ipVersion: 4 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"country_code": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "geolite2-country/geolite2-country-ipv4.csv"
fields: ["from", "to", "country_code"]
output: geolite2-country-mmdb/geolite2-country-ipv4.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["country_code"]
reset: ["location"]
- name: "GEOLITE2-COUNTRY ipv6"
mmdb:
ipVersion: 6 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"country_code": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "geolite2-country/geolite2-country-ipv6.csv"
fields: ["from", "to", "country_code"]
output: geolite2-country-mmdb/geolite2-country-ipv6.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["country_code"]
reset: ["location"]
45 changes: 45 additions & 0 deletions tools/conf/iptoasn-asn.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
databases:
- name: "IPTOASN-ASN ipv4"
mmdb:
ipVersion: 4 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"autonomous_system_number": uint32
"autonomous_system_organization": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "asn/asn-ipv4.csv"
fields: ["from", "to", "autonomous_system_number", "autonomous_system_organization"]
output: iptoasn-asn-mmdb/iptoasn-asn-ipv4.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["autonomous_system_number"]
reset: ["location"]
- name: "IPTOASN-ASN ipv6"
mmdb:
ipVersion: 6 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"autonomous_system_number": uint32
"autonomous_system_organization": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "iptoasn-asn/iptoasn-asn-ipv6.csv"
fields: ["from", "to", "autonomous_system_number", "autonomous_system_organization"]
output: iptoasn-asn-mmdb/iptoasn-asn-ipv6.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["autonomous_system_number"]
reset: ["location"]
43 changes: 43 additions & 0 deletions tools/conf/iptoasn-country.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
databases:
- name: "IPTOASN-COUNTRY ipv4"
mmdb:
ipVersion: 4 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"country_code": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "iptoasn-country/iptoasn-country-ipv4.csv"
fields: ["from", "to", "country_code"]
output: iptoasn-country-mmdb/iptoasn-country-ipv4.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["country_code"]
reset: ["location"]
- name: "IPTOASN-COUNTRY ipv6"
mmdb:
ipVersion: 6 # Note: IPv4 mmdb can only hold IPv4.
recordSize: 24 # One of 24, 28, 32. Start small, increase if it fails.
types: # Best to always use the same established keys as MaxMind.
"country_code": string
inputs: # Source data and their mapping.
# Inputs are processed as listed. Earlier entries are overwritten by later entries.
- file: "iptoasn-country/iptoasn-country-ipv6.csv"
fields: ["from", "to", "country_code"]
output: iptoasn-country-mmdb/iptoasn-country-ipv6.mmdb
optimize:
floatDecimals: 2 # Limit floats (eg. coordinates) to decimals for smaller DB size. (0=off, set to -1 to no decimals)
forceIPVersion: true # Check IPs and discard IPs with the wrong version. (IPv4 and live in IPv6 mmdb)
maxPrefix: 0 # Remove any network prefixes greater than maxPrefix for smaller DB size. (0=off)
merge:
conditionalResets: # Reset set of top level entries if another set is changed.
# Reset the location entry when the country is changed.
# If the new entry also has a location, it is kept, but a different country without location resets the location.
- ifChanged: ["country_code"]
reset: ["location"]
17 changes: 17 additions & 0 deletions tools/copy-package-readme.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
const fs = require('fs')
const path = require('path')

var topDir = path.resolve(__dirname, '..')
fs.readdir(topDir, function(err, dirs){
if(err) return console.error(err);
var text = fs.readFileSync(path.join(topDir, 'README.md'), {encoding: "utf8"});
for(var dir of dirs){
if(dir.endsWith('-mmdb')){
fs.writeFileSync(path.join(topDir, dir, 'README.md'), text.replace(/^#\s*ip-location-db/, '# ip-location-db/' + dir), {encoding: "utf8"})
var json = JSON.parse(fs.readFileSync(path.join(topDir, dir.replace('-mmdb', ''), 'package.json'), {encoding: "utf8"}))
json.name += '-mmdb'
json.keywords.push('mmdb')
fs.writeFileSync(path.join(topDir, dir, 'package.json'), JSON.stringify(json, null, 2), {encoding: "utf8"})
}
}
})
3 changes: 3 additions & 0 deletions tools/create-all-mmdb.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
while read -d $'\0' file; do
test/mmdbmeld/cmd/mmdbmeld/mmdbmeld $file
done < <(find tools/conf -name '*.yml' -print0)
12 changes: 12 additions & 0 deletions tools/create-mmdb-dir.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
const fs = require('fs')
const path = require('path')

var topDir = path.resolve(__dirname, '..')
var dirs = fs.readdirSync(topDir)
for(var dir of dirs){
if(dir.endsWith('-city') || dir.endsWith('asn') || dir.endsWith('-country')){
if(!dir.startsWith('webnet')){
fs.mkdirSync(path.join(topDir, dir + '-mmdb'))
}
}
}
49 changes: 49 additions & 0 deletions tools/create-mmdb.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
const fs = require('fs')
const path = require('path')
const execSync = require('child_process').execSync
const isWin = process.platform === 'win32'
const mmdbCmd = 'ip-location-to-mmdb-' + (isWin ? 'windows' : 'linux') + '-x64.' + (isWin ? 'exe' : 'bin')

var run = function(){
var cmd = 'git diff --name-only --diff-filter=M'
var changedFiles = execSync(cmd, {encoding: 'utf-8'}).split(/\r?\n/)
var packageFiles = changedFiles.filter(p => p.endsWith('package.json') && p.includes('-mmdb'))
var topDir = path.resolve(__dirname, '..')
var dirs = packageFiles.map(p => path.dirname(p))
if(process.argv.includes('--all')){
dirs = fs.readdirSync(topDir).filter(p => p.includes('-mmdb'))
}
for(var dir of dirs){
var nonMmdbDir = dir.replace('-mmdb', '')
// var confFile = path.join(__dirname, 'conf', nonMmdbDir + '.yml')
var inputV4 = path.join(topDir, nonMmdbDir, nonMmdbDir + '-ipv4.csv')
var inputV6 = path.join(topDir, nonMmdbDir, nonMmdbDir + '-ipv6.csv')
var outputV4 = path.join(topDir, dir, dir + '-ipv4.mmdb')
var outputV6 = path.join(topDir, dir, dir + '-ipv6.mmdb')
console.log(dir)
for(var recordSize of [24, 28]){
try{
if(dir.includes('-city')){
// uncompress .csv.gz files
execSync('7z e -aoa -bd -bso0 -bsp0 -o' + path.join(topDir, nonMmdbDir) + ' ' + inputV4 + '.gz')
execSync('7z e -aoa -bd -bso0 -bsp0 -o' + path.join(topDir, nonMmdbDir) + ' ' + inputV6 + '.gz')
}
execSync(mmdbCmd + ' -i ' + inputV4 + ' -o ' + outputV4 + ' -r ' + (dir.includes('dbip-city') ? 28 : recordSize))
execSync(mmdbCmd + ' -i ' + inputV6 + ' -o ' + outputV6 + ' -r ' + recordSize)
if(dir.includes('-city')){
fs.unlinkSync(path.join(topDir, nonMmdbDir, nonMmdbDir + '-ipv4.csv'))
fs.unlinkSync(path.join(topDir, nonMmdbDir, nonMmdbDir + '-ipv6.csv'))
}
}catch(e){
console.warn(e)
if(dir.includes('-city')){
continue;
}
}
break;
}
}
}

run()

30 changes: 30 additions & 0 deletions tools/publish-mmdb.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@

curdir=`pwd`
dirs=`find . -depth -maxdepth 1 -mindepth 1 -type d -name "*mmdb"`
for dir in $dirs;
do
echo $dir
cd $curdir
cd $dir
bdir=`pwd`
echo "${bdir}"
RES1=`find . -name "*.mmdb" -size -1k 2>/dev/null`
if [ -z "${RES1}" ] ; then
if ! git diff --quiet HEAD -- *.mmdb; then
npm publish
git add .
else
cd $curdir
git checkout HEAD -- ${bdir}
fi
else
cd $curdir
git checkout HEAD -- ${bdir}
fi
done

cd $curdir
cd dist
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --local user.name "github-actions[bot]"
git commit -m "mmdb update"

0 comments on commit 301f894

Please sign in to comment.