Skip to content
This repository has been archived by the owner on Mar 8, 2024. It is now read-only.

Commit

Permalink
Add iploc-fetch
Browse files Browse the repository at this point in the history
  • Loading branch information
kayon committed May 20, 2018
1 parent 729312b commit 1c77e58
Show file tree
Hide file tree
Showing 8 changed files with 318 additions and 106 deletions.
4 changes: 3 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,7 @@ install:
- go get -u golang.org/x/text/encoding/simplifiedchinese
- go get -u github.com/google/btree
- make install
- make prepare
script:
- go test
- make test
- make clean
21 changes: 14 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,18 +1,25 @@
IPLOC := $(GOPATH)/bin/iploc
IPLOC_GEN = $(GOPATH)/bin/iploc-gen
IPLOC_CONV = $(GOPATH)/bin/iploc-conv
IPLOC_FETCH := $(GOPATH)/bin/iploc-fetch
IPLOC_GEN = $(GOPATH)/bin/iploc-gen


.PHONY: all test clean install
.PHONY: all test clean install prepare

all: install test
all: install prepare test

clean:
rm -f $(IPLOC) $(IPLOC_GEN) $(IPLOC_CONV)
rm -f $(IPLOC_CONV) $(IPLOC_FETCH) $(IPLOC_GEN)
rm -f qqwry.dat

install:
cd cmd/iploc-gen; go install
cd cmd/iploc; $(IPLOC_GEN) ../../qqwry.dat -n; go install
cd cmd/iploc-conv; go install
cd cmd/iploc-fetch; go install
cd cmd/iploc-gen; go install

prepare:
iploc-fetch ./qqwry.gbk.dat
iploc-conv -s qqwry.gbk.dat -d qqwry.dat -n
rm qqwry.gbk.dat

test:
go test -v
Expand Down
65 changes: 49 additions & 16 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@

[![Build Status](https://travis-ci.org/kayon/iploc.svg?branch=master)](https://travis-ci.org/kayon/iploc)

使用纯真IP库 `qqwry.dat`,高性能,线程安全
使用纯真IP库 `qqwry.dat`,高性能,线程安全,并对国内数据格式化到省、市、县

> 需要 go 1.9 或更高
> 附带的 `qqwry.dat``UTF-8` 编码 `2018-05-15版本`
> <del>附带的 `qqwry.dat``UTF-8` 编码 `2018-05-15版本`</del>
> 不再提供 `qqwry.dat`, 新增命令行工具 `iploc-fetch`, 可在线获取官方最新版本的 `qqwry.dat`

## 安装

Expand All @@ -22,9 +25,29 @@ cd $GOPATH/src/golang.org/x
git clone https://github.com/golang/text.git
```

## 更新 qqwry.dat
## 获取&更新 qqwry.dat

#### 1. 下载 `qqwry.dat`

##### 方法一:使用命令行工具 [iploc-fetch](#iploc-fetch)

下载到当前目录,保存为 `qqwry.gbk.dat`

```
$ iploc-fetch qqwry.gbk.dat
```

在纯真官网[下载最新的 qqwry.dat](http://www.cz88.net/fox/ipdat.shtml) 并转换为 `UTF-8` 使用命令行工具 [iploc-conv](#iploc-conv)
##### 方法二:手动下载

[纯真官网下载](http://www.cz88.net/fox/ipdat.shtml)并安装,复制安装目录中的 `qqwry.dat`

#### 2. 转换为 `UTF-8`

使用命令行工具 [iploc-conv](#iploc-conv) 将刚刚下载的 `qqwry.gbk.dat` 转换为 `UTF-8` 保存为 `qqwry.dat`

```
iploc-conv -s qqwry.gbk.dat -d qqwry.dat
```


## Benchmarks
Expand Down Expand Up @@ -55,6 +78,15 @@ func main() {
// output
// IP:8.8.0.8; 网段: 8.7.245.0 - 8.8.3.255; 美国 科罗拉多州布隆菲尔德市Level 3通信股份有限公司
// true true
detail = loc.Find("1.24.41.0")
fmt.Println(detail.String())
fmt.Println(detail.Country, detail.Province, detail.City, detail.County)
// output
// 内蒙古锡林郭勒盟苏尼特右旗 联通
// 中国 内蒙古 锡林郭勒盟 苏尼特右旗
}
```

Expand Down Expand Up @@ -82,16 +114,6 @@ func main() {

## 命令行工具

#### iploc

命令行版IP查询

```
$ iploc 127.1
$ 127.0.0.1 本机地址 N/A
```
> DAT编译到二进制执行文件中,不依赖 `qqwry.dat` 位置
#### <a name="iploc-conv"></a>iploc-conv

将原版 `qqwry.dat``GBK` 转换为 `UTF-8`
Expand All @@ -104,20 +126,31 @@ $ iploc-conv -s src.gbk.dat -d dst.utf8.dat
> 修正原 qqwry.dat 中几处错误的重定向 (qqwry.dat 2018-05-10),并将 "CZ88.NET" 替换为 "N/A"
#### <a name="iploc-fetch"></a>iploc-fetch

从纯真官网下载最新 `qqwry.dat`

```
$ iploc-fetch qqwry.gbk.dat
```

> 下载后别忘了使用 `iploc-conv` 转换为 `UTF-8`
#### iploc-gen

创建静态版本的 **iploc** 集成到你的项目中

`iploc-gen` 会在当前目录创建 iploc-binary.go 文件,拷贝到你的项目中,通过变量名 *IPLoc* 直接可以使用

```
$ iploc-gen /path/qqwry.dat
$ iploc-gen path/qqwry.dat
```

> `--pkg` 指定 package name, 默认 main
> `-n` 使用 `OpenWithoutIndexes` 初始化,无索引

## 静态编译 iploc 和 qqwry.dat 并集成到你的项目中

编译后的二进制没有 `qqwry.dat` 依赖,不需要再带着 `qqwry.dat` 一起打包了
Expand All @@ -128,7 +161,7 @@ $ iploc-gen /path/qqwry.dat

```
$ mkdir myloc && cd myloc
$ iploc-gen $GOPATH/src/github.com/kayon/iploc/qqwry.dat --pkg myloc
$ iploc-gen path/qqwry.dat --pkg myloc
```

> $GOPATH/src/myproject/main.go
Expand Down
6 changes: 3 additions & 3 deletions cmd/iploc-conv/iploc-conv.go
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ func check() {
fmt.Fprintf(os.Stderr, "it's not the same version")
os.Exit(1)
}
if logger, err = os.Create("output.log"); err != nil {
if logger, err = os.Create("iploc-conv.log"); err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
Expand Down Expand Up @@ -238,9 +238,9 @@ func check() {
fmt.Printf("\rCheck %6.2f%% %*d/%d %s\n", 100.0, l, count, count, time.Since(st))
}
if warning > 0 {
fmt.Printf("%d warnings. please see output.log for more details\n", warning)
fmt.Printf("%d warnings. please see iploc-conv.log for more details\n", warning)
} else {
os.Remove("output.log")
os.Remove("iploc-conv.log")
}
}

Expand Down
183 changes: 183 additions & 0 deletions cmd/iploc-fetch/iploc-fetch.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
package main

import (
"bytes"
"compress/zlib"
"encoding/binary"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"strconv"

flag "github.com/spf13/pflag"
"golang.org/x/text/encoding/simplifiedchinese"
"golang.org/x/text/transform"
)

const (
urlCopywrite = "http://update.cz88.net/ip/copywrite.rar"
urlQqwryDat = "http://update.cz88.net/ip/qqwry.rar"
)

var (
outputFile string
quiet, help bool

fileSize uint64
fileSizeW int
)

type FetchWriter struct {
n uint64
}

func (w *FetchWriter) Write(p []byte) (int, error) {
n := len(p)
w.n += uint64(n)
w.progress()
return n, nil
}

func (w *FetchWriter) progress() {
printf("\rfetch: %6.2f%% %*d/%d bytes", float64(w.n)/float64(fileSize)*100, fileSizeW, w.n, fileSize)
}

func fetch(key uint32) error {
resp, err := http.Get(urlQqwryDat)
if err != nil {
return err
}
defer resp.Body.Close()
size, err := strconv.ParseUint(resp.Header.Get("Content-Length"), 10, 32)
if err != nil {
return err
}
if size != fileSize {
return fmt.Errorf("the file size of the agreement is different")
}

var b []byte
buffer := bytes.NewBuffer(b[:])

_, err = io.Copy(buffer, io.TeeReader(resp.Body, &FetchWriter{}))
printf("\n")
if err != nil {
return err
}
b = buffer.Bytes()

// 解码前512字节
for i := 0; i < 0x200; i++ {
key *= 0x805
key++
key &= 0xFF
b[i] = b[i] ^ byte(key)
}

r, err := zlib.NewReader(bytes.NewBuffer(b))
if err != nil {
return err
}
defer r.Close()

f, err := os.Create(outputFile)
if err != nil {
return err
}
_, err = io.Copy(f, r)
return err
}

func init() {
flag.BoolVarP(&quiet, "quiet", "q", false, "only output error")
flag.BoolVarP(&help, "help", "h", false, "this help")
flag.CommandLine.SortFlags = false
flag.Parse()

flag.Usage = func() {
fmt.Fprintf(os.Stderr, "iploc-fetch: fetch qqwry.dat.\nUsage: iploc-fetch [output filename] [arguments]\nOptions:\n")
flag.PrintDefaults()
}

outputFile = flag.Arg(0)

if outputFile == "" || help {
flag.Usage()
if help {
os.Exit(0)
}
os.Exit(1)
}
}

func main() {
resp, err := http.Get(urlCopywrite)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
b, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()

fatal(err)

s := readVersion(b)
if s == nil {
fatal(fmt.Errorf("invalid file description"))
}
fileSize = uint64(binary.LittleEndian.Uint32(b[12:]))
fileSizeW = len(fmt.Sprint(fileSize))
key := binary.LittleEndian.Uint32(b[20:])

printf("version: %s\n", toUTF8(s))
printf("fetch: ...")

fatal(fetch(key))
}

func readVersion(p []byte) []byte {
var start = 24
var end int

if start >= len(p) {
return nil
}

// 0x20 ASCII space
for p[start] != 0x20 {
start++
}
start += 1
end = start

if end >= len(p) {
return nil
}

for p[end] != 0x00 {
end++
}
return p[start:end]
}

func toUTF8(s []byte) (b []byte) {
r := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())
b, _ = ioutil.ReadAll(r)
return
}

func fatal(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, err.Error() + "\n")
os.Exit(1)
}
}

func printf(format string, args ...interface{}) {
if quiet {
return
}
fmt.Printf(format, args...)
}
15 changes: 0 additions & 15 deletions cmd/iploc/iploc-binary.go

This file was deleted.

Loading

0 comments on commit 1c77e58

Please sign in to comment.