Skip to content

Commit

Permalink
支持读取calibre sqlite库更新索引
Browse files Browse the repository at this point in the history
  • Loading branch information
jianyun8023 committed May 7, 2024
1 parent bda5ea8 commit 7b7e080
Show file tree
Hide file tree
Showing 6 changed files with 196 additions and 39 deletions.
17 changes: 6 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ GET /read/:id/file/*path --> 读取书籍中的文件
GET /book/:id --> 获取书籍信息
GET /search --> 搜索书籍(参数q 搜索词,更多参数参考[meilisearch search](https://docs.meilisearch.com/reference/api/search.html))
POST /search --> 搜索书籍(参数q 搜索词,更多参数参考[meilisearch search](https://docs.meilisearch.com/reference/api/search.html))
POST /index/update --> 读取calibre数据库更新索引
```

## 数据导入
Expand All @@ -39,11 +40,12 @@ curl -X "PATCH" "http://localhost:7700/indexes/books/settings" \
],
"filterableAttributes": [
"authors",
"formats",
"file_path",
"id",
"last_modified",
"pubdate",
"publisher",
"isbn",
"tags"
],
"searchableAttributes": [
Expand All @@ -60,16 +62,9 @@ curl -X "PATCH" "http://localhost:7700/indexes/books/settings" \
}'
```

使用calibredb将数据导出为json,
然后上传到meilisearch中

使用下面命令更新索引
```shell
maxID=(curl -s "http://localhost:7700/indexes/books/search?q&limit=1&sort=id%3Adesc&attributesToRetrieve=id" | jq '.hits[0].id')
calibredb --with-library=library list -f all --for-machine --search="id:>$maxID" >> data.json
curl \
-X PUT 'http://localhost:7700/indexes/books/documents' \
-H 'Content-Type: application/json' \
--data-binary @data.json
curl -X "POST" "http://localhost:8080/index/update" -H 'Content-Type: application/json'
```

## 接口
Expand Down Expand Up @@ -199,4 +194,4 @@ CALIBRE_STORAGE_WEBDAV_PATH
"content": "//body"
}
}
```
```
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ require (
github.com/jedib0t/go-pretty/v6 v6.4.3
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213
github.com/kapmahc/epub v0.1.1
github.com/mattn/go-sqlite3 v1.14.22
github.com/meilisearch/meilisearch-go v0.22.0
github.com/minio/minio-go/v7 v7.0.45
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,8 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/meilisearch/meilisearch-go v0.22.0 h1:1J8VO5M8+WpGCOAjeUZy239RsLMWyRxqC2oN703J1cM=
github.com/meilisearch/meilisearch-go v0.22.0/go.mod h1:XmVwi0ZyCdkEQ4cQvA3nh5TT0UByux4kBEWs4WUEp20=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
Expand Down
85 changes: 67 additions & 18 deletions internal/calibre/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ func (c Api) SetupRouter(r *gin.Engine) {
r.GET("/book/:id", c.getBook)
r.GET("/search", c.search)
r.POST("/search", c.search)
r.POST("/index/update", c.updateIndex)
}

func NewClient(config *Config) Api {
Expand Down Expand Up @@ -100,10 +101,7 @@ func (c Api) search(r *gin.Context) {
}
id := book.ID
book.Cover = "/get/cover/" + strconv.FormatInt(id, 10) + ".jpg"
for i := range book.Formats {
s := book.Formats[i]
book.Formats[i] = "/get/book/" + strconv.FormatInt(id, 10) + path.Ext(s)
}
book.FilePath = "/get/book/" + strconv.FormatInt(id, 10) + ".epub"
books[i] = book
}

Expand All @@ -127,13 +125,12 @@ func (c Api) getBook(r *gin.Context) {
err := c.bookIndex.GetDocument(id, nil, &book)

if err != nil {
r.JSON(http.StatusInternalServerError, err)
// 返回文件找不到
r.JSON(http.StatusNotFound, "book not found")
return
}
book.Cover = "/get/cover/" + id + ".jpg"
for i := range book.Formats {
s := book.Formats[i]
book.Formats[i] = "/get/book/" + id + path.Ext(s)
}
book.FilePath = "/get/book/" + id + ".epub"
r.JSON(http.StatusOK, book)

}
Expand All @@ -145,7 +142,7 @@ func (c Api) getBookToc(r *gin.Context) {
if err != nil {
r.JSON(http.StatusInternalServerError, err)
} else {
filepath, _ := c.getFileOrCache(c.fixPath(book.Formats[0]), id)
filepath, _ := c.getFileOrCache(book.FilePath, id)
book, _ := epub.Open(filepath)
points := c.expansionTree(book.Ncx.Points)
var p []epub.NavPoint
Expand Down Expand Up @@ -196,8 +193,7 @@ func (c Api) getBookContent(r *gin.Context) {
if err != nil {
r.JSON(http.StatusInternalServerError, err)
} else {
p := c.fixPath(book.Formats[0])
filepath, _ := c.getFileOrCache(p, id)
filepath, _ := c.getFileOrCache(book.FilePath, id)

destDir := path.Join(c.baseDir, id)

Expand All @@ -212,7 +208,10 @@ func (c Api) getBookContent(r *gin.Context) {

err := unzipSource(filepath, destDir)
if err != nil {
r.JSON(http.StatusInternalServerError, err)
r.JSON(http.StatusInternalServerError, gin.H{
"code": 500,
"message": err.Error(),
})
}
r.FileFromFS(path1, http.Dir(destDir))
}
Expand All @@ -237,12 +236,15 @@ func (c Api) getBookFile(r *gin.Context) {
var book Book
err := c.bookIndex.GetDocument(id, nil, &book)
if err != nil {
r.JSON(http.StatusInternalServerError, err)
log.Println(err)
r.JSON(http.StatusInternalServerError, gin.H{
"code": 500,
"message": "book not found",
})
} else {
p := c.fixPath(book.Formats[0])
info, reader, _ := c.getFile(p)
info, reader, _ := c.getFile(book.FilePath)
defer reader.Close()
r.DataFromReader(http.StatusOK, info.Size(), "", reader, nil)
r.DataFromReader(http.StatusOK, info.Size(), "application/epub+zip", reader, nil)
}
}

Expand All @@ -253,7 +255,11 @@ func (c Api) getCover(r *gin.Context) {
err := c.bookIndex.GetDocument(id, nil, &book)

if err != nil {
r.JSON(http.StatusInternalServerError, err)
log.Println(err)
r.JSON(http.StatusInternalServerError, gin.H{
"code": 500,
"message": "book not found",
})
} else {
info, reader, _ := c.getFile(c.fixPath(book.Cover))
defer reader.Close()
Expand Down Expand Up @@ -286,3 +292,46 @@ func (c Api) getFileOrCache(filepath string, id string) (string, error) {
}
return filename, err
}

func (c Api) getDbFileOrCache() (string, error) {
filename := path.Join(c.baseDir, "metadata.db")
_, err := os.Stat(filename)
if Exists(filename) {
return filename, nil
}
_, closer, err := c.getFile("metadata.db")
if err != nil {
return "", err
}
b, err := ioutil.ReadAll(closer)
if err != nil {
return "", err
}
closer.Close()

f, err := os.Create(filename)
defer f.Close()
if err != nil {
fmt.Println(err.Error())
} else {
_, err = f.Write(b)
}
return filename, err
}

func (c Api) updateIndex(c2 *gin.Context) {
dbPath, err := c.getDbFileOrCache()
newDb, _ := NewDb(dbPath)
books, _ := newDb.queryBooks()
println(len(books))
_, err = c.bookIndex.UpdateDocumentsInBatches(books, 20)
if err != nil {
log.Println(err)
c2.JSON(http.StatusInternalServerError, err)
return
}
c2.JSON(http.StatusOK, gin.H{
"code": 200,
"message": "success",
})
}
95 changes: 95 additions & 0 deletions internal/calibre/sqlite.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package calibre

import (
"database/sql"
)
import _ "github.com/mattn/go-sqlite3"

const (
queryAllBooks = `SELECT
books.id AS id,
books.timestamp AS last_modified,
books.pubdate AS pubdate,
books.title AS title,
group_concat(DISTINCT authors.name) AS authors,
group_concat(DISTINCT authors.sort) AS authors_sort,
group_concat(DISTINCT publishers.name) AS publisher,
books.path || '/' || data.name || '.epub' AS file_path,
books.path || '/' || 'cover.jpg' AS cover,
data.uncompressed_size AS size,
identifiers.val AS isbn
FROM
books
LEFT JOIN
books_authors_link ON books.id = books_authors_link.book
LEFT JOIN
authors ON books_authors_link.author = authors.id
LEFT JOIN
books_publishers_link ON books.id = books_publishers_link.book
LEFT JOIN
publishers ON books_publishers_link.publisher = publishers.id
LEFT JOIN
data ON books.id = data.book AND data.format = 'EPUB'
LEFT JOIN
identifiers ON books.id = identifiers.book AND identifiers.type = 'isbn'
GROUP BY
books.id`
)

type Db struct {
dbPath string
db *sql.DB
}

func NewDb(dbPath string) (*Db, error) {
db, err := getDb(dbPath)
if err != nil {
return nil, err
}
return &Db{dbPath: dbPath, db: db}, nil
}

func getDb(sqlitePath string) (*sql.DB, error) {
db, err := sql.Open("sqlite3", sqlitePath)
if err != nil {
return nil, err
}
return db, nil
}

func (d *Db) Close() error {
return d.db.Close()
}

func (d Db) queryBooks() (books []Book, err error) {
rows, err := d.db.Query(queryAllBooks)
if err != nil {
return nil, err
}
defer rows.Close()
// rows to books
for rows.Next() {
var book BookRaw
if err := rows.Scan(&book.ID, &book.LastModified, &book.Pubdate, &book.Title, &book.Authors, &book.AuthorSort, &book.Publisher, &book.FilePath, &book.Cover, &book.Size, &book.Isbn); err != nil {
return nil, err
}

// convert BookRaw to Book
newBook := Book{
ID: book.ID,
AuthorSort: book.AuthorSort,
Authors: book.Authors,
Cover: book.Cover,
FilePath: book.FilePath,
Isbn: book.Isbn.String,
LastModified: book.LastModified,
Pubdate: book.Pubdate,
Publisher: book.Publisher.String,
Size: book.Size,
Tags: []string{},
Title: book.Title,
}
books = append(books, newBook)
}
return books, nil
}
35 changes: 25 additions & 10 deletions internal/calibre/types.go
Original file line number Diff line number Diff line change
@@ -1,18 +1,16 @@
package calibre

import (
"database/sql"
"time"
)

type Book struct {
AuthorSort string `json:"author_sort"`
Authors string `json:"authors"`
Cover string `json:"cover"`
Formats []string `json:"formats"`
ID int64 `json:"id"`
Identifiers struct {
MobiAsin string `json:"mobi-asin"`
} `json:"identifiers"`
AuthorSort string `json:"author_sort"`
Authors string `json:"authors"`
Cover string `json:"cover"`
FilePath string `json:"file_path"`
ID int64 `json:"id"`
Isbn string `json:"isbn"`
Languages []string `json:"languages"`
LastModified time.Time `json:"last_modified"`
Expand All @@ -21,9 +19,26 @@ type Book struct {
SeriesIndex float64 `json:"series_index"`
Size int64 `json:"size"`
Tags []string `json:"tags"`
Timestamp time.Time `json:"timestamp"`
Title string `json:"title"`
UUID string `json:"uuid"`
}

type BookRaw struct {
AuthorSort string `json:"author_sort"`
Authors string `json:"authors"`
Cover string `json:"cover"`
FilePath string `json:"file_path"`
ID int64 `json:"id"`
Isbn sql.NullString `json:"isbn"`
Languages []string `json:"languages"`
LastModified time.Time `json:"last_modified"`
Pubdate time.Time `json:"pubdate"`
Publisher sql.NullString `json:"publisher"`
SeriesIndex float64 `json:"series_index"`
Size int64 `json:"size"`
Tags []string `json:"tags"`
Timestamp time.Time `json:"timestamp"`
Title string `json:"title"`
UUID string `json:"uuid"`
}

type Config struct {
Expand Down

0 comments on commit 7b7e080

Please sign in to comment.