diff --git a/README.md b/README.md index 7a8024b..8c1103d 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ GET /read/:id/file/*path --> 读取书籍中的文件 GET /book/:id --> 获取书籍信息 GET /search --> 搜索书籍(参数q 搜索词，更多参数参考[meilisearch search](https://docs.meilisearch.com/reference/api/search.html)) POST /search --> 搜索书籍(参数q 搜索词，更多参数参考[meilisearch search](https://docs.meilisearch.com/reference/api/search.html)) +POST /index/update --> 读取calibre数据库更新索引 ``` ## 数据导入 @@ -39,11 +40,12 @@ curl -X "PATCH" "http://localhost:7700/indexes/books/settings" \ ], "filterableAttributes": [ "authors", - "formats", + "file_path", "id", "last_modified", "pubdate", "publisher", + "isbn", "tags" ], "searchableAttributes": [ @@ -60,16 +62,9 @@ curl -X "PATCH" "http://localhost:7700/indexes/books/settings" \ }' ``` -使用calibredb将数据导出为json， -然后上传到meilisearch中 - +使用下面命令更新索引 ```shell -maxID=(curl -s "http://localhost:7700/indexes/books/search?q&limit=1&sort=id%3Adesc&attributesToRetrieve=id" | jq '.hits[0].id') -calibredb --with-library=library list -f all --for-machine --search="id:>$maxID" >> data.json -curl \ - -X PUT 'http://localhost:7700/indexes/books/documents' \ - -H 'Content-Type: application/json' \ - --data-binary @data.json +curl -X "POST" "http://localhost:8080/index/update" -H 'Content-Type: application/json' ``` ## 接口 @@ -199,4 +194,4 @@ CALIBRE_STORAGE_WEBDAV_PATH "content": "//body" } } -``` +``` \ No newline at end of file diff --git a/go.mod b/go.mod index 203800c..2393db9 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/jedib0t/go-pretty/v6 v6.4.3 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 github.com/kapmahc/epub v0.1.1 + github.com/mattn/go-sqlite3 v1.14.22 github.com/meilisearch/meilisearch-go v0.22.0 github.com/minio/minio-go/v7 v7.0.45 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db diff --git a/go.sum b/go.sum index 811bf49..6acdb21 100644 --- a/go.sum +++ b/go.sum @@ -196,6 +196,8 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/meilisearch/meilisearch-go v0.22.0 h1:1J8VO5M8+WpGCOAjeUZy239RsLMWyRxqC2oN703J1cM= github.com/meilisearch/meilisearch-go v0.22.0/go.mod h1:XmVwi0ZyCdkEQ4cQvA3nh5TT0UByux4kBEWs4WUEp20= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= diff --git a/internal/calibre/api.go b/internal/calibre/api.go index 39a3d1c..ab47730 100644 --- a/internal/calibre/api.go +++ b/internal/calibre/api.go @@ -34,6 +34,7 @@ func (c Api) SetupRouter(r *gin.Engine) { r.GET("/book/:id", c.getBook) r.GET("/search", c.search) r.POST("/search", c.search) + r.POST("/index/update", c.updateIndex) } func NewClient(config *Config) Api { @@ -100,10 +101,7 @@ func (c Api) search(r *gin.Context) { } id := book.ID book.Cover = "/get/cover/" + strconv.FormatInt(id, 10) + ".jpg" - for i := range book.Formats { - s := book.Formats[i] - book.Formats[i] = "/get/book/" + strconv.FormatInt(id, 10) + path.Ext(s) - } + book.FilePath = "/get/book/" + strconv.FormatInt(id, 10) + ".epub" books[i] = book } @@ -127,13 +125,12 @@ func (c Api) getBook(r *gin.Context) { err := c.bookIndex.GetDocument(id, nil, &book) if err != nil { - r.JSON(http.StatusInternalServerError, err) + // 返回文件找不到 + r.JSON(http.StatusNotFound, "book not found") + return } book.Cover = "/get/cover/" + id + ".jpg" - for i := range book.Formats { - s := book.Formats[i] - book.Formats[i] = "/get/book/" + id + path.Ext(s) - } + book.FilePath = "/get/book/" + id + ".epub" r.JSON(http.StatusOK, book) } @@ -145,7 +142,7 @@ func (c Api) getBookToc(r *gin.Context) { if err != nil { r.JSON(http.StatusInternalServerError, err) } else { - filepath, _ := c.getFileOrCache(c.fixPath(book.Formats[0]), id) + filepath, _ := c.getFileOrCache(book.FilePath, id) book, _ := epub.Open(filepath) points := c.expansionTree(book.Ncx.Points) var p []epub.NavPoint @@ -196,8 +193,7 @@ func (c Api) getBookContent(r *gin.Context) { if err != nil { r.JSON(http.StatusInternalServerError, err) } else { - p := c.fixPath(book.Formats[0]) - filepath, _ := c.getFileOrCache(p, id) + filepath, _ := c.getFileOrCache(book.FilePath, id) destDir := path.Join(c.baseDir, id) @@ -212,7 +208,10 @@ func (c Api) getBookContent(r *gin.Context) { err := unzipSource(filepath, destDir) if err != nil { - r.JSON(http.StatusInternalServerError, err) + r.JSON(http.StatusInternalServerError, gin.H{ + "code": 500, + "message": err.Error(), + }) } r.FileFromFS(path1, http.Dir(destDir)) } @@ -237,12 +236,15 @@ func (c Api) getBookFile(r *gin.Context) { var book Book err := c.bookIndex.GetDocument(id, nil, &book) if err != nil { - r.JSON(http.StatusInternalServerError, err) + log.Println(err) + r.JSON(http.StatusInternalServerError, gin.H{ + "code": 500, + "message": "book not found", + }) } else { - p := c.fixPath(book.Formats[0]) - info, reader, _ := c.getFile(p) + info, reader, _ := c.getFile(book.FilePath) defer reader.Close() - r.DataFromReader(http.StatusOK, info.Size(), "", reader, nil) + r.DataFromReader(http.StatusOK, info.Size(), "application/epub+zip", reader, nil) } } @@ -253,7 +255,11 @@ func (c Api) getCover(r *gin.Context) { err := c.bookIndex.GetDocument(id, nil, &book) if err != nil { - r.JSON(http.StatusInternalServerError, err) + log.Println(err) + r.JSON(http.StatusInternalServerError, gin.H{ + "code": 500, + "message": "book not found", + }) } else { info, reader, _ := c.getFile(c.fixPath(book.Cover)) defer reader.Close() @@ -286,3 +292,46 @@ func (c Api) getFileOrCache(filepath string, id string) (string, error) { } return filename, err } + +func (c Api) getDbFileOrCache() (string, error) { + filename := path.Join(c.baseDir, "metadata.db") + _, err := os.Stat(filename) + if Exists(filename) { + return filename, nil + } + _, closer, err := c.getFile("metadata.db") + if err != nil { + return "", err + } + b, err := ioutil.ReadAll(closer) + if err != nil { + return "", err + } + closer.Close() + + f, err := os.Create(filename) + defer f.Close() + if err != nil { + fmt.Println(err.Error()) + } else { + _, err = f.Write(b) + } + return filename, err +} + +func (c Api) updateIndex(c2 *gin.Context) { + dbPath, err := c.getDbFileOrCache() + newDb, _ := NewDb(dbPath) + books, _ := newDb.queryBooks() + println(len(books)) + _, err = c.bookIndex.UpdateDocumentsInBatches(books, 20) + if err != nil { + log.Println(err) + c2.JSON(http.StatusInternalServerError, err) + return + } + c2.JSON(http.StatusOK, gin.H{ + "code": 200, + "message": "success", + }) +} diff --git a/internal/calibre/sqlite.go b/internal/calibre/sqlite.go new file mode 100644 index 0000000..a30bc34 --- /dev/null +++ b/internal/calibre/sqlite.go @@ -0,0 +1,95 @@ +package calibre + +import ( + "database/sql" +) +import _ "github.com/mattn/go-sqlite3" + +const ( + queryAllBooks = `SELECT + books.id AS id, + books.timestamp AS last_modified, + books.pubdate AS pubdate, + books.title AS title, + group_concat(DISTINCT authors.name) AS authors, + group_concat(DISTINCT authors.sort) AS authors_sort, + group_concat(DISTINCT publishers.name) AS publisher, + books.path || '/' || data.name || '.epub' AS file_path, + books.path || '/' || 'cover.jpg' AS cover, + data.uncompressed_size AS size, + identifiers.val AS isbn +FROM + books +LEFT JOIN + books_authors_link ON books.id = books_authors_link.book +LEFT JOIN + authors ON books_authors_link.author = authors.id +LEFT JOIN + books_publishers_link ON books.id = books_publishers_link.book +LEFT JOIN + publishers ON books_publishers_link.publisher = publishers.id +LEFT JOIN + data ON books.id = data.book AND data.format = 'EPUB' +LEFT JOIN + identifiers ON books.id = identifiers.book AND identifiers.type = 'isbn' +GROUP BY + books.id` +) + +type Db struct { + dbPath string + db *sql.DB +} + +func NewDb(dbPath string) (*Db, error) { + db, err := getDb(dbPath) + if err != nil { + return nil, err + } + return &Db{dbPath: dbPath, db: db}, nil +} + +func getDb(sqlitePath string) (*sql.DB, error) { + db, err := sql.Open("sqlite3", sqlitePath) + if err != nil { + return nil, err + } + return db, nil +} + +func (d *Db) Close() error { + return d.db.Close() +} + +func (d Db) queryBooks() (books []Book, err error) { + rows, err := d.db.Query(queryAllBooks) + if err != nil { + return nil, err + } + defer rows.Close() + // rows to books + for rows.Next() { + var book BookRaw + if err := rows.Scan(&book.ID, &book.LastModified, &book.Pubdate, &book.Title, &book.Authors, &book.AuthorSort, &book.Publisher, &book.FilePath, &book.Cover, &book.Size, &book.Isbn); err != nil { + return nil, err + } + + // convert BookRaw to Book + newBook := Book{ + ID: book.ID, + AuthorSort: book.AuthorSort, + Authors: book.Authors, + Cover: book.Cover, + FilePath: book.FilePath, + Isbn: book.Isbn.String, + LastModified: book.LastModified, + Pubdate: book.Pubdate, + Publisher: book.Publisher.String, + Size: book.Size, + Tags: []string{}, + Title: book.Title, + } + books = append(books, newBook) + } + return books, nil +} diff --git a/internal/calibre/types.go b/internal/calibre/types.go index 5c7a4a9..7d00aef 100644 --- a/internal/calibre/types.go +++ b/internal/calibre/types.go @@ -1,18 +1,16 @@ package calibre import ( + "database/sql" "time" ) type Book struct { - AuthorSort string `json:"author_sort"` - Authors string `json:"authors"` - Cover string `json:"cover"` - Formats []string `json:"formats"` - ID int64 `json:"id"` - Identifiers struct { - MobiAsin string `json:"mobi-asin"` - } `json:"identifiers"` + AuthorSort string `json:"author_sort"` + Authors string `json:"authors"` + Cover string `json:"cover"` + FilePath string `json:"file_path"` + ID int64 `json:"id"` Isbn string `json:"isbn"` Languages []string `json:"languages"` LastModified time.Time `json:"last_modified"` @@ -21,9 +19,26 @@ type Book struct { SeriesIndex float64 `json:"series_index"` Size int64 `json:"size"` Tags []string `json:"tags"` - Timestamp time.Time `json:"timestamp"` Title string `json:"title"` - UUID string `json:"uuid"` +} + +type BookRaw struct { + AuthorSort string `json:"author_sort"` + Authors string `json:"authors"` + Cover string `json:"cover"` + FilePath string `json:"file_path"` + ID int64 `json:"id"` + Isbn sql.NullString `json:"isbn"` + Languages []string `json:"languages"` + LastModified time.Time `json:"last_modified"` + Pubdate time.Time `json:"pubdate"` + Publisher sql.NullString `json:"publisher"` + SeriesIndex float64 `json:"series_index"` + Size int64 `json:"size"` + Tags []string `json:"tags"` + Timestamp time.Time `json:"timestamp"` + Title string `json:"title"` + UUID string `json:"uuid"` } type Config struct {