From 0675f72f4f13a5a72a132a6a901316afd1ed4ebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Mon, 20 Apr 2020 18:00:01 +0200 Subject: [PATCH] Updated examples and README for 2.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Michał Matczuk --- Makefile | 4 + README.md | 161 ++++---- doc.go | 9 +- example_test.go | 812 ++++++++++++++++++++++++++++++--------- go.mod | 1 + go.sum | 2 + gocqlxtest/gocqlxtest.go | 7 +- queryx.go | 7 + table/example_test.go | 95 ----- 9 files changed, 747 insertions(+), 351 deletions(-) delete mode 100644 table/example_test.go diff --git a/Makefile b/Makefile index 8f213d0..f2a0e0f 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,10 @@ test: bench: @go test -cpu $(GOTEST_CPU) -tags all -run=XXX -bench=. -benchmem ./... +.PHONY: run-examples +run-examples: + @go test -tags all -v -run=Example + .PHONY: run-scylla run-scylla: @echo "==> Running test instance of Scylla $(SCYLLA_VERSION)" diff --git a/README.md b/README.md index 107b279..025d45b 100644 --- a/README.md +++ b/README.md @@ -1,104 +1,118 @@ -# GoCQLX [![GoDoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](http://godoc.org/github.com/scylladb/gocqlx) [![Go Report Card](https://goreportcard.com/badge/github.com/scylladb/gocqlx)](https://goreportcard.com/report/github.com/scylladb/gocqlx) [![Build Status](https://travis-ci.org/scylladb/gocqlx.svg?branch=master)](https://travis-ci.org/scylladb/gocqlx) + # 🚀 GocqlX [![GoDoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](http://godoc.org/github.com/scylladb/gocqlx) [![Go Report Card](https://goreportcard.com/badge/github.com/scylladb/gocqlx)](https://goreportcard.com/report/github.com/scylladb/gocqlx) [![Build Status](https://travis-ci.org/scylladb/gocqlx.svg?branch=master)](https://travis-ci.org/scylladb/gocqlx) -Package `gocqlx` is an idiomatic extension to `gocql` that provides usability features. With gocqlx you can bind the query parameters from maps and structs, use named query parameters (:identifier) and scan the query results into structs and slices. It comes with a fluent and flexible CQL query builder and a database migrations module. +GocqlX makes working with Scylla easy and error prone without sacrificing performance. +It’s inspired by [Sqlx](https://github.com/jmoiron/sqlx), a tool for working with SQL databases, but it goes beyond what Sqlx provides. -## Installation +## Features - go get -u github.com/scylladb/gocqlx +* Binding query parameters from struct fields, map, or both +* Scanning query results into structs based on field names +* Convenient functions for common tasks such as loading a single row into a struct or all rows into a slice (list) of structs +* Making any struct a UDT without implementing marshalling functions +* GocqlX is fast. Its performance is comparable to raw driver. You can find some benchmarks [here](#performance). -## Features +Subpackages provide additional functionality: -* Binding query parameters form struct -* Scanning results into struct or slice -* Automated UDT support -* CRUD operations based on table model ([package table](https://github.com/scylladb/gocqlx/blob/master/table)) * CQL query builder ([package qb](https://github.com/scylladb/gocqlx/blob/master/qb)) +* CRUD operations based on table model ([package table](https://github.com/scylladb/gocqlx/blob/master/table)) * Database migrations ([package migrate](https://github.com/scylladb/gocqlx/blob/master/migrate)) -* Fast! -## Training and Scylla University +## Installation + + go get -u github.com/scylladb/gocqlx + +## Getting started -[Scylla University](https://university.scylladb.com/) includes training material and online courses which will help you become a Scylla NoSQL database expert. -The course [Using Scylla Drivers](https://university.scylladb.com/courses/using-scylla-drivers/) explains how to use drivers in different languages to interact with a Scylla cluster. -The lesson, [Golang and Scylla Part 3](https://university.scylladb.com/courses/using-scylla-drivers/lessons/golang-and-scylla-part-3-gocqlx/) includes a sample application that uses the GoCQXL package. -It connects to a Scylla cluster, displays the contents of a table, inserts and deletes data, and shows the contents of the table after each action. -Courses in [Scylla University](https://university.scylladb.com/) cover a variety of topics dealing with Scylla data modeling, administration, architecture and also covering some basic NoSQL concepts. +Wrap gocql Session: -## Example +```go +// Create gocql cluster. +cluster := gocql.NewCluster(hosts...) +// Wrap session on creation, gocqlx session embeds gocql.Session pointer. +session, err := gocqlx.WrapSession(cluster.CreateSession()) +if err != nil { + t.Fatal(err) +} +``` + +Specify table model: ```go +// metadata specifies table name and columns it must be in sync with schema. +var personMetadata = table.Metadata{ + Name: "person", + Columns: []string{"first_name", "last_name", "email"}, + PartKey: []string{"first_name"}, + SortKey: []string{"last_name"}, +} + +// personTable allows for simple CRUD operations based on personMetadata. +var personTable = table.New(personMetadata) + // Person represents a row in person table. // Field names are converted to camel case by default, no need to add special tags. // If you want to disable a field add `db:"-"` tag, it will not be persisted. type Person struct { - FirstName string - LastName string - Email []string + FirstName string + LastName string + Email []string } +``` -// Insert, bind data from struct. -{ - stmt, names := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email").ToCql() - q := gocqlx.Query(session.Query(stmt), names).BindStruct(p) +Bind data from a struct and insert a row: - if err := q.ExecRelease(); err != nil { - t.Fatal(err) - } -} -// Get first result into a struct. -{ - var p Person - stmt, names := qb.Select("gocqlx_test.person").Where(qb.Eq("first_name")).ToCql() - q := gocqlx.Query(session.Query(stmt), names).BindMap(qb.M{ - "first_name": "Patricia", - }) - if err := q.GetRelease(&p); err != nil { - t.Fatal(err) - } +```go +p := Person{ + "Michał", + "Matczuk", + []string{"michal@scylladb.com"}, } -// Load all the results into a slice. -{ - var people []Person - stmt, names := qb.Select("gocqlx_test.person").Where(qb.In("first_name")).ToCql() - q := gocqlx.Query(session.Query(stmt), names).BindMap(qb.M{ - "first_name": []string{"Patricia", "Igy", "Ian"}, - }) - if err := q.SelectRelease(&people); err != nil { - t.Fatal(err) - } +q := session.Query(personTable.Insert()).BindStruct(p) +if err := q.ExecRelease(); err != nil { + t.Fatal(err) } +``` -// metadata specifies table name and columns it must be in sync with schema. -var personMetadata = table.Metadata{ - Name: "person", - Columns: []string{"first_name", "last_name", "email"}, - PartKey: []string{"first_name"}, - SortKey: []string{"last_name"}, +Load a single row to a struct: + +```go +p := Person{ + "Michał", + "Matczuk", + nil, // no email +} +q := session.Query(personTable.Get()).BindStruct(p) +if err := q.GetRelease(&p); err != nil { + t.Fatal(err) } +t.Log(p) +// stdout: {Michał Matczuk [michal@scylladb.com]} +``` -// personTable allows for simple CRUD operations based on personMetadata. -var personTable = table.New(personMetadata) +Load all rows in to a slice: -// Get by primary key. -{ - p := Person{ - "Patricia", - "Citizen", - nil, // no email - } - stmt, names := personTable.Get() // you can filter columns too - q := gocqlx.Query(session.Query(stmt), names).BindStruct(p) - if err := q.GetRelease(&p); err != nil { - t.Fatal(err) - } +```go +var people []Person +q := session.Query(personTable.Select()).BindMap(qb.M{"first_name": "Michał"}) +if err := q.SelectRelease(&people); err != nil { + t.Fatal(err) } +t.Log(people) +// stdout: [{Michał Matczuk [michal@scylladb.com]}] ``` -See more examples in [example_test.go](https://github.com/scylladb/gocqlx/blob/master/example_test.go) and [table/example_test.go](https://github.com/scylladb/gocqlx/blob/master/table/example_test.go). +## Examples + +You can find lots of other examples in [example_test.go](https://github.com/scylladb/gocqlx/blob/master/example_test.go), go and run the examples locally: + +```bash +make run-scylla +make run-examples +``` ## Performance -With regards to performance `gocqlx` package is comparable to the raw `gocql` baseline. +GocqlX performance is comparable to the raw `gocql` driver. Below benchmark results running on my laptop. ``` @@ -110,7 +124,12 @@ BenchmarkBaseGocqlSelect 747 1664365 ns/op 49415 BenchmarkGocqlxSelect 667 1877859 ns/op 42521 B/op 932 allocs/op ``` -See the benchmark in [benchmark_test.go](https://github.com/scylladb/gocqlx/blob/master/benchmark_test.go). +See the benchmark in [benchmark_test.go](https://github.com/scylladb/gocqlx/blob/master/benchmark_test.go), you can run the benchmark locally: + +```bash +make run-scylla +make bench +``` ## License diff --git a/doc.go b/doc.go index f1fbd77..33a36df 100644 --- a/doc.go +++ b/doc.go @@ -2,9 +2,8 @@ // Use of this source code is governed by a ALv2-style // license that can be found in the LICENSE file. -// Package gocqlx is an idiomatic extension to gocql that provides usability -// features. With gocqlx you can bind the query parameters from maps and -// structs, use named query parameters (:identifier) and scan the query results -// into structs and slices. It comes with a fluent and flexible CQL query -// builder and a database migrations module. +// Package gocqlx makes working with Scylla easy and error prone without sacrificing performance. +// It’s inspired by Sqlx, a tool for working with SQL databases, but it goes beyond what Sqlx provides. +// +// For more details consult README. package gocqlx diff --git a/example_test.go b/example_test.go index 8ffeb3e..1d87741 100644 --- a/example_test.go +++ b/example_test.go @@ -7,252 +7,710 @@ package gocqlx_test import ( + "fmt" + "math" "testing" "time" + "github.com/gocql/gocql" "github.com/scylladb/gocqlx" - . "github.com/scylladb/gocqlx/gocqlxtest" + "github.com/scylladb/gocqlx/gocqlxtest" "github.com/scylladb/gocqlx/qb" + "github.com/scylladb/gocqlx/table" + "golang.org/x/sync/errgroup" ) +// Running examples locally: +// make run-scylla +// make run-examples func TestExample(t *testing.T) { - session := CreateSession(t) + cluster := gocqlxtest.CreateCluster() + + session, err := gocqlx.WrapSession(cluster.CreateSession()) + if err != nil { + t.Fatal("create session:", err) + } defer session.Close() - const personSchema = ` -CREATE TABLE IF NOT EXISTS gocqlx_test.person ( - first_name text, - last_name text, - email list, - salary int, - PRIMARY KEY(first_name, last_name) -)` + session.ExecStmt(`DROP KEYSPACE examples`) + + basicCreateAndPopulateKeyspace(t, session) + basicReadScyllaVersion(t, session) + + datatypesBlob(t, session) + datatypesUserDefinedType(t, session) + datatypesUserDefinedTypeWrapper(t, session) + datatypesJson(t, session) + + pagingForwardPaging(t, session) + pagingEfficientFullTableScan(t, session) - if err := session.ExecStmt(personSchema); err != nil { + lwtLock(t, session) +} + +// This example shows how to use query builders and table models to build +// queries. It uses "BindStruct" function for parameter binding and "Select" +// function for loading data to a slice. +func basicCreateAndPopulateKeyspace(t *testing.T, session gocqlx.Session) { + err := session.ExecStmt(`CREATE KEYSPACE IF NOT EXISTS examples WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}`) + if err != nil { + t.Fatal("create keyspace:", err) + } + + type Song struct { + ID gocql.UUID + Title string + Album string + Artist string + Tags []string + Data []byte + } + + type PlaylistItem struct { + ID gocql.UUID + Title string + Album string + Artist string + SongID gocql.UUID + } + + err = session.ExecStmt(`CREATE TABLE IF NOT EXISTS examples.songs ( + id uuid PRIMARY KEY, + title text, + album text, + artist text, + tags set, + data blob)`) + if err != nil { t.Fatal("create table:", err) } - // Person represents a row in person table. - // Field names are converted to camel case by default, no need to add special tags. - // If you want to disable a field add `db:"-"` tag, it will not be persisted. - type Person struct { - FirstName string - LastName string - Email []string - Salary int + err = session.ExecStmt(`CREATE TABLE IF NOT EXISTS examples.playlists ( + id uuid, + title text, + album text, + artist text, + song_id uuid, + PRIMARY KEY (id, title, album, artist))`) + if err != nil { + t.Fatal("create table:", err) } - p := Person{ - "Patricia", - "Citizen", - []string{"patricia.citzen@gocqlx_test.com"}, - 500, + playlistMetadata := table.Metadata{ + Name: "examples.playlists", + Columns: []string{"id", "title", "album", "artist", "song_id"}, + PartKey: []string{"id"}, + SortKey: []string{"title", "album", "artist", "song_id"}, + } + playlistTable := table.New(playlistMetadata) + + // Insert song using query builder. + stmt, names := qb.Insert("examples.songs"). + Columns("id", "title", "album", "artist", "tags", "data").ToCql() + insertSong := session.Query(stmt, names) + + insertSong.BindStruct(Song{ + ID: mustParseUUID("756716f7-2e54-4715-9f00-91dcbea6cf50"), + Title: "La Petite Tonkinoise", + Album: "Bye Bye Blackbird", + Artist: "Joséphine Baker", + Tags: []string{"jazz", "2013"}, + Data: []byte("music"), + }) + if err := insertSong.ExecRelease(); err != nil { + t.Fatal("ExecRelease() failed:", err) } - // Insert, bind data from struct. - { - stmt, names := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email").ToCql() - q := session.Query(stmt, names).BindStruct(p) + // Insert playlist using table model. + insertPlaylist := session.Query(playlistTable.Insert()) + + insertPlaylist.BindStruct(PlaylistItem{ + ID: mustParseUUID("2cc9ccb7-6221-4ccb-8387-f22b6a1b354d"), + Title: "La Petite Tonkinoise", + Album: "Bye Bye Blackbird", + Artist: "Joséphine Baker", + SongID: mustParseUUID("756716f7-2e54-4715-9f00-91dcbea6cf50"), + }) + if err := insertPlaylist.ExecRelease(); err != nil { + t.Fatal("ExecRelease() failed:", err) + } - if err := q.ExecRelease(); err != nil { - t.Fatal(err) - } + // Query and displays data. + queryPlaylist := session.Query(playlistTable.Select()) + + queryPlaylist.BindStruct(&PlaylistItem{ + ID: mustParseUUID("2cc9ccb7-6221-4ccb-8387-f22b6a1b354d"), + }) + + var items []*PlaylistItem + if err := queryPlaylist.Select(&items); err != nil { + t.Fatal("Select() failed:", err) } - // Insert with TTL and timestamp, bind data from struct and map. - { - stmt, names := qb.Insert("gocqlx_test.person"). - Columns("first_name", "last_name", "email"). - TTL(86400 * time.Second). - Timestamp(time.Now()). - ToCql() - q := session.Query(stmt, names).BindStruct(p) + for _, i := range items { + t.Logf("%+v", *i) + } +} - if err := q.ExecRelease(); err != nil { - t.Fatal(err) - } +// This example shows how to load a single value using "Get" function. +// Get can also work with UDTs and types that implement gocql marshalling functions. +func basicReadScyllaVersion(t *testing.T, session gocqlx.Session) { + var releaseVersion string + + err := session.Query("SELECT release_version FROM system.local", nil).Get(&releaseVersion) + if err != nil { + t.Fatal("Get() failed:", err) } - // Update email, bind data from struct. - { - p.Email = append(p.Email, "patricia1.citzen@gocqlx_test.com") + t.Logf("Scylla version is: %s", releaseVersion) +} - stmt, names := qb.Update("gocqlx_test.person"). - Set("email"). - Where(qb.Eq("first_name"), qb.Eq("last_name")). - ToCql() - q := session.Query(stmt, names).BindStruct(p) +// This examples shows how to bind data from a map using "BindMap" function, +// override field name mapping using the "db" tags, and use "Unsafe" function +// to handle situations where driver returns more coluns that we are ready to +// consume. +func datatypesBlob(t *testing.T, session gocqlx.Session) { + err := session.ExecStmt(`CREATE KEYSPACE IF NOT EXISTS examples WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}`) + if err != nil { + t.Fatal("create keyspace:", err) + } - if err := q.ExecRelease(); err != nil { - t.Fatal(err) - } + err = session.ExecStmt(`CREATE TABLE IF NOT EXISTS examples.blobs(k int PRIMARY KEY, b blob, m map)`) + if err != nil { + t.Fatal("create table:", err) } - // Add email to a list. - { - stmt, names := qb.Update("gocqlx_test.person"). - AddNamed("email", "new_email"). - Where(qb.Eq("first_name"), qb.Eq("last_name")). - ToCql() - q := session.Query(stmt, names).BindStructMap(p, qb.M{ - "new_email": []string{"patricia2.citzen@gocqlx_test.com", "patricia3.citzen@gocqlx_test.com"}, - }) + // One way to get a byte buffer is to allocate it and fill it yourself: + var buf [16]byte + for i := range buf { + buf[i] = 0xff + } - if err := q.ExecRelease(); err != nil { - t.Fatal(err) - } + insert := session.Query(qb.Insert("examples.blobs").Columns("k", "b", "m").ToCql()) + insert.BindMap(qb.M{ + "k": 1, + "b": buf[:], + "m": map[string][]byte{"test": buf[:]}, + }) + + if err := insert.ExecRelease(); err != nil { + t.Fatal("ExecRelease() failed:", err) } - // Batch insert two rows in a single query. - { - i := qb.Insert("gocqlx_test.person").Columns("first_name", "last_name", "email") - - stmt, names := qb.Batch(). - AddWithPrefix("a", i). - AddWithPrefix("b", i). - ToCql() - - batch := struct { - A Person - B Person - }{ - A: Person{ - "Igy", - "Citizen", - []string{"igy.citzen@gocqlx_test.com"}, - 500, - }, - B: Person{ - "Ian", - "Citizen", - []string{"ian.citzen@gocqlx_test.com"}, - 500, - }, - } - q := session.Query(stmt, names).BindStruct(&batch) + row := &struct { + Buffer []byte `db:"b"` + Mapping map[string][]byte `db:"m"` + }{} + q := session.Query(qb.Select("examples.blobs").Where(qb.EqLit("k", "1")).ToCql()) - if err := q.ExecRelease(); err != nil { - t.Fatal(err) - } + // Unsafe is used here to override validation error that check if all + // requested columns are consumed `failed: missing destination name "k" in struct` error + if err := q.Iter().Unsafe().Get(row); err != nil { + t.Fatal("Get() failed:", err) } - // Get first result into a struct. - { - var p Person + t.Logf("%+v", row.Buffer) + t.Logf("%+v", row.Mapping) +} - stmt, names := qb.Select("gocqlx_test.person").Where(qb.Eq("first_name")).ToCql() - q := session.Query(stmt, names).BindMap(qb.M{ - "first_name": "Patricia", - }) +type Coordinates struct { + gocqlx.UDT + X int + Y int +} - if err := q.GetRelease(&p); err != nil { - t.Fatal(err) - } +// This example shows how to add User Defined Type marshalling capabilities by +// adding a single line - embedding gocqlx.UDT. +func datatypesUserDefinedType(t *testing.T, session gocqlx.Session) { + err := session.ExecStmt(`CREATE KEYSPACE IF NOT EXISTS examples WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}`) + if err != nil { + t.Fatal("create keyspace:", err) + } + + err = session.ExecStmt(`CREATE TYPE IF NOT EXISTS examples.coordinates(x int, y int)`) + if err != nil { + t.Fatal("create type:", err) + } - t.Log(p) - // stdout: {Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com patricia2.citzen@gocqlx_test.com patricia3.citzen@gocqlx_test.com]} + err = session.ExecStmt(`CREATE TABLE IF NOT EXISTS examples.udts(k int PRIMARY KEY, c coordinates)`) + if err != nil { + t.Fatal("create table:", err) } - // Load all the results into a slice. - { - var people []Person + coordinates1 := Coordinates{X: 12, Y: 34} + coordinates2 := Coordinates{X: 56, Y: 78} - stmt, names := qb.Select("gocqlx_test.person").Where(qb.In("first_name")).ToCql() - q := session.Query(stmt, names).BindMap(qb.M{ - "first_name": []string{"Patricia", "Igy", "Ian"}, - }) + insert := session.Query(qb.Insert("examples.udts").Columns("k", "c").ToCql()) + insert.BindMap(qb.M{ + "k": 1, + "c": coordinates1, + }) + if err := insert.Exec(); err != nil { + t.Fatal("Exec() failed:", err) + } + insert.BindMap(qb.M{ + "k": 2, + "c": coordinates2, + }) + if err := insert.Exec(); err != nil { + t.Fatal("Exec() failed:", err) + } - if err := q.SelectRelease(&people); err != nil { - t.Fatal(err) - } + var coordinates []Coordinates + q := session.Query(qb.Select("examples.udts").Columns("c").ToCql()) + if err := q.Select(&coordinates); err != nil { + t.Fatal("Select() failed:", err) + } - t.Log(people) - // stdout: [{Ian Citizen [ian.citzen@gocqlx_test.com]} {Igy Citizen [igy.citzen@gocqlx_test.com]} {Patricia Citizen [patricia.citzen@gocqlx_test.com patricia1.citzen@gocqlx_test.com patricia2.citzen@gocqlx_test.com patricia3.citzen@gocqlx_test.com]}] + for _, c := range coordinates { + t.Logf("%+v", c) } +} - // Support for token based pagination. - { - p := &Person{ - "Ian", - "Citizen", - []string{"ian.citzen@gocqlx_test.com"}, - 500, - } +type coordinates struct { + X int + Y int +} + +// This example shows how to add User Defined Type marshalling capabilities to +// types that we cannot modify, like library or transfer objects, without +// rewriting them in runtime. +func datatypesUserDefinedTypeWrapper(t *testing.T, session gocqlx.Session) { + err := session.ExecStmt(`CREATE KEYSPACE IF NOT EXISTS examples WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}`) + if err != nil { + t.Fatal("create keyspace:", err) + } + + err = session.ExecStmt(`CREATE TYPE IF NOT EXISTS examples.coordinates(x int, y int)`) + if err != nil { + t.Fatal("create type:", err) + } + + err = session.ExecStmt(`CREATE TABLE IF NOT EXISTS examples.udts_wrapper(k int PRIMARY KEY, c coordinates)`) + if err != nil { + t.Fatal("create table:", err) + } + + // Embed coordinates within CoordinatesUDT + c1 := &coordinates{X: 12, Y: 34} + c2 := &coordinates{X: 56, Y: 78} + + type CoordinatesUDT struct { + gocqlx.UDT + *coordinates + } + + coordinates1 := CoordinatesUDT{coordinates: c1} + coordinates2 := CoordinatesUDT{coordinates: c2} + + insert := session.Query(qb.Insert("examples.udts_wrapper").Columns("k", "c").ToCql()) + insert.BindMap(qb.M{ + "k": 1, + "c": coordinates1, + }) + if err := insert.Exec(); err != nil { + t.Fatal("Exec() failed:", err) + } + insert.BindMap(qb.M{ + "k": 2, + "c": coordinates2, + }) + if err := insert.Exec(); err != nil { + t.Fatal("Exec() failed:", err) + } + + var coordinates []Coordinates + q := session.Query(qb.Select("examples.udts_wrapper").Columns("c").ToCql()) + if err := q.Select(&coordinates); err != nil { + t.Fatal("Select() failed:", err) + } + + for _, c := range coordinates { + t.Logf("%+v", c) + } +} + +// This example shows how to use query builder to work with +func datatypesJson(t *testing.T, session gocqlx.Session) { + err := session.ExecStmt(`CREATE KEYSPACE IF NOT EXISTS examples WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}`) + if err != nil { + t.Fatal("create keyspace:", err) + } + + err = session.ExecStmt(`CREATE TABLE IF NOT EXISTS examples.querybuilder_json(id int PRIMARY KEY, name text, specs map)`) + if err != nil { + t.Fatal("create table:", err) + } + + insert := session.Query(qb.Insert("examples.querybuilder_json").Json().ToCql()) + + insert.Bind(`{ "id": 1, "name": "Mouse", "specs": { "color": "silver" } }`) + if err := insert.Exec(); err != nil { + t.Fatal("Exec() failed:", err) + } + insert.Bind(`{ "id": 2, "name": "Keyboard", "specs": { "layout": "qwerty" } }`) + if err := insert.Exec(); err != nil { + t.Fatal("Exec() failed:", err) + } + + // fromJson lets you provide individual columns as JSON: + stmt, names := qb.Insert("examples.querybuilder_json"). + Columns("id", "name"). + FuncColumn("specs", qb.Fn("fromJson", "json")). + ToCql() + + insertFromJson := session.Query(stmt, names) + insertFromJson.BindMap(qb.M{ + "id": 3, + "name": "Screen", + "json": `{ "size": "24-inch" }`, + }) + if err := insertFromJson.Exec(); err != nil { + t.Fatal("Exec() failed:", err) + } - stmt, names := qb.Select("gocqlx_test.person"). - Columns("first_name"). - Where(qb.Token("first_name").Gt()). - Limit(10). - ToCql() - q := session.Query(stmt, names).BindStruct(p) + // Reading the whole row as a JSON object: + stmt, names = qb.Select("examples.querybuilder_json"). + Json(). + Where(qb.EqLit("id", "1")). + ToCql() + q := session.Query(stmt, names) - var people []Person - if err := q.SelectRelease(&people); err != nil { - t.Fatal(err) + var jsonString string + + if err := q.Get(&jsonString); err != nil { + t.Fatal("Get() failed:", err) + } + t.Logf("Entry #1 as JSON: %s", jsonString) + + // Extracting a particular column as JSON: + stmt, names = qb.Select("examples.querybuilder_json"). + Columns("id", "toJson(specs) AS json_specs"). + Where(qb.EqLit("id", "2")). + ToCql() + q = session.Query(stmt, names) + + row := &struct { + ID int + JsonSpecs string + }{} + if err := q.Get(row); err != nil { + t.Fatal("Get() failed:", err) + } + t.Logf("Entry #%d's specs as JSON: %s", row.ID, row.JsonSpecs) +} + +type Video struct { + UserID int + UserName string + Added time.Time + VideoID int + Title string +} + +func pagingFillTable(t *testing.T, insert *gocqlx.Queryx) { + t.Helper() + + // 3 users + for i := 0; i < 3; i++ { + // 49 videos each + for j := 0; j < 49; j++ { + insert.BindStruct(Video{ + UserID: i, + UserName: fmt.Sprint("user ", i), + Added: time.Unix(int64(j)*100, 0), + VideoID: i*100 + j, + Title: fmt.Sprint("video ", i*100+j), + }) + + if err := insert.Exec(); err != nil { + t.Fatal("Exec() failed:", err) + } } + } +} + +// This example shows how to use stateful paging and how "Select" function +// can be used to fetch single page only. +func pagingForwardPaging(t *testing.T, session gocqlx.Session) { + err := session.ExecStmt(`CREATE KEYSPACE IF NOT EXISTS examples WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}`) + if err != nil { + t.Fatal("create keyspace:", err) + } + + err = session.ExecStmt(`CREATE TABLE IF NOT EXISTS examples.paging_forward_paging( + user_id int, + user_name text, + added timestamp, + video_id int, + title text, + PRIMARY KEY (user_id, added, video_id) + ) WITH CLUSTERING ORDER BY (added DESC, video_id ASC)`) + if err != nil { + t.Fatal("create table:", err) + } + + videoMetadata := table.Metadata{ + Name: "examples.paging_forward_paging", + Columns: []string{"user_id", "user_name", "added", "video_id", "title"}, + PartKey: []string{"user_id"}, + SortKey: []string{"added", "video_id"}, + } + videoTable := table.New(videoMetadata) + + pagingFillTable(t, session.Query(videoTable.Insert())) + + // Query and displays data. Iterate over videos of user "1" 10 entries per request. + + const itemsPerPage = 10 - t.Log(people) - // [{Patricia []} {Igy []}] + getUserVideos := func(userID int, page []byte) (userVideos []Video, nextPage []byte, err error) { + q := session.Query(videoTable.Select()).Bind(userID) + defer q.Release() + q.PageState(page) + q.PageSize(itemsPerPage) + + iter := q.Iter() + return userVideos, iter.PageState(), iter.Select(&userVideos) } - // Support for named parameters in query string. - { - const query = "INSERT INTO gocqlx_test.person (first_name, last_name, email) VALUES (:first_name, :last_name, :email)" - stmt, names, err := gocqlx.CompileNamedQuery([]byte(query)) + var ( + userVideos []Video + nextPage []byte + ) + + for i := 1; ; i++ { + userVideos, nextPage, err = getUserVideos(1, nextPage) if err != nil { - t.Fatal(err) + t.Fatalf("oad page %d: %s", i, err) } - p := &Person{ - "Jane", - "Citizen", - []string{"jane.citzen@gocqlx_test.com"}, - 500, + t.Logf("Page %d:", i) + for _, v := range userVideos { + t.Logf("%+v", v) + } + if len(nextPage) == 0 { + break } - q := session.Query(stmt, names).BindStruct(p) + } +} + +// This example shows how to efficiently process all rows in a table using +// the "token" function. It implements idea from blog post [1]: +// As a bonus we use "CompileNamedQueryString" to get named parameters out of +// CQL query placeholders like in Python or Java driver. +// +// [1] https://www.scylladb.com/2017/02/13/efficient-full-table-scans-with-scylla-1-6/. +func pagingEfficientFullTableScan(t *testing.T, session gocqlx.Session) { + err := session.ExecStmt(`CREATE KEYSPACE IF NOT EXISTS examples WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}`) + if err != nil { + t.Fatal("create keyspace:", err) + } + + err = session.ExecStmt(`CREATE TABLE IF NOT EXISTS examples.paging_efficient_full_table_scan( + user_id int, + user_name text, + added timestamp, + video_id int, + title text, + PRIMARY KEY (user_id, added, video_id) + ) WITH CLUSTERING ORDER BY (added DESC, video_id ASC)`) + if err != nil { + t.Fatal("create table:", err) + } - if err := q.ExecRelease(); err != nil { - t.Fatal(err) + videoMetadata := table.Metadata{ + Name: "examples.paging_efficient_full_table_scan", + Columns: []string{"user_id", "user_name", "added", "video_id", "title"}, + PartKey: []string{"user_id"}, + SortKey: []string{"added", "video_id"}, + } + videoTable := table.New(videoMetadata) + + pagingFillTable(t, session.Query(videoTable.Insert())) + + // Calculate optimal number of workers for the cluster: + var ( + nodesInCluster = 1 + coresInNode = 1 + smudgeFactor = 3 + ) + workers := nodesInCluster * coresInNode * smudgeFactor + + t.Logf("Workers %d", workers) + + type tokenRange struct { + Start int64 + End int64 + } + buf := make(chan tokenRange) + + // sequencer pushes token ranges to buf + sequencer := func() error { + span := int64(math.MaxInt64 / (50 * workers)) + + tr := tokenRange{math.MinInt64, math.MinInt64 + span} + for tr.End > tr.Start { + buf <- tr + tr.Start = tr.End + tr.End += span } + + tr.End = math.MaxInt64 + buf <- tr + close(buf) + + return nil } - // Support for Lightweight Transactions - { + // worker queries a token ranges generated by sequencer + worker := func() error { + const cql = `SELECT * FROM examples.paging_efficient_full_table_scan WHERE + token(user_id) >= :start AND + token(user_id) < :end` - p := Person{ - "Stephen", - "Johns", - []string{"stephen.johns@gocqlx_test.com"}, - 500, + stmt, names, err := gocqlx.CompileNamedQueryString(cql) + if err != nil { + return err + } + q := session.Query(stmt, names) + defer q.Release() + + var v Video + for { + tr, ok := <-buf + if !ok { + break + } + + iter := q.BindStruct(tr).Iter() + for iter.StructScan(&v) { + t.Logf("%+v:", v) + } + if err := iter.Close(); err != nil { + return err + } } - stmt, names := qb.Insert("gocqlx_test.person"). - Columns("first_name", "last_name", "email", "salary"). - Unique(). - ToCql() + return nil + } + + // Query and displays data. + + var wg errgroup.Group + wg.Go(sequencer) + for i := 0; i < workers; i++ { + wg.Go(worker) + } + + if err := wg.Wait(); err != nil { + t.Fatal(err) + } +} + +// This example shows how to use Lightweight Transactions (LWT) aka. +// Compare-And-Set (CAS) functions. +// See: https://docs.scylladb.com/using-scylla/lwt/ for more details. +func lwtLock(t *testing.T, session gocqlx.Session) { + err := session.ExecStmt(`CREATE KEYSPACE IF NOT EXISTS examples WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}`) + if err != nil { + t.Fatal("create keyspace:", err) + } + + type Lock struct { + Name string + Owner string + TTL int64 + } + + err = session.ExecStmt(`CREATE TABLE examples.lock (name text PRIMARY KEY, owner text)`) + if err != nil { + t.Fatal("create table:", err) + } + + extend := func(lock Lock) bool { + q := session.Query(qb.Update("examples.lock"). + Set("owner"). + Where(qb.Eq("name")). + If(qb.Eq("owner")). + TTLNamed("ttl"). + ToCql()) + q.BindStruct(lock) - applied, err := session.Query(stmt, names).BindStruct(p).ExecCASRelease() + applied, err := q.ExecCASRelease() if err != nil { - t.Fatal(err) + t.Fatal("ExecCASRelease() failed:", err) } + return applied + } + + acquire := func(lock Lock) (applied bool) { + var prev Lock - t.Log(applied) + defer func() { + t.Logf("Acquire %+v applied %v owner %+v)", lock, applied, prev) + }() - stmt, names = qb.Update("gocqlx_test.person"). - SetNamed("salary", "new_salary"). - Where(qb.Eq("first_name"), qb.Eq("last_name")). - If(qb.LtNamed("salary", "old_salary")). - ToCql() - q := session.Query(stmt, names).BindStructMap(&p, qb.M{ - "old_salary": 1000, - "new_salary": 1500, - }) + q := session.Query(qb.Insert("examples.lock"). + Columns("name", "owner"). + TTLNamed("ttl"). + Unique(). + ToCql(), + ) + q.BindStruct(lock) - applied, err = q.GetCAS(&p) + applied, err = q.GetCASRelease(&prev) if err != nil { - t.Fatal(err) + t.Fatal("GetCASRelease() failed:", err) + } + if applied { + return true + } + if prev.Owner == lock.Owner { + return extend(lock) } + return false + } + + const ( + resource = "acme" + ttl = time.Second + ) + + l1 := Lock{ + Name: resource, + Owner: "1", + TTL: qb.TTL(ttl), + } + + l2 := Lock{ + Name: resource, + Owner: "2", + TTL: qb.TTL(ttl), + } + + if !acquire(l1) { + t.Fatal("l1 failed to acquire lock") + } + if acquire(l2) { + t.Fatal("unexpectedly l2 acquired lock") + } + if !acquire(l1) { + t.Fatal("l1 failed to extend lock") + } + time.Sleep(time.Second) + if !acquire(l2) { + t.Fatal("l2 failed to acquire lock") + } + if acquire(l1) { + t.Fatal("unexpectedly l1 acquired lock") + } +} - t.Log(applied, p) +func mustParseUUID(s string) gocql.UUID { + u, err := gocql.ParseUUID(s) + if err != nil { + panic(err) } + return u } diff --git a/go.mod b/go.mod index 274f741..91e2927 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/golang/snappy v0.0.1 // indirect github.com/google/go-cmp v0.2.0 github.com/scylladb/go-reflectx v1.0.1 + golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a gopkg.in/inf.v0 v0.9.1 ) diff --git a/go.sum b/go.sum index db8d2b7..e0c1551 100644 --- a/go.sum +++ b/go.sum @@ -28,5 +28,7 @@ github.com/scylladb/go-reflectx v1.0.1/go.mod h1:rWnOfDIRWBGN0miMLIcoPt/Dhi2doCM github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/gocqlxtest/gocqlxtest.go b/gocqlxtest/gocqlxtest.go index 048f519..2f28dfb 100644 --- a/gocqlxtest/gocqlxtest.go +++ b/gocqlxtest/gocqlxtest.go @@ -28,13 +28,14 @@ var ( var initOnce sync.Once -// CreateSession creates a new gocql session from flags. +// CreateSession creates a new gocqlx session from flags. func CreateSession(tb testing.TB) gocqlx.Session { - cluster := createCluster() + cluster := CreateCluster() return createSessionFromCluster(cluster, tb) } -func createCluster() *gocql.ClusterConfig { +// CreateCluster creates gocql ClusterConfig from flags. +func CreateCluster() *gocql.ClusterConfig { if !flag.Parsed() { flag.Parse() } diff --git a/queryx.go b/queryx.go index 0a1f715..4501e6d 100644 --- a/queryx.go +++ b/queryx.go @@ -15,6 +15,13 @@ import ( "github.com/scylladb/go-reflectx" ) +// CompileNamedQueryString translates query with named parameters in a form +// ':' to query with '?' placeholders and a list of parameter names. +// If you need to use ':' in a query, i.e. with maps or UDTs use '::' instead. +func CompileNamedQueryString(qs string) (stmt string, names []string, err error) { + return CompileNamedQuery([]byte(qs)) +} + // CompileNamedQuery translates query with named parameters in a form // ':' to query with '?' placeholders and a list of parameter names. // If you need to use ':' in a query, i.e. with maps or UDTs use '::' instead. diff --git a/table/example_test.go b/table/example_test.go deleted file mode 100644 index 2a36189..0000000 --- a/table/example_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (C) 2017 ScyllaDB -// Use of this source code is governed by a ALv2-style -// license that can be found in the LICENSE file. - -// +build all integration - -package table_test - -import ( - "testing" - - . "github.com/scylladb/gocqlx/gocqlxtest" - "github.com/scylladb/gocqlx/qb" - "github.com/scylladb/gocqlx/table" -) - -func TestExample(t *testing.T) { - session := CreateSession(t) - defer session.Close() - - const personSchema = ` -CREATE TABLE IF NOT EXISTS gocqlx_test.person ( - first_name text, - last_name text, - email list, - PRIMARY KEY(first_name, last_name) -)` - if err := session.ExecStmt(personSchema); err != nil { - t.Fatal("create table:", err) - } - - // metadata specifies table name and columns it must be in sync with schema. - var personMetadata = table.Metadata{ - Name: "person", - Columns: []string{"first_name", "last_name", "email"}, - PartKey: []string{"first_name"}, - SortKey: []string{"last_name"}, - } - - // personTable allows for simple CRUD operations based on personMetadata. - var personTable = table.New(personMetadata) - - // Person represents a row in person table. - // Field names are converted to camel case by default, no need to add special tags. - // If you want to disable a field add `db:"-"` tag, it will not be persisted. - type Person struct { - FirstName string - LastName string - Email []string - } - - // Insert, bind data from struct. - { - p := Person{ - "Patricia", - "Citizen", - []string{"patricia.citzen@gocqlx_test.com"}, - } - - q := session.Query(personTable.Insert()).BindStruct(p) - if err := q.ExecRelease(); err != nil { - t.Fatal(err) - } - } - - // Get by primary key. - { - p := Person{ - "Patricia", - "Citizen", - nil, // no email - } - - q := session.Query(personTable.Get()).BindStruct(p) - if err := q.GetRelease(&p); err != nil { - t.Fatal(err) - } - - t.Log(p) - // stdout: {Patricia Citizen [patricia.citzen@gocqlx_test.com]} - } - - // Load all rows in a partition to a slice. - { - var people []Person - - q := session.Query(personTable.Select()).BindMap(qb.M{"first_name": "Patricia"}) - if err := q.SelectRelease(&people); err != nil { - t.Fatal(err) - } - - t.Log(people) - // stdout: [{Patricia Citizen [patricia.citzen@gocqlx_test.com]}] - } -}