From 46a3603ac1c17e218fbc069116bb57a159c06010 Mon Sep 17 00:00:00 2001 From: chand1012 Date: Sat, 24 Feb 2024 12:30:50 -0500 Subject: [PATCH] Write initial README and fix some bugs --- README.md | 61 +++++++++++++++++++++++++++++++++++++++----- cmd/root.go | 1 + pkg/db/from_json.go | 4 +-- pkg/db/from_jsonl.go | 4 +-- pkg/db/load_sql.go | 17 ++++++++++++ 5 files changed, 76 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 7be154d..b8137b8 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,57 @@ -# sq +

sq

+

Convert and query JSON, JSONL, CSV, and SQLite with ease!

-Like `jq` , but for SQLite. +`sq` is a simple yet powerful command line tool for query and converting from and to SQLite, CSV, JSON, and JSONL files, heavily inspired by [ `jq` ](https://jqlang.github.io/jq/). -## TODO +## Features -* [ ] Fix JSON input -* [ ] Finish README -* [ ] Write some docs -* [ ] Make a build for all major platforms +* Convert between SQLite, CSV, JSON, and JSONL files. +* Query CSV, JSON, and JSONL using real SQLite queries. +* Allows for rapid scripting and conversion of data. +* Pipe in data or read from files. + +## Installation + +```bash +go install github.com/chand1012/sq@latest +``` + +## Examples + +Query some orders from a CSV file. Column names for CSV files are converted to lower case and spaces are replaced with underscores. + +```bash +$ sq -r orders.csv 'select country from sq where seller_amount > 20 not null;' +United States of America +United States of America +United States of America +United States of America +United States of America +United States of America +United Kingdom +Canada +United States of America +United States of America +United States of America +United States of America +United States of America +Canada +United States of America +Canada +Canada +Australia +United States of America +... +``` + +Download and query some JSONL datasets. + +```bash +$ curl https://raw.githubusercontent.com/TimeSurgeLabs/llm-finetuning/4e934ce602f34f62f4d803c40cd1e7825d216192/data/fingpt-sentiment-1k.jsonl | sq 'select * from sq where output = "positive";' -f jsonl > positive.jsonl +``` + +You can even use it with `jq` ! + +```bash +$ curl https://api.gogopool.com/stakers | jq '.stakers' | sq -t stakers 'SELECT stakerAddr,avaxValidating FROM stakers WHERE avaxValidating > 0;' -f json > stakers.json +``` diff --git a/cmd/root.go b/cmd/root.go index 667e3ff..7f67686 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -80,6 +80,7 @@ func run(cmd *cobra.Command, args []string) { log.Debugf("Input file path: %s", inputFilePath) d, _, err = db.LoadFile(inputFilePath) if err != nil { + log.Debug("Input file is not a SQLite database, reading as a regular file") file, err := os.ReadFile(inputFilePath) if err != nil { logger.HandlePanic(log, err, verbose) diff --git a/pkg/db/from_json.go b/pkg/db/from_json.go index 4de6277..5e4fdaa 100644 --- a/pkg/db/from_json.go +++ b/pkg/db/from_json.go @@ -44,8 +44,8 @@ func FromJSON(b []byte, tableName string) (*sql.DB, string, error) { columns, types := utils.BreakOutMap(typeMap) - // preprocess the column names - columns = processColumnNames(columns) + // // preprocess the column names + // columns = processColumnNames(columns) createQuery := genCreateTableQuery(tableName, columns, types) diff --git a/pkg/db/from_jsonl.go b/pkg/db/from_jsonl.go index 8d87a51..377ed0c 100644 --- a/pkg/db/from_jsonl.go +++ b/pkg/db/from_jsonl.go @@ -48,8 +48,8 @@ func FromJSONL(b []byte, tableName string) (*sql.DB, string, error) { columns, types := utils.BreakOutMap(typeMap) - // preprocess the column names - columns = processColumnNames(columns) + // // preprocess the column names + // columns = processColumnNames(columns) createQuery := genCreateTableQuery(tableName, columns, types) diff --git a/pkg/db/load_sql.go b/pkg/db/load_sql.go index 10bbf71..923b964 100644 --- a/pkg/db/load_sql.go +++ b/pkg/db/load_sql.go @@ -2,6 +2,7 @@ package db import ( "database/sql" + "errors" "os" _ "github.com/glebarez/go-sqlite" @@ -22,6 +23,19 @@ func createTempDB() (*sql.DB, string, error) { return db, tmpFile.Name(), nil } +// check if the file is a valid sqlite db +func IsValidDB(fileName string) bool { + db, err := sql.Open("sqlite", fileName) + if err != nil { + return false + } + defer db.Close() + + // try to query the db + _, err = db.Query("SELECT * FROM sqlite_master") + return err == nil +} + // load a sql file from bytes func LoadStdin(bytes []byte) (*sql.DB, string, error) { tmpFile, err := os.CreateTemp(os.TempDir(), "sq-*.sql") @@ -44,6 +58,9 @@ func LoadStdin(bytes []byte) (*sql.DB, string, error) { } func LoadFile(fileName string) (*sql.DB, string, error) { + if !IsValidDB(fileName) { + return nil, "", errors.New("file is not a valid SQLite database") + } db, err := sql.Open("sqlite", fileName) if err != nil { return nil, fileName, err