Skip to content

Commit

Permalink
Use json log
Browse files Browse the repository at this point in the history
  • Loading branch information
@nenadalm
nenadalm committed Aug 1, 2024
1 parent c55b875 commit 3c073b4
Show file tree
Hide file tree
Showing 9 changed files with 28 additions and 242 deletions.
10 changes: 3 additions & 7 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ First download jar in [releases](https://github.com/nenadalm/postgresql-log-view
#### [Portal](https://github.com/djblue/portal#portal)

```shell
tail -f /var/lib/pgsql/data/log/postgresql.csv | java -jar ./plv-portal.jar
tail -f /var/lib/pgsql/data/log/postgresql-$(date +%a).json | java -jar ./plv-portal.jar
```

Command window: `Ctrl+Shift+P` or `Meta+Shift+P`
Expand All @@ -28,20 +28,16 @@ See [this video](https://youtu.be/gByyg-m0XOg?t=175) on how it can be used.
#### [Reveal](https://vlaaad.github.io/reveal/)

```shell
tail -f /var/lib/pgsql/data/log/postgresql.csv | java -jar ./plv-reveal.jar
tail -f /var/lib/pgsql/data/log/postgresql-$(date +%a).json | java -jar ./plv-reveal.jar
```

## Requirements

### Postgresql version

13

### Postgresl settings

```
# this tool parses csv log only
log_destination = 'csvlog'
log_destination = 'jsonlog'
# in order to see statements in log, this has to be enabled
log_statement = 'all'
Expand Down
4 changes: 2 additions & 2 deletions deps.edn
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
{:paths ["src" "resources"]
:deps {org.clojure/clojure {:mvn/version "1.10.3"}
org.clojure/data.csv {:mvn/version "1.0.0"}
metosin/jsonista {:mvn/version "0.3.9"}
com.github.vertical-blank/sql-formatter {:mvn/version "2.0.2"}}
:aliases
{:test {:extra-paths ["test"]
:extra-deps {lambdaisland/kaocha {:mvn/version "1.0.887"}
org.clojure/test.check {:mvn/version "1.1.0"}}
:main-opts ["-m" "kaocha.runner"]}
:portal {:extra-paths ["extra-src/portal"]
:extra-deps {djblue/portal {:mvn/version "0.14.0"}}
:extra-deps {djblue/portal {:mvn/version "0.57.1"}}
:main-opts ["-m" "app.portal"]}
:reveal {:extra-paths ["extra-src/reveal"]
:extra-deps {vlaaad/reveal {:mvn/version "1.3.212"}}
Expand Down
21 changes: 10 additions & 11 deletions extra-src/portal/app/portal.clj
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
(ns app.portal
(:require
[app.postgresql-log.core :as log]
[app.postgresql-log.parser :as log-parser]
[app.postgresql-log.reader :as log-reader]
[jsonista.core :as j]
[app.config :refer [config]]
[portal.api :as portal])
(:gen-class))
Expand All @@ -13,25 +12,25 @@
(proxy [Thread] []
(run []
(portal/close))))
(portal/open)
(portal/tap))
(portal/open {:theme :portal.colors/nord})
(add-tap #'portal/submit))

(defn- format-log [log]
(with-meta
log
{'clojure.core.protocols/nav (fn [_ k v]
(case k
:log/message (log/format-message log)
:message (log/format-message log)
v))}))

(defn -main [& _]
(println (str "App info:\n"
" Version: " (:app.config/version config) "\n"
" UI: portal\n"))
(init-portal)
(log-reader/safe-read
(doseq [line (log-reader/read-csv *in*)]
(-> line
log-parser/line->log
format-log
tap>))))
(loop []
(-> (read-line)
(j/read-value j/keyword-keys-object-mapper)
format-log
tap>)
(recur)))
15 changes: 7 additions & 8 deletions extra-src/reveal/app/reveal.clj
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
(ns app.reveal
(:require
[app.postgresql-log.core :as log]
[app.postgresql-log.parser :as log-parser]
[app.postgresql-log.reader :as log-reader]
[jsonista.core :as j]
[app.config :refer [config]]
[vlaaad.reveal :as reveal]
[vlaaad.reveal.ext :as ve])
Expand Down Expand Up @@ -35,9 +34,9 @@
" Version: " (:app.config/version config) "\n"
" UI: reveal\n"))
(init-reveal)
(log-reader/safe-read
(doseq [line (log-reader/read-csv *in*)]
(-> line
log-parser/line->log
format-log
tap>))))
(loop []
(-> (read-line)
(j/read-value j/keyword-keys-object-mapper)
format-log
tap>)
(recur)))
4 changes: 2 additions & 2 deletions src/app/postgresql_log/core.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@
params))

(defn format-message [log]
(let [message (parser/parse-message (:log/message log))]
(let [message (parser/parse-message (:message log))]
(case (:message/type message)
("statement" "execute") (format-query
(inline-params
(:message/content message)
(parser/parse-detail-params (:log/detail log))))
(parser/parse-detail-params (:detail log))))
(:message/content message))))
43 changes: 0 additions & 43 deletions src/app/postgresql_log/parser.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,49 +2,6 @@
(:require
[clojure.string]))

(def ^:private csv-header
[:log/log-time
:log/user-name
:log/database-name
:log/process-id
:log/connection-from
:log/session-id
:log/session-line-num
:log/command-tag
:log/session-start-time
:log/virtual-transaction-id
:log/transaction-id
:log/error-severity
:log/sql-state-code
:log/message
:log/detail
:log/hint
:log/internal-query
:log/internal-query-pos
:log/context
:log/query
:log/query-pos
:log/location
:log/application-name
:log/backend-type])

(defn valid-line? [line]
(= (count line) (count csv-header)))

(defn drop-invalid-lines [lines]
(drop-while #(not (valid-line? %)) lines))

(defn line->log [line]
(reduce-kv
(fn [m k v]
(if (= "" v)
m
(assoc m k v)))
{}
(zipmap
csv-header
line)))

(defn parse-message [message]
(when message
(if-let [res (re-matches #"(?s)(?<type>[^ :]*)[^:]*: (?<s>.*)" message)]
Expand Down
30 changes: 0 additions & 30 deletions src/app/postgresql_log/reader.clj
View file

This file was deleted.

8 changes: 4 additions & 4 deletions test/app/postgresql_log/core_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@ WHERE
username = 'admin',
id = $3"
(core/format-message
{:log/detail "parameters: $1 = 'admin', $2 = 3"
:log/message "execute <unnamed>: SELECT username FROM user_user WHERE username = $1, id = $3"}))))
{:detail "parameters: $1 = 'admin', $2 = 3"
:message "execute <unnamed>: SELECT username FROM user_user WHERE username = $1, id = $3"}))))
(testing "query with problematic json operator"
(is (= "SELECT
'{\"a\": 1}' :: jsonb -> 'a'"
(core/format-message
{:log/message "execute <unnamed>: SELECT '{\"a\": 1}'::jsonb->'a'"}))))
{:message "execute <unnamed>: SELECT '{\"a\": 1}'::jsonb->'a'"}))))
(testing "duration"
(is (= "3 ms"
(core/format-message
{:log/message "duration: 3 ms"})))))
{:message "duration: 3 ms"})))))
135 changes: 0 additions & 135 deletions test/app/postgresql_log/parser_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,141 +3,6 @@
[app.postgresql-log.parser :as parser]
[clojure.test :refer [deftest testing is]]))

(deftest valid-line?-test []
(is (true? (parser/valid-line?
["2020-10-22 17:54:15.466 CEST"
"postgres"
"db"
"1146674"
"127.0.0.1:55272"
"5f91aaa8.117f32"
"5"
"SELECT"
"2020-10-22 17:52:08 CEST"
"3/24597"
"0"
"LOG"
"00000"
"execute <unnamed>: SELECT username FROM user_user WHERE username = $1"
"parameters: $1 = 'admin'"
""
""
""
""
""
""
""
"PostgreSQL JDBC Driver"
"client backend"])))
(testing "one missing item"
(is (false? (parser/valid-line?
["2020-10-22 17:54:15.466 CEST"
"postgres"
"db"
"1146674"
"127.0.0.1:55272"
"5f91aaa8.117f32"
"5"
"SELECT"
"2020-10-22 17:52:08 CEST"
"3/24597"
"0"
"LOG"
"00000"
"execute <unnamed>: SELECT username FROM user_user WHERE username = $1"
"parameters: $1 = 'admin'"
""
""
""
""
""
""
""
""])))))

(deftest line->log-test []
(testing "select with param"
(is (= {:log/application-name "PostgreSQL JDBC Driver"
:log/command-tag "SELECT"
:log/connection-from "127.0.0.1:55272"
:log/database-name "db"
:log/detail "parameters: $1 = 'admin'"
:log/error-severity "LOG"
:log/log-time "2020-10-22 17:54:15.466 CEST"
:log/message "execute <unnamed>: SELECT username FROM user_user WHERE username = $1"
:log/process-id "1146674"
:log/session-id "5f91aaa8.117f32"
:log/session-line-num "5"
:log/session-start-time "2020-10-22 17:52:08 CEST"
:log/sql-state-code "00000"
:log/transaction-id "0"
:log/user-name "postgres"
:log/virtual-transaction-id "3/24597"}
(parser/line->log
["2020-10-22 17:54:15.466 CEST"
"postgres"
"db"
"1146674"
"127.0.0.1:55272"
"5f91aaa8.117f32"
"5"
"SELECT"
"2020-10-22 17:52:08 CEST"
"3/24597"
"0"
"LOG"
"00000"
"execute <unnamed>: SELECT username FROM user_user WHERE username = $1"
"parameters: $1 = 'admin'"
""
""
""
""
""
""
""
"PostgreSQL JDBC Driver"]))))
(testing "select without params"
(is (= {:log/application-name "PostgreSQL JDBC Driver"
:log/command-tag "SELECT"
:log/connection-from "127.0.0.1:55272"
:log/database-name "db"
:log/error-severity "LOG"
:log/log-time "2020-10-22 18:07:49.876 CEST"
:log/message "execute <unnamed>: SELECT username FROM user_user"
:log/process-id "1146674"
:log/session-id "5f91aaa8.117f32"
:log/session-line-num "6"
:log/session-start-time "2020-10-22 17:52:08 CEST"
:log/sql-state-code "00000"
:log/transaction-id "0"
:log/user-name "postgres"
:log/virtual-transaction-id "3/24598"}
(parser/line->log
["2020-10-22 18:07:49.876 CEST"
"postgres"
"db"
"1146674"
"127.0.0.1:55272"
"5f91aaa8.117f32"
"6"
"SELECT"
"2020-10-22 17:52:08 CEST"
"3/24598"
"0"
"LOG"
"00000"
"execute <unnamed>: SELECT username FROM user_user"
""
""
""
""
""
""
""
""
"PostgreSQL JDBC Driver"])))))

(deftest parse-message-test []
(is (nil? (parser/parse-message nil)))
(is (= {:message/type "statement"
Expand Down

0 comments on commit 3c073b4

Please sign in to comment.