[feature] package event log & statistics facility

Inspired by eXist-db#18: - log each time a package is downloaded - log when a package is published - only info logged is the package URI, version number, and date stamp - no personal info is collected about requester - to put these statistics to basic use, the admin page now contains 2 basic reports: (1) top 5 downloads, (2) date each package was uploaded, along with eXist version requirements
joewiz · Feb 22, 2021 · 9741de9 · 9741de9
1 parent 4ad679e
commit 9741de9
Show file tree

Hide file tree

Showing 7 changed files with 312 additions and 27 deletions.
diff --git a/admin.html b/admin.html
@@ -2,14 +2,70 @@
 <div xmlns="http://www.w3.org/1999/xhtml" data-template="templates:surround" data-template-with="templates/page.html" data-template-at="container-body">
     <div class="row">
         <div class="col-md-12">
-            <h1>Package Upload</h1>
-            <p>Upload xar packages to the public repository. New packages will appear in the list of
-                available packages immediately upon upload.</p>
-            <p data-template="app:rebuild-package-groups-metadata"/>
+            <h1>Admin</h1>
+            <p>View administrative information about packages and upload new ones.</p>
         </div>
     </div>
     <div class="row">
-        <div class="col-md-6">
+        <div class="col-md-8">
+            <div>
+                <h2>Statistics</h2>
+                <h3>Top 5 packages</h3>
+                <ol data-template="app:load-get-package-logs-for-admin-table" data-template-top-n="5">
+                    <li data-template="templates:each" data-template-from="package-logs" data-template-to="package-log">
+                        <span data-template="app:get-package-stats"/>
+                    </li>
+                </ol>
+            </div>
+            <div data-template="app:load-package-groups">
+                <h2>All package groups</h2>
+                <div data-template="templates:each" data-template-from="package-groups" data-template-to="package-group">
+                    <h3><span data-template="app:package-group-abbrev"/></h3>
+                    <dl>
+                        <dt>Abbrev</dt>
+                        <dd><span data-template="app:package-group-abbrev"/></dd>
+                        <dt>Name</dt>
+                        <dd><span data-template="app:package-group-name"/></dd>
+                        <dt>Title</dt>
+                        <dd><span data-template="app:package-group-title"/></dd>
+                    </dl>
+                    <table data-template="app:load-packages" class="table table-bordered">
+                        <thead>
+                            <tr>
+                                <th>Version</th>
+                                <th>eXist Requirement</th>
+                                <th>Date Published</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            <tr data-template="templates:each" data-template-from="packages" data-template-to="package">
+                                <td><span data-template="app:package-version"/></td>
+                                <td><span data-template="app:package-requires"/></td>
+                                <td><span data-template="app:package-date-published"/></td>
+                            </tr>
+                        </tbody>
+                    </table>
+                </div>
+            </div>
+        </div>
+        <div class="col-md-4">
+            <p>
+                <a href="index.html" class="btn btn-primary">
+                    <i class="glyphicon glyphicon-share"/> Return to list 
+                </a>
+            </p>
+            <p>
+                <a href="?rebuild-package-groups-metadata=true" class="btn btn-primary">
+                    <i class="glyphicon glyphicon-repeat"/> Rebuild package-groups metadata 
+                </a>
+            </p>
+            <a href="index.html?logout=true" class="btn btn-default">
+                <i class="glyphicon glyphicon-log-out"/> Log out</a>
+            <h2>Upload Packages</h2>
+            <p>Upload xar packages to the public repository. New packages will appear in the list of
+                available packages immediately upon upload.</p>
+            <p data-template="app:rebuild-package-groups-metadata"/>
+
             <span class="btn btn-success fileinput-button">
                 <i class="glyphicon glyphicon-plus"/>
                 <span>Select files...</span>
@@ -28,20 +84,6 @@ <h1>Package Upload</h1>
                 <tbody id="files"/>
             </table>
         </div>
-        <div class="col-md-6">
-            <p>
-                <a href="index.html" class="btn btn-primary">
-                    <i class="glyphicon glyphicon-share"/> Return to list 
-                </a>
-            </p>
-            <p>
-                <a href="?rebuild-package-groups-metadata=true" class="btn btn-primary">
-                    <i class="glyphicon glyphicon-repeat"/> Rebuild package-groups metadata 
-                </a>
-            </p>
-            <a href="index.html?logout=true" class="btn btn-default">
-                <i class="glyphicon glyphicon-log-out"/> Log out</a>
-        </div>
     </div>
     <script type="text/javascript" src="resources/scripts/jquery.ui.widget.js"/>
     <script type="text/javascript" src="resources/scripts/jquery.iframe-transport.js"/>

diff --git a/modules/app.xqm b/modules/app.xqm
@@ -8,10 +8,120 @@ module namespace app="http://exist-db.org/xquery/app";
 
 import module namespace config="http://exist-db.org/xquery/apps/config" at "config.xqm";
 import module namespace scanrepo="http://exist-db.org/xquery/admin/scanrepo" at "scan.xqm";
+import module namespace templates="http://exist-db.org/xquery/templates";
 import module namespace versions="http://exist-db.org/apps/public-repo/versions" at "versions.xqm";
 
 declare namespace request="http://exist-db.org/xquery/request";
 declare namespace response="http://exist-db.org/xquery/response";
+declare namespace xmldb="http://exist-db.org/xquery/xmldb";
+
+(:~
+ : Load the package groups document for the admin page's package-groups section
+ :)
+declare %templates:wrap function app:load-package-groups($node as node(), $model as map(*)) {
+    map { 
+        "package-groups": 
+            for $package-group in doc($config:package-groups-doc)//package-group 
+            order by $package-group/abbrev[not(@type)]
+            return
+                $package-group,
+        "dates-published": 
+            for $put in collection($config:logs-col)//event[type eq "put-package"]
+            group by $package-name := $put/package-name/string()
+            return
+                map {
+                    "package-name": $package-name,
+                    "versions": $put ! map { "version": ./package-version/string(), "date-published": ./dateTime cast as xs:dateTime }
+                }
+    }
+};
+
+(:~
+ : Load the package title for the admin page's package-groups section
+ :)
+declare function app:package-group-title($node as node(), $model as map(*)) {
+    $model?package-group/title/string()
+};
+
+(:~
+ : Load the package title for the admin page's package-groups section
+ :)
+declare function app:package-group-abbrev($node as node(), $model as map(*)) {
+    $model?package-group/abbrev/string()
+};
+
+(:~
+ : Load the package title for the admin page's package-groups section
+ :)
+declare function app:package-group-name($node as node(), $model as map(*)) {
+    $model?package-group/name/string()
+};
+
+(:~
+ : Load the packages for each package-group
+ :)
+declare %templates:wrap function app:load-packages($node as node(), $model as map(*)) {
+    map { "packages": $model?package-group//package }
+};
+
+
+(:~
+ : Load the package version
+ :)
+declare function app:package-version($node as node(), $model as map(*)) {
+    $model?package/version
+};
+
+(:~
+ : Load the package version
+ :)
+declare function app:package-requires($node as node(), $model as map(*)) {
+    let $requires := $model?package/requires[@processor eq "http://exist-db.org"]
+    return
+        ($requires/@* except $requires/@processor) ! (./name() || ": " || ./string())
+};
+
+(:~
+ : Load the package version
+ :)
+declare function app:package-date-published($node as node(), $model as map(*)) {
+    let $date-published := $model?dates-published[?package-name eq $model?package/name]?versions[?version eq $model?package/version]?date-published => head()
+    return
+        if (exists($date-published)) then
+            if (xs:date($date-published) = current-date()) then
+                format-dateTime($date-published, "Today [H00]:[m00]:[s00]")
+            else
+                format-dateTime($date-published, "[M00]/[D00]/[Y0000] [H00]:[m00]:[s00]")
+        else
+            "- (Predates logging)"
+};
+
+(:~
+ : Load the get-package logs for the admin section's table
+ :)
+declare %templates:wrap function app:load-get-package-logs-for-admin-table($node as node(), $model as map(*), $top-n as xs:integer) {
+    let $package-logs := 
+        for $event in collection($config:logs-col)//event[type eq "get-package"]
+        group by $package-name := $event/package-name/string()
+        let $count := count($event)
+        order by $count descending
+        return
+            map {
+                "package-name": $package-name,
+                "count": $count
+            }
+    return
+        map { "package-logs": 
+            subsequence($package-logs, 1, $top-n)
+        }
+};
+
+(:~
+ : Load the package title for the admin section's table
+ :)
+declare function app:get-package-stats($node as node(), $model as map(*)) {
+    $model?package-log?package-name || " (" || $model?package-log?count || ")"
+};
 
 (:~
  : Rebuild the package-groups metadata
@@ -316,3 +426,27 @@ declare function app:requires-to-english($requires as element()) {
     else
         " version " || $config:default-exist-version
 };
+
+(:~
+ : Utility function for app:mkcol
+ :)
+declare 
+    %private
+function app:mkcol-recursive($collection as xs:string, $components as xs:string*) {
+    if (exists($components)) then
+        let $newColl := concat($collection, "/", $components[1])
+        return (
+            xmldb:create-collection($collection, $components[1]),
+            app:mkcol-recursive($newColl, subsequence($components, 2))
+        )
+    else
+        ()
+};
+
+(:~
+ : Recursively create a collection hierarchy
+ :)
+declare function app:mkcol($collection as xs:string, $path as xs:string) {
+    app:mkcol-recursive($collection, tokenize($path, "/"))
+};
+
diff --git a/modules/config.xqm b/modules/config.xqm
@@ -9,8 +9,8 @@ module namespace config="http://exist-db.org/xquery/apps/config";
 
 declare namespace system="http://exist-db.org/xquery/system";
 
-declare namespace repo="http://exist-db.org/xquery/repo";
 declare namespace expath="http://expath.org/ns/pkg";
+declare namespace repo="http://exist-db.org/xquery/repo";
 
 (: Determine the application root collection from the current module load path :)
 
@@ -38,17 +38,23 @@ declare variable $config:app-data-col-name := "public-repo-data";
 declare variable $config:packages-col-name := "packages";
 declare variable $config:icons-col-name := "icons";
 declare variable $config:metadata-col-name := "metadata";
+declare variable $config:logs-col-name := "logs";
 
 declare variable $config:app-data-col := $config:app-data-parent-col || "/" || $config:app-data-col-name;
 declare variable $config:packages-col := $config:app-data-col || "/" || $config:packages-col-name;
 declare variable $config:icons-col := $config:app-data-col || "/" || $config:icons-col-name;
 declare variable $config:metadata-col := $config:app-data-col || "/" || $config:metadata-col-name;
+declare variable $config:logs-col := $config:app-data-col || "/" || $config:logs-col-name;
+declare function config:log-subcol($date as xs:date) { format-date($date, "[Y]/[M01]") };
+declare function config:log-col($date as xs:date) { $config:logs-col || "/" || config:log-subcol($date) };
 
 declare variable $config:package-groups-doc-name := "package-groups.xml";
 declare variable $config:raw-packages-doc-name := "raw-packages.xml";
+declare function config:log-doc-name($date as xs:date) { "public-repo-log-" || format-date($date, "[Y]-[M01]-[D01]") || ".xml" };
 
 declare variable $config:package-groups-doc := $config:metadata-col || "/" || $config:package-groups-doc-name;
 declare variable $config:raw-packages-doc := $config:metadata-col || "/" || $config:raw-packages-doc-name;
+declare function config:log-doc($date as xs:date) { config:log-col($date) || "/" || config:log-doc-name($date) };
 
 (: The default version number here is assumed when a client does not send a version parameter.
    It is set to 2.2.0 because this version was the last one known to work with most older packages

diff --git a/modules/get-package.xq b/modules/get-package.xq
@@ -8,11 +8,36 @@ xquery version "3.1";
  : - /exist/apps/public-repo/public/eXide-1.0.0.xar.zip
  :)
 
+import module namespace app="http://exist-db.org/xquery/app" at "app.xqm";
 import module namespace config="http://exist-db.org/xquery/apps/config" at "config.xqm";
 
 declare namespace request="http://exist-db.org/xquery/request";
 declare namespace response="http://exist-db.org/xquery/response";
 declare namespace util="http://exist-db.org/xquery/util";
+declare namespace xmldb="http://exist-db.org/xquery/xmldb";
+
+declare function local:log-get-package-event($filename as xs:string) as empty-sequence() {
+    let $package := doc($config:raw-packages-doc)//package[@path eq $filename]
+    let $today := current-date()
+    let $log-doc := config:log-doc($today)
+    let $event := 
+        element event {
+            element dateTime { current-dateTime() },
+            element type { "get-package" },
+            element package-name { $package/name/string() },
+            element package-version { $package/version/string() }
+        }
+    let $update-log :=
+        if (doc-available($log-doc)) then 
+            update insert $event into doc($log-doc)/public-repo-log
+        else
+            ( 
+                app:mkcol($config:logs-col, config:log-subcol($today)),
+                xmldb:store(config:log-col($today), config:log-doc-name($today), element public-repo-log { $event } )
+            )
+    return
+        ()
+};
 
 let $filename := request:get-parameter("filename", ())
 let $xar-filename :=
@@ -25,6 +50,7 @@ let $path := $config:packages-col || "/" || $xar-filename
 return
     if (util:binary-doc-available($path)) then
         let $xar := util:binary-doc($config:packages-col || "/" || $xar-filename)
+        let $log := local:log-get-package-event($filename)
         return
             response:stream-binary($xar, "application/zip")
     else

diff --git a/modules/put-package.xq b/modules/put-package.xq
@@ -4,6 +4,7 @@ xquery version "3.1";
  : Receives uploaded packages and immediately publishes them to the package repository
  :)
 
+import module namespace app="http://exist-db.org/xquery/app" at "app.xqm";
 import module namespace config="http://exist-db.org/xquery/apps/config" at "config.xqm";
 import module namespace scanrepo="http://exist-db.org/xquery/admin/scanrepo" at "scan.xqm";
 
@@ -16,6 +17,29 @@ declare namespace output="http://www.w3.org/2010/xslt-xquery-serialization";
 declare option output:method "json";
 declare option output:media-type "application/json";
 
+declare function local:log-put-package-event($filename as xs:string) as empty-sequence() {
+    let $package := doc($config:raw-packages-doc)//package[@path eq $filename]
+    let $today := current-date()
+    let $log-doc := config:log-doc($today)
+    let $event := 
+        element event {
+            element dateTime { current-dateTime() },
+            element type { "put-package" },
+            element package-name { $package/name/string() },
+            element package-version { $package/version/string() }
+        }
+    let $update-log :=
+        if (doc-available($log-doc)) then 
+            update insert $event into doc($log-doc)/public-repo-log
+        else
+            ( 
+                app:mkcol($config:logs-col, config:log-subcol($today)),
+                xmldb:store(config:log-col($today), config:log-doc-name($today), element public-repo-log { $event } )
+            )
+    return
+        ()
+};
+
 declare function local:upload-and-publish($xar-filename as xs:string, $xar-binary as xs:base64Binary) {
     let $path := xmldb:store($config:packages-col, $xar-filename, $xar-binary)
     let $publish := scanrepo:publish-package($xar-filename)
@@ -38,7 +62,8 @@ let $required-group := config:repo-permissions()?group
 return
     if (exists($user) and sm:get-user-groups($user) = $required-group) then
         try {
-            local:upload-and-publish($xar-filename, $xar-binary)
+            local:upload-and-publish($xar-filename, $xar-binary),
+            local:log-put-package-event($xar-filename)
         } catch * {
             map {
                 "result":