build quickstatements (wmde#64)

.github/workflows/built_and_test.yml

@@ -139,6 +139,25 @@ jobs:
           if-no-files-found: error
           path: artifacts/wikibase-bundle.docker.tar.gz
 
+  build_quickstatements:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: c-py/action-dotenv-to-setenv@v2
+        with:
+          env-file: ${{ env.env_file }}
+
+      - name: Build QuickStatements Image 
+        id: download_step
+        run: bash build.sh quickstatements ${{ env.env_file }}
+
+      - name: Archive docker production artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: BuildArtifacts
+          if-no-files-found: error
+          path: artifacts/quickstatements.docker.tar.gz
+
   build_queryservice:
     runs-on: ubuntu-latest
     steps:

Docker/build/QuickStatements/Dockerfile

@@ -0,0 +1,44 @@
+FROM ubuntu:xenial as fetcher
+
+RUN apt-get update && \
+    apt-get install --yes --no-install-recommends git=1:2.* ca-certificates=201* && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+COPY artifacts/quickstatements.tar.gz artifacts/quickstatements.tar.gz
+RUN tar xzf artifacts/quickstatements.tar.gz
+
+FROM composer:1 as composer
+
+COPY --from=fetcher /quickstatements /quickstatements
+
+WORKDIR /quickstatements
+RUN composer install --no-dev
+
+FROM php:7.2-apache
+
+# Install envsubst
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends gettext-base=0.19.* jq=1.5* && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY --from=composer /quickstatements /var/www/html/quickstatements
+COPY --from=fetcher /magnustools /var/www/html/magnustools
+
+COPY entrypoint.sh /entrypoint.sh
+
+COPY config.json /templates/config.json
+COPY oauth.ini /templates/oauth.ini
+COPY php.ini /templates/php.ini
+
+ENV APACHE_DOCUMENT_ROOT /var/www/html/quickstatements/public_html
+RUN sed -ri -e "s!/var/www/html!${APACHE_DOCUMENT_ROOT}!g" /etc/apache2/sites-available/*.conf
+RUN sed -ri -e "s!/var/www/!${APACHE_DOCUMENT_ROOT}!g" /etc/apache2/apache2.conf /etc/apache2/conf-available/*.conf
+
+ENV MW_SITE_NAME=wikibase-docker\
+    MW_SITE_LANG=en\
+    PHP_TIMEZONE=UTC
+
+RUN install -d -owww-data /var/log/quickstatements
+
+ENTRYPOINT ["/bin/bash"]
+CMD ["/entrypoint.sh"]

Docker/build/QuickStatements/README.md

@@ -0,0 +1,70 @@
+# Quickstatements docker image
+
+Quickstatements2 as seen at  [https://github.com/magnusmanske/quickstatements](https://github.com/magnusmanske/quickstatements)
+
+### Environment variables
+
+Variable                   | Default  | Description                                            
+-------------------------- | -------- | -----------
+`WIKIBASE_SCHEME_AND_HOST` | NONE     | Host and port of wikibase instance as seen by QS container                                                  
+`WB_PUBLIC_HOST_AND_PORT`  | NONE     | Host and port of wikibase as seen by the user's browser
+`QS_PUBLIC_HOST_AND_PORT`  | NONE     | Host and port of QS as seen by the user's browser      
+`OAUTH_CONSUMER_KEY`       | NONE     | OAuth consumer key (obtained from wikibase)            
+`OAUTH_CONSUMER_SECRET`    | NONE     | OAuth consumer key (obtained from wikibase)            
+`PHP_TIMEZONE`             | UTC      | setting of php.ini date.timezone                       
+
+### Filesystem layout
+
+Directory                                   | Description                                                                   
+---------------------------------           | ------------------------------------------------------------------------------
+`/var/www/html/quickstatements`             | Base quickstatements directory                                                
+`/var/www/html/quickstatements/public_html` | The Apache Root folder                                                        
+`/var/www/html/magnustools`                 | Base magnustools directory                                                    
+
+File                      | Description                                                                                                                              
+------------------------- | ------------------------------------------------------------------------------                                                           
+`/templates/config.json`  | Template for Quickstatements' config.json (substituted to `/var/www/html/quickstatements/public_html/config.json` at runtime)            
+`/templates/oauth.ini`    | Template for Quickstatements' oauth.ini (substituted to `/var/www/html/quickstatements/oauth.ini` at runtime)                            
+`/templates/php.ini`      | php config (default provided sets date.timezone to prevent php complaining substituted to `/usr/local/etc/php/conf.d/php.ini` at runtime)
+
+
+### How to setup and use
+
+#### Set up quickstatements
+In order for quickstatements to communicate with wikibase it needs to know where your instance is and how it can find it.
+This must be done by setting the ENV variable `WIKIBASE_SCHEME_AND_HOST`. n.b. This should reflect how this container when running
+sees the wikibase container. For example the docker container alias like wikibase.svc.
+
+The user's browser will also be redirected to the Wikibase instance and finally back to quickstatements. The address
+the user sees for the Wikibase may be different from how the running container sees it. For example: it may be running
+on localhost on a specific port. e.g. http://localhost:8181. This should be passed to the quickstatements container as
+WB_PUBLIC_HOST_AND_PORT
+
+One must also know how this container will be visible to the user as well so it can ask the wikibase to redirect the
+user back here. This should be passed as `QS_PUBLIC_HOST_AND_PORT`
+
+You can pass the consumer and secret token you got from the wikibase to this container as the environment variables
+ `OAUTH_CONSUMER_KEY` and `OAUTH_CONSUMER_SECRET`. If you don't, docker-compose automatically handles this.
+
+You can now test it works by navigating to http://\<your quickstatements host\> and logging in using the button top right.
+
+You should be redirected to the wiki where you can authorize this Quickstatements to act on your behalf
+
+Finally you should be redirected back to Quickstatements and you should appear logged in.
+
+Use Quickstatements as normal with the Run button. Currently "Run in background" is not supported by this image.
+
+#### Troubleshooting
+If you see an error such as mw-oauth exception when trying to log in check that you have passed the right consumer token
+and secret token to quickstatements.
+
+If you have changed the value of $wgSecretKey $wgOAuthSecretKey since you made the consumer you'll need to make another new consumer or
+reissue the secret token for the old one.
+
+### Development
+
+These images are build from the master branch of the quickstatements and magnustools repos.
+
+For this reason the images may sometimes break due to upstream changes and need fixing.
+
+Additional images for new releases do not need to be created.

Docker/build/QuickStatements/config.json

@@ -0,0 +1,25 @@
+{
+	"site" : "${MW_SITE_NAME}" ,
+	"bot_config_file" : "/var/www/html/bot.ini" ,
+	"logfile" : "/var/log/quickstatements/tool.log" ,
+	"sites" : {
+		"${MW_SITE_NAME}" : {
+			"oauth" : {
+				"language":"${MW_SITE_LANG}" ,
+				"project":"${MW_SITE_NAME}" ,
+				"ini_file":"/quickstatements/data/oauth.ini" ,
+				"publicMwOAuthUrl":"${WB_PUBLIC_SCHEME_HOST_AND_PORT}/w/index.php?title=Special:OAuth" ,
+				"mwOAuthUrl":"${WIKIBASE_SCHEME_AND_HOST}/w/index.php?title=Special:OAuth" ,
+				"mwOAuthIW":"mw"
+			} ,
+			"server" : "${WB_PUBLIC_HOST_AND_PORT}" ,
+			"api" : "${WIKIBASE_SCHEME_AND_HOST}/w/api.php" ,
+			"pageBase" : "${WB_PUBLIC_SCHEME_HOST_AND_PORT}/wiki/" ,
+			"toolBase" : "${QS_PUBLIC_SCHEME_HOST_AND_PORT}/" ,
+			"types" : {
+				"P" : { "type":"property" , "ns":"${WB_PROPERTY_NAMESPACE}" , "ns_prefix":"${WB_PROPERTY_PREFIX}" } ,
+				"Q" : { "type":"item" , "ns":"${WB_ITEM_NAMESPACE}" , "ns_prefix":"${WB_ITEM_PREFIX}" }
+			}
+		}
+	}
+}

Docker/build/QuickStatements/entrypoint.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# Test if required environment variables have been set
+REQUIRED_VARIABLES=(QS_PUBLIC_SCHEME_HOST_AND_PORT WB_PUBLIC_SCHEME_HOST_AND_PORT WIKIBASE_SCHEME_AND_HOST WB_PROPERTY_NAMESPACE WB_PROPERTY_PREFIX WB_ITEM_NAMESPACE WB_ITEM_PREFIX)
+for i in "${REQUIRED_VARIABLES[@]}"; do
+    if ! [[ -v "$i" ]]; then
+    echo "$i is required but isn't set. You should pass it to docker. See: https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file";
+    exit 1;
+    fi
+done
+
+if [[ -v "$OAUTH_CONSUMER_KEY" && "$OAUTH_CONSUMER_SECRET" ]]; then
+    envsubst < /templates/oauth.ini > /quickstatements/data/oauth.ini;
+fi
+
+envsubst < /templates/config.json > /var/www/html/quickstatements/public_html/config.json
+envsubst < /templates/php.ini > /usr/local/etc/php/conf.d/php.ini
+
+docker-php-entrypoint apache2-foreground

Docker/build/QuickStatements/oauth.ini

@@ -0,0 +1,5 @@
+; HTTP User-Agent header
+agent = 'Wikibase Docker QuickStatements'
+; assigned by Special:OAuthConsumerRegistration (request modelled after https://www.wikidata.org/wiki/Special:OAuthListConsumers/view/77b4ae5506dd7dbb0bb07f80e3ae3ca9)
+consumerKey = '${OAUTH_CONSUMER_KEY}'
+consumerSecret = '${OAUTH_CONSUMER_SECRET}'

Docker/build/QuickStatements/php.ini

@@ -0,0 +1 @@
+date.timezone = "${PHP_TIMEZONE}"

Makefile

@@ -26,6 +26,10 @@ wikibase_bundle:
 	bash update_cache.sh extensions
 	eval ". ./build/build_wikibase.sh; bash build/build_wikibase_bundle_docker.sh ${WIKIBASE_IMAGE_NAME} ${WIKIBASE_BUNDLE_IMAGE_NAME}"
 
+quickstatements:
+	bash update_cache.sh services
+	eval ". ./build/build_quickstatements.sh; bash build/build_quickstatements_docker.sh ${QUICKSTATEMENTS_IMAGE_NAME}"
+
 queryservice:
 	eval ". ./build/build_queryservice.sh; bash build/build_queryservice_docker.sh ${QUERYSERVICE_IMAGE_NAME}"
 
@@ -44,4 +48,4 @@ clean:
 	rm -rf artifacts/*.tar.gz
 	rm -rf artifacts/*.env
 
-all: mediawiki wikibase wikibase_bundle elasticsearch queryservice queryservice-ui
+all: mediawiki wikibase wikibase_bundle elasticsearch queryservice queryservice-ui quickstatements

build/build_quickstatements.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e
+
+ROOT="$(pwd)"
+TARBALL=quickstatements.tar.gz
+TARBALL_PATH="$ROOT/artifacts/$TARBALL"
+
+# clone and clean quickstatements
+TEMP_GIT_DIR=$(mktemp -d)
+QS_GIT_DIR="$TEMP_GIT_DIR/quickstatements"
+UPDATE_SUBMODULE=1 bash "$ROOT"/build/clone_repo.sh "$QUICKSTATEMENTS_COMMIT_HASH" "$ROOT/git_cache/services/quickstatements.git" quickstatements "$QS_GIT_DIR" master
+bash "$ROOT"/build/clean_repo.sh "$QS_GIT_DIR"
+
+# clone and clean magnustools
+MAGNUSTOOLS_GIT_DIR="$TEMP_GIT_DIR/magnustools"
+UPDATE_SUBMODULE=0 bash "$ROOT"/build/clone_repo.sh "$MAGNUSTOOLS_COMMIT_HASH" "$ROOT/git_cache/services/magnustools.git" magnustools "$MAGNUSTOOLS_GIT_DIR" master
+bash "$ROOT"/build/clean_repo.sh "$MAGNUSTOOLS_GIT_DIR"
+
+cd "$TEMP_GIT_DIR"
+GZIP=-9 tar -C "$TEMP_GIT_DIR" -zcvf "$TARBALL_PATH" magnustools quickstatements
+cd -
+
+if [ -n "$GITHUB_ENV" ]; then
+    echo "TARBALL_PATH=$TARBALL_PATH" >> "$GITHUB_ENV"
+else
+    export TARBALL_PATH
+fi

build/build_quickstatements_docker.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -ex
+
+mkdir -p Docker/build/QuickStatements/artifacts
+cp "$TARBALL_PATH" Docker/build/QuickStatements/artifacts/
+
+docker build --pull Docker/build/QuickStatements/ -t "$1"
+docker save "$1" | gzip -9f > artifacts/"$1".docker.tar.gz

update_cache.sh

@@ -46,8 +46,18 @@ for arg in "$@"; do
 			fetch_all "$extension.git"
 		done
 	elif [ "$arg" = "services" ]; then
+		# queryservice
 		clone_if_not_present "services/wikidata-query-gui.git" "https://gerrit.wikimedia.org/r/wikidata/query/gui"
 		fetch_all "services/wikidata-query-gui.git"
+
+		# quickstatements
+		clone_if_not_present "services/quickstatements.git" https://github.com/magnusmanske/quickstatements
+		fetch_all "services/quickstatements.git"
+
+		# magnus tools
+		clone_if_not_present "services/magnustools.git" https://bitbucket.org/magnusmanske/magnustools.git
+		fetch_all "services/magnustools.git"
+
 	elif [ "$arg" = "core" ]; then
 		clone_if_not_present core.git "https://gerrit.wikimedia.org/r/mediawiki/core"
 		fetch_all "core.git"

variables.env

@@ -5,6 +5,7 @@ QUERYSERVICE_UI_IMAGE_NAME=wdqs-frontend
 MEDIAWIKI_IMAGE_NAME=wbmediawiki
 ELASTICSEARCH_IMAGE_NAME=elasticsearch
 WIKIBASE_BUNDLE_IMAGE_NAME=wikibase-bundle
+QUICKSTATEMENTS_IMAGE_NAME=quickstatements
 
 # Releasing tarballs
 RELEASE_DIR=/srv/org/wikimedia/releases/wikibase

versions/REL1_34.env

@@ -3,6 +3,9 @@ MEDIAWIKI_BRANCH_NAME=REL1_34
 QUERYSERVICE_VERSION=0.3.40
 QUERYSERVICE_UI_COMMIT_HASH=e84ab35125557ff073f42ba522a684d35c288b38
 
+QUICKSTATEMENTS_COMMIT_HASH=7830824139e1929d036c900587ea61fc8f168183
+MAGNUSTOOLS_COMMIT_HASH=7d7ec4182ebefc013778bdfb4090f0e099ce66dd
+
 MEDIAWIKI_MAJOR_VERSION=1.34
 MEDIAWIKI_VERSION=1.34.0
 

versions/REL1_35.env

@@ -3,6 +3,9 @@ MEDIAWIKI_BRANCH_NAME=REL1_35
 QUERYSERVICE_VERSION=0.3.40
 QUERYSERVICE_UI_COMMIT_HASH=e84ab35125557ff073f42ba522a684d35c288b38
 
+QUICKSTATEMENTS_COMMIT_HASH=7830824139e1929d036c900587ea61fc8f168183
+MAGNUSTOOLS_COMMIT_HASH=7d7ec4182ebefc013778bdfb4090f0e099ce66dd
+
 MEDIAWIKI_MAJOR_VERSION=1.35
 MEDIAWIKI_VERSION=1.35.0