Merge branch 'master' into build_id

.circleci/Dockerfile

@@ -1,11 +1,9 @@
 FROM floopcz/tensorflow_cc:ubuntu-shared-cuda
+ARG DEBIAN_FRONTEND=noninteractive
 
 RUN wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && apt-get update && apt-get install -y intel-mkl-64bit-2018.2-046
-RUN apt-get install -y clang-6.0 ninja-build python3-pip nvidia-opencl-dev libopenblas-dev libboost-dev nvidia-cuda-dev nvidia-cuda-toolkit libgtest-dev git ssh tar gzip ca-certificates sudo
+RUN apt-get install -y clang ninja-build python3-pip nvidia-opencl-dev libopenblas-dev libboost-dev libgtest-dev git ssh tar gzip ca-certificates sudo
+RUN apt-get install -y g++-8
+RUN apt-get install -y cuda
 RUN pip3 install meson
 RUN ln -s /usr/include/ /usr/include/openblas
-
-RUN curl -OL https://github.com/google/protobuf/releases/download/v3.5.1/protoc-3.5.1-linux-x86_64.zip
-RUN unzip protoc-3.5.1-linux-x86_64.zip -d protoc3
-RUN sudo mv protoc3/bin/* /usr/local/bin/
-RUN sudo mv protoc3/include/* /usr/local/include/

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: danieluranga/leela_chess_zero-lc0_ubuntu_builder:0.0.4
+      - image: danieluranga/leela_chess_zero-lc0_ubuntu_builder:0.0.7
     steps:
       - checkout
       - run:
@@ -11,15 +11,34 @@ jobs:
             git submodule init
             git submodule update --remote
       - run:
-          name: Build clang version
-          command: CC=clang-6.0 CXX=clang++-6.0 ./build.sh
+          name: Create Meson build dirs
+          command: mkdir build-gcc && mkdir build-clang
       - run:
-          command: cp build/release/lc0 /tmp/lc0-clang
+          name: Meson Clang
+          environment:
+            CC: clang
+            CXX: clang++
+          command: meson build-clang
       - run:
-          name: Build g++ version
-          command: ./build.sh
+          name: Meson GCC
+          environment:
+            CC: gcc-8
+            CXX: g++-8
+          command: meson build-gcc
       - run:
-          command: cp build/release/lc0 /tmp/lc0-g++
+          name: Build Clang
+          command: |
+            cd build-clang
+            ninja
+      - run:
+          name: Build GCC
+          command: |
+            cd build-gcc
+            ninja -j 4
+      - run:
+          command: cp build-clang/lc0 /tmp/lc0-clang
+      - run:
+          command: cp build-gcc/lc0 /tmp/lc0-g++
       - store_artifacts:
           path: /tmp/lc0-clang
           destination: lc0-ubuntu-18-04-clang

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,65 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**WARNING: Please only use GitHub issues for BUG REPORTS and FEATURE REQUESTS**
+Lengthy algorithm discussions etc are fine too, but also consider creating a wiki page or a page on website.
+
+Here is what to do with other types of questions:
+1. Build problems: ask in #help channel in our discord: https://discord.gg/pKujYxD
+2. Configuration questions: ask in #help channel in our discord: https://discord.gg/pKujYxD
+3. Running problems: ask in #help channel in our discord: https://discord.gg/pKujYxD
+
+
+If you are filing a bug report, please fill the fields below.
+Otherwise, feel free to remove this text and type a free-form issue as usual.
+
+BUG REPORT
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Steps to Reproduce**
+1. 
+2. 
+3. 
+4. 
+Expected behavior:
+Observed behavior:
+
+**Lc0 version**
+Include Lc0 version/operating system/backend type.
+
+**Lc0 parameters**
+Command line, if not default.
+Include screenshot of configuration window, if using through GUI.
+
+**Hardware**
+* Number and model of GPUs that you use.
+* Amount of RAM in the system
+* Other specs (CPU etc) if it may be relevant
+
+**Lc0 logs**
+Please attach Lc0 logs. Here is how to produce them (e.g. for D:\logfile.txt):
+
+Set the following UCI option:
+**Logfile:** D:\\logfile.txt
+OR
+pass this as a command line argument:
+`--logfile=D:\logfile.txt`
+OR
+Create **lc0.config** file in the same directory as your **lc0.exe** is located, with the following contents:
+```
+logfile=D:\logfile.txt
+```
+
+After running Lc0, **D:\logfile.txt** should appear.
+
+
+**Chess GUI logs**
+If there is a problem with particular GUI (cutechess/arena/etc), also attach logs of that program.

.gitignore

@@ -1,11 +1,14 @@
+!subprojects/*.wrap
+*.swp
+.clang_complete
+.DS_Store
+.clangd/
 build/
-testdata/
-LC0VSProj/
+compile_commands.json
 CUDA_NN/
-.DS_Store
-xcuserdata
-subprojects/*
-!subprojects/*.wrap
 lc0.xcodeproj/
-*.swp
-.clang_complete
+LC0VSProj/
+src/.vs/
+subprojects/*
+testdata/
+xcuserdata

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 These are the guidelines and standards followed by this codebase.
 
-The language is C++, specifically C++14. As such, manual `new` and `delete` memory mangement is strongly discouraged; use the standard library tools for managing memory (such as `unique_ptr`, `shared_ptr` etc.). When compiler support is more widespread, the project may upgrade to the C++17 standard in the future.
+The language is C++, specifically C++17. As such, manual `new` and `delete` memory mangement is strongly discouraged; use the standard library tools for managing memory (such as `unique_ptr`, `shared_ptr` etc.).
 
 This codebase uses semantic versioning. A release is the final commit for that version number, and all subsequent commits are development for the next version. `master` is the default branch, and the active development branch (as such, all Pull Requests go here); it always targets a minor (or major) version which succeeds the current relase. `release` is always equivalent to the latest tag.
 
@@ -56,21 +56,11 @@ The internal code dependency structure looks like this:
 
 ### Git history
 
-Pull Requests are squahsed when merged. This means all commits in the branch will be squashed into one commit applied onto master, so branches and their PRs should stick to *one* topic only. If you think changes deserve separate commits, make separate PRs for each commit.
+Pull Requests are squashed when merged. This means all commits in the branch will be squashed into one commit applied onto master, so branches and their PRs should stick to *one* topic only. If you think changes deserve separate commits, make separate PRs for each commit.
 
 This also means it's not possible to reuse one branch for multiple PRs; new PRs must either use entirely new branches, or else you could use `git reset --hard` on the current branch.
 
 
-### Command line/UCI options
-
-The options code handles both UCI options and command line options at the same time; in fact they are one and the same. Each option has a "flag name" and a "description". The flag name is used as the command line `--flag-name`. The description serves a dual purpose: it is the text printed by `./lc0 --help`, but it also serves as the *name* of the UCI option as well. Therefore the description should:
-
- * Not end with a period (per the UCI specification)
- * Be clear and succinct, to serve as both a help message and standalone UCI option name
- * Be short (to fit as a UCI option in chess GUIs)
- * Be different from the flag name (since it's a help message)
-
-
 ### Allowed features
 
 Lc0 is still in early stages of development, and has not yet reached the point where we are ready to add small tweaks to add few points of a rating. Large code changes still happen, and having lots of small optimizations adds overhead to larger changes, slowing development.

FLAGS.md

@@ -52,7 +52,7 @@ List of command line flags:
 | --fpu-reduction=NUM | First Play Urgency reduction | Default: `0.2` |
 | --cache-history-length=NUM | The length of history to include in the cache | Default: `7` |
 | --extra-virtual-loss=NUM | Extra virtual loss | Default: `0` |
-| -l,<br>--debuglog=FILENAME | Do debug logging into a file | Default is off (empty string) |
+| -l,<br>--logfile=FILENAME | Do debug logging into a file | Default is off (empty string) |
 
 
 ## Configuration Files
@@ -65,7 +65,7 @@ List of command line flags:
 # The -- is optional.  The following flags will work as well:
 weights=10445.txt.gz
 syzygy-paths=syzygy
-debuglog=lc0.log
+logfile=lc0.log
 ```
 You can tell `lc0` to ignore the default configuration file by passing `--config=` on the command line.  Command line arguments will override any arguments that also exist in the configuration file.
 

README.md

@@ -9,19 +9,27 @@ Lc0 is a UCI-compliant chess engine designed to play chess via neural network, s
 
 Lc0 can be acquired either via a git clone or an archive download from GitHub. Be aware that there is a required submodule which isn't included in source archives.
 
-For essentially all purposes, including selfplay game generation and match play, we highly recommend using the latest `release/version` branch (for example `release/0.19`), which is equivalent to using the latest version tag.
+For essentially all purposes, including selfplay game generation and match play, we highly recommend using the latest `release/version` branch (for example `release/0.25`), which is equivalent to using the latest version tag.
 
 Versioning follows the Semantic Versioning guidelines, with major, minor and patch sections. The training server enforces game quality using the versions output by the client and engine.
 
 
 Download using git:
 
 ```
-git clone -b release/0.21 --recurse-submodules https://github.com/LeelaChessZero/lc0.git
+git clone -b release/0.25 --recurse-submodules https://github.com/LeelaChessZero/lc0.git
 ```
 
+If you have cloned already an old version, fetch, view and checkout a new branch:
+```
+git fetch --all
+git branch --all
+git checkout -t remotes/origin/release/0.25
+```
+
+
 If you prefer to download an archive, you need to also download and place the submodule:
- * Download the [.zip](https://api.github.com/repos/LeelaChessZero/lc0/zipball/release/0.21) file ([.tar.gz](https://api.github.com/repos/LeelaChessZero/lc0/tarball/release/0.21) archive is also available)
+ * Download the [.zip](https://api.github.com/repos/LeelaChessZero/lc0/zipball/release/0.24) file ([.tar.gz](https://api.github.com/repos/LeelaChessZero/lc0/tarball/release/0.24) archive is also available)
  * Extract
  * Download https://github.com/LeelaChessZero/lczero-common/archive/master.zip (also available as [.tar.gz](https://github.com/LeelaChessZero/lczero-common/archive/master.tar.gz))
  * Move the second archive into the first archive's `libs/lczero-common/` folder and extract
@@ -34,9 +42,13 @@ Having successfully acquired Lc0 via either of these methods, proceed to the bui
 
 Building should be easier now than it was in the past. Please report any problems you have.
 
-Aside from the git submodule, lc0 requires the Meson build system and at least one backend library for evaluating the neural network, as well as the required libraries `protobuf` and `zlib`. (`gtest` is optionally used for the test suite.) If your system already has those two libraries installed, they will be used; otherwise Meson will generate its own copy of the two (a "subproject"), which in turn requires that git is installed (yes, separately from cloning the actual lc0 repository). Meson also requires python and Ninja.
+Aside from the git submodule, lc0 requires the Meson build system and at least one backend library for evaluating the neural network, as well as the required `zlib`. (`gtest` is optionally used for the test suite.) If your system already has this library installed, they will be used; otherwise Meson will generate its own copy of the two (a "subproject"), which in turn requires that git is installed (yes, separately from cloning the actual lc0 repository). Meson also requires python and Ninja.
 
-Backend support includes (in theory) any CBLAS-compatible library for CPU usage, such as OpenBLAS or Intel's MKL. For GPUs, OpenCL and CUDA+cudnn are supported.
+Backend support includes (in theory) any CBLAS-compatible library for CPU usage, such as OpenBLAS or Intel's DNNL or MKL. For GPUs, OpenCL and CUDA+cudnn are supported, while DX-12 can be used in Windows 10 with latest drivers.
+
+Finally, lc0 requires a compiler supporting C++17. Minimal versions seem to be g++ v8.0, clang v5.0 (with C++17 stdlib) or Visual Studio 2017.
+
+*Note* that cuda checks the compiler version and stops even with newer compilers, and to work around this we have added the `nvcc_ccbin` build option. This is more of an issue with new Linux versions, where we recommend to install `g++-7` and add `-Dnvcc_ccbin=g++-7` to the `build.sh` command.
 
 Given those basics, the OS and backend specific instructions are below.
 
@@ -80,33 +92,36 @@ in ("Deep Learning").
 
 #### Ubuntu 18.04
 
-For Ubuntu 18.04 you need the latest version of meson and clang-6.0 before performing the steps above:
+For Ubuntu 18.04 you need the latest version of meson, g++-8 and clang-6.0 before performing the steps above:
 
-    sudo apt-get install clang-6.0 ninja-build pkg-config protobuf-compiler libprotobuf-dev meson
+    sudo apt-get install gcc-8 g++-8 clang-6.0 ninja-build pkg-config
+    pip3 install meson --user
     CC=clang-6.0 CXX=clang++-6.0 INSTALL_PREFIX=~/.local ./build.sh
 
 Make sure that `~/.local/bin` is in your `PATH` environment variable. You can now type `lc0 --help` and start.
 
 #### Ubuntu 16.04
 
-For Ubuntu 16.04 you need the latest version of meson and clang-6.0 before performing the steps above:
+For Ubuntu 16.04 you need the latest version of meson, ninja and also gcc-8.0 before performing the steps above:
 
-    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
-    sudo apt-add-repository 'deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main'
+    sudo add-apt-repository ppa:ubuntu-toolchain-r/test
     sudo apt-get update
-    sudo apt-get install clang-6.0 ninja-build protobuf-compiler libprotobuf-dev
+    sudo apt-get install gcc-8 g++-8
     pip3 install meson --user
-    CC=clang-6.0 CXX=clang++-6.0 INSTALL_PREFIX=~/.local ./build.sh
+    pip3 install ninja --user
+    CC=gcc-8 CXX=g++-8 INSTALL_PREFIX=~/.local ./build.sh
 
 Make sure that `~/.local/bin` is in your `PATH` environment variable. You can now type `lc0 --help` and start.
 
+If you want to build with clang-6.0 you still need g++-8 for the library. Replace the last line above with:
+
+    sudo apt-get install clang-6.0
+    CC=clang-6.0 CXX=clang++-6.0 INSTALL_PREFIX=~/.local ./build.sh
 
 #### openSUSE (all versions)
 
-Instructions, packages and tools for building on openSUSE are at the following page
+Instructions, packages and tools for building on openSUSE are at [openSUSE_install.md](openSUSE_install.md)
 
-[openSUSE_install.md](openSUSE_install.md)
-=======
 #### Docker
 
 Use https://github.com/vochicong/lc0-docker
@@ -115,24 +130,23 @@ to run latest releases of lc0 and the client inside a Docker container.
 
 ### Windows
 
-0. Install Microsoft Visual Studio
-1. Install [CUDA](https://developer.nvidia.com/cuda-zone) (v9.2 is fine)
+Here are the brief instructions for CUDA/CuDNN, for details and other options see `windows-build.md`.
+
+0. Install Microsoft Visual Studio (2017 or later)
+1. Install [CUDA](https://developer.nvidia.com/cuda-zone)
 2. Install [cuDNN](https://developer.nvidia.com/cudnn).
 3. Install Python3
 4. Install Meson: `pip3 install --upgrade meson`
-5. Edit `build-cuda.cmd`:
+5. Edit `build.cmd`:
 
-* If you use MSVS other than 2015 (or if it's installed into non-standard location):
-    * `C:\Program Files (x86)\Microsoft Visual Studio 14.0\` replace 14.0 with your version
-    * `--backend 2015` replace 2015 with your version
-* `C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.2\lib\x64` replace with your CUDA path
-* `C:\dev\cuDNN\` replace with your cuDNN directory
+* Set `CUDA_PATH` with your CUDA directory
+* Set `CUDNN_PATH` with your cuDNN directory (may be the same with CUDA_PATH)
 
-6. Run `build-cuda.cmd`. It will generate MSVS project and pause.
+6. Run `build.cmd`. It will ask permission to delete the build directory, then generate MSVS project and pause.
 
 Then either:
 
-7. Hit <Enter> to build it.
+7. Hit `Enter` to build it.
 8. Resulting binary will be `build/lc0.exe`
 
 Or.
@@ -141,16 +155,23 @@ Or.
 
 ### Mac
 
+First you need to install some required packages:
 1. Install brew as per the instructions at https://brew.sh/
 2. Install python3: `brew install python3`
 3. Install meson: `brew install meson`
 4. Install ninja: `brew install ninja`
 5. When using Mojave install SDK headers: installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /
-6. Run `./build.sh`
-7. The resulting binary will be in build/release
+
+Now download the lc0 source, if you haven't already done so, following the instructions earlier in the page.
+
+6. Go to the lc0 directory.
+7. Run `./build.sh`
+8. The resulting binary will be in build/release
 
 ### Raspberry Pi
 
+You'll need to be running the latest raspbian "buster".
+
 1. Install OpenBLAS
 
 ```
@@ -165,19 +186,13 @@ cd ..
 
 ```
 pip3 install meson
+pip3 install ninja
 ```
 
-3. Install clang
+3. Install compiler and standard libraries
 
 ```
-wget http://releases.llvm.org/6.0.0/clang+llvm-6.0.0-armv7a-linux-gnueabihf.tar.xz
-tar -xf clang+llvm-6.0.0-armv7a-linux-gnueabihf.tar.xz
-rm clang+llvm-6.0.0-armv7a-linux-gnueabihf.tar.xz
-mv clang+llvm-6.0.0-armv7a-linux-gnueabihf clang_6.0.0
-sudo mv clang_6.0.0 /usr/local
-echo 'export PATH=/usr/local/clang_6.0.0/bin:~/.local/bin:$PATH' >> .bashrc
-echo 'export LD_LIBRARY_PATH=/usr/local/clang_6.0.0/lib:$LD_LIBRARY_PATH' >> .bashrc
-source .bashrc
+sudo apt install clang-6.0 libstdc++-8-dev
 ```
 
 4. Clone lc0 and compile
@@ -186,7 +201,7 @@ source .bashrc
 git clone https://github.com/LeelaChessZero/lc0.git
 cd lc0
 git submodule update --init --recursive
-CC=clang CXX=clang++ ./build.sh -Ddefault_library=static
+CC=clang-6.0 CXX=clang++-6.0 ./build.sh -Ddefault_library=static
 ```
 
 5. The resulting binary will be in build/release