Use xbuild instead of setup.py (#2276)

- Replace setup.py-based build process with the process based on xbuild;
- Added 'gitver' command for ext.py;
- Added make commands for installing different subsets of requirements: `make extra_install`, `make test_install` and `make docs_install`;

.travis.yml

@@ -27,22 +27,13 @@ python:
   - "3.5"
   - "3.6"
 
-install:
-  - pip install --upgrade setuptools
-  - pip install --upgrade python-dateutil
-  - pip install --upgrade numpy
-  - pip install --upgrade colorama
-  - pip install --upgrade docutils
-  - pip install --upgrade typesentry
-  - pip install --upgrade blessed
-  - pip install --upgrade pandas
-  - pip install --upgrade pytest
-  - pip install --upgrade virtualenv
-  - pip install --upgrade wheel
-
 script:
+  - pip install 'typesentry>=0.2.6' blessed
   - make test_install
-  - make
+  - make extra_install
+  - make gitver
+  - make build
+  - make test
 
 git:
   depth: 2

Makefile

@@ -21,7 +21,7 @@ PLATFORM := $(ARCH)-$(OS)
 DIST_DIR := dist/$(PLATFORM)
 
 .PHONY: all clean mrproper build install uninstall test_install test \
-		benchmark debug bi coverage dist fast
+		asan benchmark debug bi coverage dist fast xcoverage
 
 ifeq ($(MAKECMDGOALS), fast)
 -include ci/fast.mk
@@ -45,7 +45,6 @@ endif
 all:
 	$(MAKE) clean
 	$(MAKE) build
-	$(MAKE) install
 	$(MAKE) test
 
 
@@ -66,10 +65,6 @@ mrproper: clean
 	git clean -f -d -x
 
 
-build:
-	$(PYTHON) setup.py build
-	cp build/lib.*/_datatable.* datatable/lib/
-
 
 install:
 	$(PYTHON) -m pip install . --upgrade --no-cache-dir
@@ -79,8 +74,14 @@ uninstall:
 	$(PYTHON) -m pip uninstall datatable -y
 
 
+extra_install:
+	$(PYTHON) -m pip install -r requirements_extra.txt
+
 test_install:
-	$(PYTHON) -m pip install ${MODULE}[testing] --no-cache-dir
+	$(PYTHON) -m pip install -r requirements_tests.txt
+
+docs_install:
+	$(PYTHON) -m pip install -r requirements_docs.txt
 
 
 test:
@@ -91,56 +92,36 @@ test:
 		--junitxml=build/test-reports/TEST-datatable.xml \
 		tests
 
-xasan:
+# In order to run with Address Sanitizer:
+#   $ make asan
+#   $ DYLD_INSERT_LIBRARIES=/usr/local/opt/llvm/lib/clang/7.0.0/lib/darwin/libclang_rt.asan_osx_dynamic.dylib ASAN_OPTIONS=detect_leaks=1 python -m pytest
+#
+asan:
 	@$(PYTHON) ext.py asan
 
-xbuild:
+build:
 	@$(PYTHON) ext.py build
 
 xcoverage:
 	@$(PYTHON) ext.py coverage
 
-xdebug:
-	@$(PYTHON) ext.py debug
-
-
-benchmark:
-	$(PYTHON) -m pytest -ra -x -v benchmarks
-
-
 debug:
-	$(MAKE) clean
-	DTDEBUG=1 \
-	$(MAKE) build
-	$(MAKE) install
-
+	@$(PYTHON) ext.py debug
 
-# In order to run with Address Sanitizer:
-#   $ make asan
-#   $ DYLD_INSERT_LIBRARIES=/usr/local/opt/llvm/lib/clang/7.0.0/lib/darwin/libclang_rt.asan_osx_dynamic.dylib ASAN_OPTIONS=detect_leaks=1 python -m pytest
-#
-asan:
-	$(MAKE) clean
-	DTASAN=1 $(MAKE) fast
+gitver:
+	@$(PYTHON) ext.py gitver
 
-bi:
-	$(MAKE) build
-	$(MAKE) install
 
 
 coverage:
-	$(eval DTCOVERAGE := 1)
-	$(eval export DTCOVERAGE)
-	$(MAKE) clean
-	$(MAKE) build
-	$(MAKE) install
+	$(PYTHON) -m pip install 'typesentry>=0.2.6' blessed
+	$(MAKE) xcoverage
 	$(MAKE) test_install
-	$(PYTHON) -m pytest -x \
-		--benchmark-skip \
+	$(MAKE) gitver
+	DTCOVERAGE=1 $(PYTHON) -m pytest -x \
 		--cov=datatable --cov-report=html:build/coverage-py \
 		tests
-	$(PYTHON) -m pytest -x \
-		--benchmark-skip \
+	DTCOVERAGE=1 $(PYTHON) -m pytest -x \
 		--cov=datatable --cov-report=xml:build/coverage.xml \
 		tests
 	chmod +x ci/llvm-gcov.sh
@@ -156,17 +137,15 @@ coverage:
 	genhtml --legend --output-directory build/coverage-c --demangle-cpp build/coverage.info
 	mv .coverage build/
 
-dist: build
-	$(PYTHON) setup.py bdist_wheel -d $(DIST_DIR)
+dist:
+	@$(PYTHON) ext.py wheel -d $(DIST_DIR)
 
 sdist:
-	$(PYTHON) setup.py sdist
+	@$(PYTHON) ext.py sdist
 
 version:
 	@$(PYTHON) ci/setup_utils.py version
 
-nothing:
-	@echo Nothing
 
 
 #-------------------------------------------------------------------------------
@@ -439,9 +418,6 @@ ubuntu_test_py36_with_numpy_in_docker:
 ubuntu_test_py36_in_docker:
 	$(MAKE) TEST_VENV=datatable-py36 ubuntu_test_in_docker_impl
 
-# Note:  We don't actually need to run mrproper in docker (as root) because
-#        the build step runs as the user.  But keep the API for consistency.
-mrproper_in_docker: mrproper
 
 printvars:
 	@echo PLATFORM=$(PLATFORM)

ci/setup_utils.py

@@ -159,11 +159,12 @@ def get_datatable_version():
             log.info("Appending suffix from CI_VERSION_SUFFIX = " + suffix)
             mm = re.match(r"(?:master|dev)[.+_-]?(\d+)", suffix)
             if mm:
-                suffix = "dev" + str(mm.group(1))
-            version += "." + suffix
+                version += ".dev" + str(mm.group(1))
+            else:
+                log.warn("Invalid CI_VERSION_SUFFIX: `%s`" % suffix)
             log.info("Final version = " + version)
         else:
-            log.info("Environment variable CI_VERSION_SUUFFIX not present")
+            log.info("Environment variable CI_VERSION_SUFFIX not present")
         return version
 
 

ext.py

@@ -32,6 +32,7 @@
 import sys
 import textwrap
 from ci import xbuild
+from ci.setup_utils import get_datatable_version, make_git_version_file
 
 
 def create_logger(verbosity):
@@ -140,7 +141,7 @@ def build_extension(cmd, verbosity=3):
 def get_meta():
     return dict(
         name="datatable",
-        version="0.10.1",
+        version=get_datatable_version(),
 
         summary="Python library for fast multi-threaded data manipulation and "
                 "munging.",
@@ -262,7 +263,8 @@ def main():
         formatter_class=argparse.RawTextHelpFormatter
     )
     parser.add_argument("cmd", metavar="CMD",
-        choices=["asan", "build", "coverage", "debug", "sdist", "wheel"],
+        choices=["asan", "build", "coverage", "debug", "gitver", "sdist",
+                 "wheel"],
         help=textwrap.dedent("""
             Specify what this script should do:
 
@@ -272,12 +274,15 @@ def main():
                        testing
             debug    : build _datatable in debug mode, optimized for gdb
                        on Linux and for lldb on MacOS
+            gitver   : generate __git__.py file
             sdist    : create source distribution of datatable
             wheel    : create wheel distribution of datatable
             """).strip())
     parser.add_argument("-v", dest="verbosity", action="count", default=1,
         help="Verbosity level of the output, specify the parameter up to 3\n"
              "times for maximum verbosity; the default level is 1.")
+    parser.add_argument("-d", dest="destination", default="dist",
+        help="Destination directory for `sdist` and `wheel` commands.")
     parser.add_argument("--audit", action="store_true",
         help="This flag can be used with cmd='wheel' only, on a Linux\n"
              "platform, which must have the 'auditwheel' external tool\n"
@@ -288,16 +293,20 @@ def main():
              "tag. Otherwise, an error will be raised.")
 
     args = parser.parse_args()
+    if args.cmd in ["sdist", "wheel"]:
+        make_git_version_file(True)
     if args.audit and "linux" not in sys.platform:
         raise ValueError("Argument --audit can be used on a Linux platform "
                          "only, current platform is `%s`" % sys.platform)
 
     if args.cmd == "wheel":
-        wheel_file = build_wheel("dist/", {"audit": args.audit})
+        wheel_file = build_wheel(args.destination, {"audit": args.audit})
         assert os.path.isfile(os.path.join("dist", wheel_file))
     elif args.cmd == "sdist":
-        sdist_file = build_sdist("dist/")
+        sdist_file = build_sdist(args.destination)
         assert os.path.isfile(os.path.join("dist", sdist_file))
+    elif args.cmd == "gitver":
+        make_git_version_file(True)
     else:
         with open("datatable/lib/.xbuild-cmd", "wt") as out:
             out.write(args.cmd)

pyproject.toml

@@ -1,6 +1,7 @@
 [build-system]
-requires = ["setuptools >= 42.0", "wheel"]
-build-backend = "setuptools.build_meta:__legacy__"
+# requires = ["setuptools >= 42.0", "wheel"]
+# build-backend = "setuptools.build_meta:__legacy__"
 # requires = ["pip >= 20.0"]
-# build-backend = "ext"
-# backend-path = ["."]
+requires = []
+build-backend = "ext"
+backend-path = ["."]

requirements_docs.txt

@@ -1,4 +1,5 @@
 docutils>=0.14
 sphinx>=1.8
+sphinx_rtd_theme
 nbsphinx>=0.3.5
-https://s3.amazonaws.com/artifacts.h2o.ai/releases/ai/h2o/pydatatable/0.9.0/x86_64-centos7/datatable-0.9.0-cp37-cp37m-linux_x86_64.whl
+datatable

requirements_extra.txt

@@ -0,0 +1,3 @@
+numpy
+pandas
+xlrd

requirements_tests.txt

@@ -0,0 +1,3 @@
+pytest >=3.1
+pytest-cov
+docutils >=0.14