forked from ibis-project/ibis
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile
183 lines (133 loc) · 6.23 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
.PHONY: all clean develop typecheck stop build start load restart init test testmost testfast testparams docclean doc black
SHELL := /bin/bash
MAKEFILE_DIR = $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
# from PYTHON_VERSION depends which `./ci/requirements-$PYTHON_VERSION-dev` file
# will be used for creating ibis image (see for additional info: `./ci/Dockerfile.dev`
# and `./ci/docker-compose.yml`)
# you can use `3.6` or `3.7` for now
PYTHON_VERSION := 3.6
PYTHONHASHSEED := random
# docker specific
COMPOSE_FILE := "$(MAKEFILE_DIR)/ci/docker-compose.yml"
DOCKER := PYTHON_VERSION=$(PYTHON_VERSION) docker-compose -f $(COMPOSE_FILE)
DOCKER_UP := $(DOCKER) up --remove-orphans -d --no-build
DOCKER_RUN := $(DOCKER) run --rm
DOCKER_BUILD := $(DOCKER) build
DOCKER_STOP := $(DOCKER) rm --force --stop
# command to be executed inside docker container
DOCKER_RUN_COMMAND := echo "you should do 'make docker_run DOCKER_RUN_COMMAND=[you command]'"
# all backends that ibis using
BACKENDS := clickhouse impala kudu-master kudu-tserver mysql omniscidb parquet postgres sqlite
# backends which are implemented as containers and can be launched through the `docker-compose`
SERVICES := omniscidb postgres mysql clickhouse impala kudu-master kudu-tserver
# the variable contains backends for which test datasets can be automatically loaded
LOADS := sqlite parquet postgres clickhouse omniscidb mysql impala
CURRENT_SERVICES := $(shell $(MAKEFILE_DIR)/ci/backends-to-start.sh "$(BACKENDS)" "$(SERVICES)")
CURRENT_LOADS := $(shell $(MAKEFILE_DIR)/ci/backends-to-start.sh "$(BACKENDS)" "$(LOADS)")
WAITER_COMMAND := $(shell $(MAKEFILE_DIR)/ci/dockerize.sh $(CURRENT_SERVICES))
# pytest specific options
PYTEST_MARKERS := $(shell $(MAKEFILE_DIR)/ci/backends-markers.sh $(BACKENDS))
PYTEST_DOCTEST_OPTIONS := --doctest-modules --doctest-ignore-import-errors
PYTEST_OPTIONS :=
REMOVE_COMPILED_PYTHON_SCRIPTS := (find /ibis -name "*.py[co]" -delete > /dev/null 2>&1 || true)
LOGLEVEL := WARNING
## Targets for code checks
typecheck:
@mypy --ignore-missing-imports $(MAKEFILE_DIR)/ibis
lint:
flake8
black:
# check that black formatting would not be applied
black --check .
check_pre_commit_hooks:
# check if all pre-commit hooks are passing
pre-commit run --all-files
## Targets for setup development environment
clean:
python setup.py clean
find $(MAKEFILE_DIR) -name '*.pyc' -type f -delete
rm -rf $(MAKEFILE_DIR)/build $(MAKEFILE_DIR)/dist $(find $(MAKEFILE_DIR) -name __pycache__ -type d)
develop: clean
python setup.py develop
pre-commit install
## DOCKER specific targets
# Targets for code checks inside containers
docker_lint: build
$(DOCKER_RUN) ibis flake8
docker_black: build
$(DOCKER_RUN) ibis black --check .
docker_check_pre_commit_hooks: build
# check if all pre-commit hooks are passing inside ibis container
$(DOCKER_RUN) ibis pre-commit run --all-files
# Targets for manipulating docker's containers
stop:
# stop all running docker compose services and remove its
$(DOCKER_STOP) $(CURRENT_SERVICES)
start:
# start all docker compose services
$(DOCKER_UP) $(CURRENT_SERVICES)
build:
# build the ibis image
$(DOCKER_BUILD) ibis
wait:
# wait for services to start
$(DOCKER_RUN) waiter $(WAITER_COMMAND)
DOCKER_CODE=$(shell echo $$?) ./ci/check-services.sh $(CURRENT_SERVICES)
load:
# load datasets for testing purpose
$(DOCKER_RUN) -e LOGLEVEL=$(LOGLEVEL) ibis ./ci/load-data.sh $(CURRENT_LOADS)
restart: stop
$(MAKE) start
$(MAKE) wait
init: restart
$(MAKE) build
$(MAKE) load
# Targets for testing ibis inside docker's containers
test: init
# use the target to run backend specific tests
$(DOCKER_RUN) -e PYTHONHASHSEED="$(PYTHONHASHSEED)" ibis bash -c "${REMOVE_COMPILED_PYTHON_SCRIPTS} && \
pytest $(PYTEST_DOCTEST_OPTIONS) $(PYTEST_OPTIONS) ${PYTEST_MARKERS} -k 'not test_import_time'"
testparallel: init
$(DOCKER_RUN) -e PYTHONHASHSEED="$(PYTHONHASHSEED)" ibis bash -c "${REMOVE_COMPILED_PYTHON_SCRIPTS} && \
pytest $(PYTEST_DOCTEST_OPTIONS) $(PYTEST_OPTIONS) ${PYTEST_MARKERS} -n auto -m 'not udf' -k 'not test_import_time'"
testall:
$(DOCKER_RUN) -e PYTHONHASHSEED="$(PYTHONHASHSEED)" ibis bash -c "${REMOVE_COMPILED_PYTHON_SCRIPTS} && \
pytest $(PYTEST_DOCTEST_OPTIONS) $(PYTEST_OPTIONS) -k 'not test_import_time'"
testmost:
$(DOCKER_RUN) -e PYTHONHASHSEED="$(PYTHONHASHSEED)" ibis bash -c "${REMOVE_COMPILED_PYTHON_SCRIPTS} && \
pytest $(PYTEST_DOCTEST_OPTIONS) $(PYTEST_OPTIONS) -n auto -m 'not (udf or impala or hdfs)' -k 'not test_import_time'"
testfast:
$(DOCKER_RUN) -e PYTHONHASHSEED="$(PYTHONHASHSEED)" ibis bash -c "${REMOVE_COMPILED_PYTHON_SCRIPTS} && \
pytest $(PYTEST_DOCTEST_OPTIONS) $(PYTEST_OPTIONS) -n auto -m 'not (udf or impala or hdfs or bigquery)' -k 'not test_import_time'"
testpandas:
$(DOCKER_RUN) -e PYTHONHASHSEED="$(PYTHONHASHSEED)" ibis bash -c "${REMOVE_COMPILED_PYTHON_SCRIPTS} && \
pytest $(PYTEST_DOCTEST_OPTIONS) $(PYTEST_OPTIONS) -n auto -m 'pandas' -k 'not test_import_time'"
testspark:
$(DOCKER_RUN) -e PYTHONHASHSEED="$(PYTHONHASHSEED)" ibis bash -c "${REMOVE_COMPILED_PYTHON_SCRIPTS} && \
pytest $(PYTEST_DOCTEST_OPTIONS) $(PYTEST_OPTIONS) -n auto -m 'pyspark' -k 'not test_import_time'"
fastopt:
@echo -m 'not (backend or bigquery or clickhouse or hdfs or impala or kudu or omniscidb or mysql or postgis or postgresql or superuser or udf)'
# Targets for documentation builds
docclean:
$(DOCKER_RUN) ibis-docs rm -rf /tmp/docs.ibis-project.org
builddoc: build
# build the ibis-docs image
$(DOCKER_BUILD) ibis-docs
doc: builddoc docclean
$(DOCKER_RUN) ibis-docs ping -c 1 impala
$(DOCKER_RUN) ibis-docs rm -rf /tmp/docs.ibis-project.org
$(DOCKER_RUN) ibis-docs git clone --branch gh-pages https://github.com/ibis-project/docs.ibis-project.org /tmp/docs.ibis-project.org --depth 1
$(DOCKER_RUN) ibis-docs find /tmp/docs.ibis-project.org \
-maxdepth 1 \
! -wholename /tmp/docs.ibis-project.org \
! -name '*.git' \
! -name '.' \
! -name CNAME \
! -name '*.nojekyll' \
-exec rm -rf {} \;
$(DOCKER_RUN) ibis-docs sphinx-build -b html docs/source /tmp/docs.ibis-project.org -W -T
# Targets for run commands inside ibis and ibis-docs containers
docker_run: build
$(DOCKER_RUN) ibis $(DOCKER_RUN_COMMAND)
docker_docs_run: builddoc
$(DOCKER_RUN) ibis-docs $(DOCKER_RUN_COMMAND)