forked from Mallik-G/sample-pyspark-application
-
Notifications
You must be signed in to change notification settings - Fork 0
/
.travis.yml
31 lines (31 loc) · 1.16 KB
/
.travis.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
language: python
python:
- 3.5
# sudo is required otherwise Spark fails with timeout errors. Seems to
# be an issue with networking while on Travis' container-based
# infrastructure.
sudo: required
cache:
- pip: true
before_install:
- curl -L -o spark.tgz https://s3.amazonaws.com/spark-related-packages/spark-2.2.0-bin-hadoop2.7.tgz
- export SPARK_HOME=./local/spark
- mkdir -p "$SPARK_HOME"
- tar -xf spark.tgz -C "$SPARK_HOME" --strip-components=1
- export PATH="$SPARK_HOME/bin:$PATH"
- export SPARK_LOCAL_IP="127.0.0.1"
# These lines here just suppress a lot of noisy log messages from Spark.
- echo "log4j.logger.org.apache.spark=WARN" > "$SPARK_HOME"/conf/log4j.properties
- echo "log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR" >> "$SPARK_HOME"/conf/log4j.properties
- echo "log4j.logger.org.apache.parquet=ERROR" >> "$SPARK_HOME"/conf/log4j.properties
install:
- pip install -U pip
- pip install -r requirements.pip
- pip install -r requirements-dev.pip
script:
# If you didn't have flake8 integrated into your test suite, you can
# just invoke it alone here.
# - flake8
- spark-submit run_tests.py
notifications:
email: false