Merge pull request #189 from dagster-io/0.2.0.develop

Upgrade master to 0.2.0

.circleci/config.yml

@@ -1,16 +1,23 @@
 version: 2
-jobs:
-  build:
-    docker:
-      # specify the version you desire here
-      # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers`
-      - image: circleci/python:3.6.1
 
-      # Specify service dependencies here if necessary
-      # CircleCI maintains a library of pre-built images
-      # documented at https://circleci.com/docs/2.0/circleci-images/
-      # - image: circleci/postgres:9.4
+workflows:
+  version: 2
+  test:
+    jobs:
+      - dagster-python-3.6
+      - dagster-python-3.5
+      - dagster-python-2.7
+      - dagit-python-3.6
+      - dagit-python-3.5
+      - dagit-python-2.7
+      - dagit-webapp
 
+jobs:
+  dagster-python-3.6: &dagster-template
+    docker:
+      - image: circleci/python:3.6.6
+    environment:
+      TOXENV: py36
     working_directory: ~/repo
 
     steps:
@@ -19,16 +26,96 @@ jobs:
       - run:
           name: Install Dependencies
           command: |
-            python3 -m venv venv
-            . venv/bin/activate
-            pip install tox
+            sudo pip install tox
 
       - run:
           name: Run Dagster Tests
           command: |
-            . venv/bin/activate
             cd python_modules/dagster
-            tox
+            tox -e $TOXENV
+
+      - store_artifacts:
+          path: test-reports
+          destination: test-reports
+
+  dagit-python-3.6: &dagit-template
+    <<: *dagster-template
+    steps:
+      - checkout
+
+      - run:
+          name: Install Dependencies
+          command: |
+            sudo pip install tox
+
+      - run:
+          name: Run Dagit Tests
+          command: |
+            cd python_modules/dagit
+            tox -e $TOXENV
+
+      - store_artifacts:
+          path: test-reports
+          destination: test-reports
+
+  dagster-python-3.5:
+    <<: *dagster-template
+    docker:
+      - image: circleci/python:3.5.6
+    environment:
+      TOXENV: py35
+
+  dagit-python-3.5:
+    <<: *dagit-template
+    docker:
+      - image: circleci/python:3.5.6
+    environment:
+      TOXENV: py35
+
+  dagster-python-2.7:
+    <<: *dagster-template
+    docker:
+      - image: circleci/python:2.7.15
+    environment:
+      TOXENV: py27
+
+  dagit-python-2.7:
+    <<: *dagit-template
+    docker:
+      - image: circleci/python:2.7.15
+    environment:
+      TOXENV: py27
+
+  dagit-webapp:
+    docker:
+      - image: circleci/node:10.6
+    working_directory: ~/repo
+    steps:
+      - checkout
+
+      - run:
+          name: Install Dependencies
+          command: |
+            cd python_modules/dagit/dagit/webapp
+            yarn
+
+      - run:
+          name: Run Typescript
+          command: |
+            cd python_modules/dagit/dagit/webapp
+            yarn run ts
+
+      - run:
+          name: Run Tests
+          command: |
+            cd python_modules/dagit/dagit/webapp
+            yarn run jest
+
+      - run:
+          name: Check coding style
+          command: |
+            cd python_modules/dagit/dagit/webapp
+            yarn run check-prettier
 
       - store_artifacts:
           path: test-reports

.gitignore

@@ -109,3 +109,4 @@ tags
 *.db
 
 docs/_build
+python_modules/dagster/docs/_build

.pre-commit-config.yaml

@@ -0,0 +1,4 @@
+-   repo: https://github.com/pre-commit/mirrors-yapf
+    sha: v0.22.0
+    hooks:
+    -   id: yapf

.read-the-docs-requirements.txt

@@ -0,0 +1 @@
+./python_modules/dagster

DEVELOPMENT.md

@@ -0,0 +1,42 @@
+Setup for developing the Dagit UI
+
+1. Install Python 3.6.
+  * You can't use Python 3.7+ yet because of https://github.com/apache/arrow/issues/1125
+
+2. Create a virtual environment
+  ```
+  python3 -m venv ~/venvs/dagit/
+  source ~/venvs/dagit/bin/activate
+  pip
+  ```
+
+3. Install all the dependencies, and make the dagit and dagster packages available by name:
+  ```
+  cd ./dagster
+  pip install -r dev-requirements.txt
+  pip install -e .
+  cd ../dagit
+  pip install -r dev-requirements.txt
+  pip install -e .
+  ```
+
+4. Run the GraphQL server from a directory that contains a repository.yml file.
+   For example:
+
+   ```
+   cd ./dagster/dagster/dagster_examples
+   python3.6 /path/to/python_modules/dagit/bin/dagit -p 3333
+   ```
+
+5. Run the JS component of the Dagit UI, pointing it to the GraphQL server:
+  ```
+  cd ./dagit/dagit/webapp
+  yarn install
+  REACT_APP_GRAPHQL_URI="http://localhost:3333/graphql" yarn start
+  ```
+
+6. You can run tests for python by running `tox` in `python_modules/dagit` or `python_modules/dagster`.
+
+7. Inside webapp directory, you can run tests too. `yarn test` runs all tests. In development it's handy to run `yarn run jest --watch` to have interactive test runner.
+
+8. Some webapp tests use *snapshots* - auto-generated results to which the test render tree is compared. Those tests are supposed to break when you change something, check that the change is sensible and run `yarn run jest -u` to update snapshot to the new result. You can also update snapshots interactively when you are in `--watch` mode.

README.rst

@@ -0,0 +1,24 @@
+.. image:: https://user-images.githubusercontent.com/28738937/44878798-b6e17e00-ac5c-11e8-8d25-2e47e5a53418.png
+   :align: center
+
+.. docs-include
+
+============
+Introduction
+============
+
+Dagster is an opinionated system and programming model for data pipelines. This process goes by
+many names -- ETL (extract-load-transform), ELT (extract-transform-load), model production, data
+integration, and so on -- but in essence they all describe the same activity: Performing a set of
+computations structured as a DAG (directed, acyclic graph) that end up producing data assets,
+whether those assets be tables, files, machine-learning models, etc.
+
+There are a few tools in this repo
+
+This repo has a few primary components:
+
+- **Dagster**: The core programming model and abstraction stack; a stateless single-node and -process execution engine; and a CLI tool for driving that engine.
+* **Dagit**: Dagit is a rich viewer for Dagster assets.
+* **Dagster GE**: A Dagster integration with Great Expectations. (see https://github.com/great-expectations/great_expectations)
+
+Go to https://dagster.readthedocs.io/en/latest/ for documentation!

python_modules/.pylintrc

@@ -20,4 +20,5 @@
 # R1705 Unnecessary else after return
 # R0902 Too many instance attributes. Sometimes big config objects are what you want
 # R1710 All returns should return an expression
-disable=C0111,C0330,C0103,duplicate-code,R0201,R0903,R0913,W0511,W1201,W1202,R1705,R0902,R1710,C0102
+# C0302 too many lines in single module
+disable=C0111,C0330,C0103,duplicate-code,R0201,R0903,R0913,W0511,W1201,W1202,R1705,R0902,R1710,C0102,C0302