diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000000..281e59bf11 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,23 @@ +name: Publish Documentation +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + name: Publish Documentation + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install ghp-import + run: pip install ghp-import + - name: Build documentation + run: "./pants docs ::" + - name: Publish Documentation + run: ghp-import --push --force --no-history --no-jekyll dist/sphinx/ diff --git a/.gitignore b/.gitignore index 11ea8fa9dd..1771a8054c 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,9 @@ webpack-stats.json # Celery Beat celerybeat-schedule + +# pants +/.pants.d/ +/dist/ +/.pids +/.pants.workdir.file_lock* diff --git a/BUILD b/BUILD new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/BUILD b/docs/BUILD new file mode 100644 index 0000000000..87caa7f4bc --- /dev/null +++ b/docs/BUILD @@ -0,0 +1,6 @@ + + +sphinx_docs( + name="sphinx-docs", + source_directory="source" +) diff --git a/docs/configure_open_edx.md b/docs/configure_open_edx.md deleted file mode 100644 index 83a0b0d2e1..0000000000 --- a/docs/configure_open_edx.md +++ /dev/null @@ -1,194 +0,0 @@ -# Configure Open edX - -In order to create user accounts in Open edX and permit authentication from mitX Online to Open edX, you need to configure mitX Online as an OAuth2 provider for Open edX. - -## Add `/etc/hosts` alias for Open edX - -If one doesn't already exist, add an alias to `/etc/hosts` for Open edX. We have standardized this alias -to `edx.odl.local`. Your `/etc/hosts` entry should look like this: - -``` -127.0.0.1 edx.odl.local -``` - -## Setup Open edX Devstack - -Following steps are inspired by [edx-devstack](https://github.com/edx/devstack). - -### Clone edx/devstack - -``` -$ git clone https://github.com/edx/devstack -$ cd devstack -$ git checkout open-release/ironwood.master -$ make requirements -$ export OPENEDX_RELEASE=ironwood.master -$ make dev.clone -``` - -### Clone and checkout edx-platform (if not already). -``` -$ git clone https://github.com/mitodl/edx-platform -$ git checkout master -``` - -### Pull latest images and run provision - -``` -$ make pull -$ make dev.provision -``` - -### Start your servers - -`make dev.up` - -### Stop your servers - -`make stop` - -## Setup social auth - -### Install `social-auth-mitxpro` in LMS - -There are two options for this: - -#### Install via pip - -- `pip install social-auth-mitxpro` - -#### Install from local Build - -- Checkout the [social-auth-mitxpro](https://github.com/mitodl/social-auth-mitxpro) project and build the package per the project instructions -- Copy the `social-auth-mitxpro-$VERSION.tar.gz` file into devstack's `edx-platform` directory -- In devstack, run `make lms-shell` and within that shell `pip install social-auth-mitxpro-$VERSION.tar.gz` - - To update to a new development version without having to actually bump the package version, simply `pip uninstall social-auth-mitxpro`, then install again - -### Install `mitxpro-openedx-extensions` in LMS - -There are two options for this: - -#### Install via pip - -- `pip install mitxpro-openedx-extensions` - -#### Install from local Build - -- Checkout the [mitxpro-openedx-extensions](https://github.com/mitodl/mitxpro-openedx-extensions) project and build the package per the project instructions -- Copy the `mitxpro-openedx-extensions-$VERSION.tar.gz` file into devstack's `edx-platform` directory -- In devstack, run `make lms-shell` and within that shell `pip install mitxpro-openedx-extensions-$VERSION.tar.gz` - - To update to a new development version without having to actually bump the package version, simply `pip uninstall -y mitxpro-openedx-extensions`, then install again - -### Configure mitX Online as a OAuth provider for Open edX - -In mitX Online: - -- go to `/admin/oauth2_provider/application/` and create a new application with these settings selected: - - `Redirect uris`: `http://:18000/auth/complete/mitxpro-oauth2/` - - _[OSX users]_ You will need redirect uris for both the local edX host alias and for `host.docker.internal`. This value should be: - ```shell - http://edx.odl.local:18000/auth/complete/mitxpro-oauth2/ - http://host.docker.internal:18000/auth/complete/mitxpro-oauth2/ - ``` - - _[Linux users]_ You will need redirect uris for both the local edX host alias and for the gateway IP of the docker-compose networking setup for mitX Online as found via `docker network inspect mitx-online_default` - ```shell - http://edx.odl.local:18000/auth/complete/mitxpro-oauth2/ - http://:18000/auth/complete/mitxpro-oauth2/ - # `GATEWAY_IP` should be something like `172.19.0.1`. - ``` - - - `Client type`: "Confidential" - - `Authorization grant type`: "Authorization code" - - `Skip authorization`: checked - - Other values are arbitrary but be sure to fill them all out. Save the client id and secret for later - -In Open edX (derived from instructions [here](https://edx.readthedocs.io/projects/edx-installing-configuring-and-running/en/latest/configuration/tpa/tpa_integrate_open/tpa_oauth.html#additional-oauth2-providers-advanced)): -- `make lms-shell` into the LMS container and ensure the following settings: - - `/edx/app/edxapp/lms.env.json`: - ``` - { - ... - "FEATURES": { - ... - "ALLOW_PUBLIC_ACCOUNT_CREATION": true, - "ENABLE_COMBINED_LOGIN_REGISTRATION": true, - "ENABLE_THIRD_PARTY_AUTH": true, - "ENABLE_OAUTH2_PROVIDER": true, - "SKIP_EMAIL_VALIDATION": true, - ... - }, - ... - "REGISTRATION_EXTRA_FIELDS": { - ... - "country": "hidden", - ... - }, - ... - "THIRD_PARTY_AUTH_BACKENDS": ["social_auth_mitxpro.backends.MITxProOAuth2"], - ... - } - ``` - - `/edx/app/edxapp/lms.auth.json`: - ``` - { - ... - "SOCIAL_AUTH_OAUTH_SECRETS": { - "mitxpro-oauth2": "" - }, - ... - } - ``` -- `make lms-restart` to pick up the configuration changes -- Login to django-admin, go to `http://:18000/admin/third_party_auth/oauth2providerconfig/`, and create a new config: - - Select the default example site - - The slug field **MUST** match the `Backend.name`, which for us is ` -mitxpro-oauth2` - - Client Id should be the client id from the mitX Online Django Oauth Toolkit Application - - Check the following checkboxes: - - Skip hinted login dialog - - Skip registration form - - Sync learner profile data - - Enable SSO id verification - - In "Other settings", put: - ``` - { - "AUTHORIZATION_URL": "http://:8013/oauth2/authorize/", - "ACCESS_TOKEN_URL": "http://:8013/oauth2/token/", - "API_ROOT": "http://:8013/" - } - ``` - - `LOCAL_MITX_ONLINE_ALIAS` should be your `/etc/hosts` alias for the mitxonline app - - `EXTERNAL_MITX_ONLINE_HOST` will depend on your OS, but it needs to be resolvable within the edx container - - Linux users: The gateway IP of the docker-compose networking setup for mitxonline as found via `docker network inspect mitx-online_default` - - OSX users: Use `host.docker.internal` - - - -### Configure Open edX to support OAuth2 authentication from mitX Online - - - In Open edX: - - go to `/admin/oauth2_provider/application/` and verify that an application named 'edx-oauth-app' exists with these settings: - - `Redirect uris`: `http://mitxonline.odl.local:8013/login/_private/complete` - - `Client type`: "Confidential" - - `Authorization grant type`: "Authorization code" - - `Skip authorization`: checked - - Other values are arbitrary but be sure to fill them all out. Save the client id and secret for later - - In mitX Online: - - Set `OPENEDX_API_CLIENT_ID` to the client id - - Set `OPENEDX_API_CLIENT_SECRET` to the client secret - - -### Configure Logout - - - In Open edX, configure `settings.IDA_LOGOUT_URI_LIST` to be a list including the full url to `://[:]/logout` in mitX Online - - For devstack, this means modifying the value in `edx-platform/lms/envs/devstack.py` to include `http://mitxonline.odl.local:8013/logout` - - For production, this setting can go in `lms.env.json` under the key `IDA_LOGOUT_URI_LIST` as a JSON array of with that string in it - - - mitX Online: - - Set `LOGOUT_REDIRECT_URL` to the full path to the edx `/logout` view. - - For local development this will be `http://:18000/logout` - - -### Configure Open edX user and token for use with mitX Online management commands - -- In Open edX, create a staff user and then under `/admin/oauth2_provider/accesstoken/` add access token. The value of said token needs to match the value set for the `OPENEDX_SERVICE_WORKER_API_TOKEN` key in the mitX Online app. diff --git a/docs/source/assets/authentication-flow.mm b/docs/source/assets/authentication-flow.mm new file mode 100644 index 0000000000..7c75f42310 --- /dev/null +++ b/docs/source/assets/authentication-flow.mm @@ -0,0 +1,25 @@ +sequenceDiagram + participant MO as MITx Online + participant OE as Open edX + + par Create Account in Open edx + MO->>OE: POST /user_api/v1/account/registration/ + OE-->>MO: Success + end + + par Create Open edX Access Token + Note right of MO: Create in-memory requests session + par Establish an Open edX session + MO->>OE: GET /auth/login/mitxpro-oauth2/?auth_entry=login + OE->>MO: Redirect to GET /oauth2/authorize + MO->>OE: Redirect to GET /auth/complete/mitxpro-oauth2/ + end + + par Link MITx Online account to Open edX Account + MO->>OE: GET /oauth2/authorize + OE-->>MO: Redirect to GET /login/_private/complete + MO->>OE: POST /oauth2/access_token + OE-->>MO: OAuth access and refresh tokens + end + + end diff --git a/docs/source/authentication.rst b/docs/source/authentication.rst new file mode 100644 index 0000000000..d312f39f23 --- /dev/null +++ b/docs/source/authentication.rst @@ -0,0 +1,4 @@ +Authentication +============== + +.. mermaid:: assets/authentication-flow.mm diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000000..f5fd7652e6 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,54 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'MITx Online' +copyright = '2021, OL Engineering' +author = 'OL Engineering' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinxcontrib.mermaid', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_permalinks_icon = '§' +html_theme = 'insipid' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] diff --git a/docs/source/configuration/index.rst b/docs/source/configuration/index.rst new file mode 100644 index 0000000000..b245316c13 --- /dev/null +++ b/docs/source/configuration/index.rst @@ -0,0 +1,7 @@ +Configuration +============= + +.. toctree:: + :maxdepth: 2 + + open_edx diff --git a/docs/source/configuration/open_edx.rst b/docs/source/configuration/open_edx.rst new file mode 100644 index 0000000000..639e45ab30 --- /dev/null +++ b/docs/source/configuration/open_edx.rst @@ -0,0 +1,218 @@ +Configure Open edX +================== + +In order to create user accounts in Open edX and permit authentication from MITx Online to Open edX, you need to configure MITx Online as an OAuth2 provider for Open edX. + +Setup Open edX Devstack +####################### + +Following steps are inspired by `edx-devstack `_. + +Add ``/etc/hosts`` alias for Open edX +------------------------------------- + +If one doesn't already exist, add an alias to ``/etc/hosts`` for Open edX. We have standardized this alias +to ``edx.odl.local``. Your ``/etc/hosts`` entry should look like this:: + + 127.0.0.1 edx.odl.local + +Clone edx/devstack +------------------ + +.. code-block:: shell + + git clone https://github.com/edx/devstack + cd devstack + git checkout open-release/ironwood.master + make requirements + export OPENEDX_RELEASE=ironwood.master + make dev.clone + +Clone and checkout edx-platform (if not already) +------------------------------------------------ + +.. code-block:: shell + + git clone https://github.com/mitodl/edx-platform + git checkout master + +Pull latest images and run provision +------------------------------------ + +.. code-block:: shell + + make pull + make dev.provision + + +Start your servers +------------------ + +.. code-block:: shell + + make dev.up + +Stop your servers +----------------- + +.. code-block:: shell + + make stop + +Setup social auth +################# + +Install ``social-auth-mitxpro`` in LMS +-------------------------------------- + +There are two options for this: + +Install via pip +^^^^^^^^^^^^^^^ + +.. code-block:: shell + + pip install social-auth-mitxpro + +Install from local Build +^^^^^^^^^^^^^^^^^^^^^^^^ + +* Checkout the `social-auth-mitxpro `_ project and build the package per the project instructions +* Copy the ``social-auth-mitxpro-$VERSION.tar.gz`` file into devstack's ``edx-platform`` directory +* In devstack, run ``make lms-shell`` and within that shell ``pip install social-auth-mitxpro-$VERSION.tar.gz`` + + * To update to a new development version without having to actually bump the package version, simply ``pip uninstall social-auth-mitxpro``, then install again + +Install ``mitxpro-openedx-extensions`` in LMS +--------------------------------------------- + +There are two options for this: + +Install via pip +^^^^^^^^^^^^^^^ + +.. code-block:: shell + + pip install mitxpro-openedx-extensions + +Install from local Build +^^^^^^^^^^^^^^^^^^^^^^^^ + +* Checkout the `mitxpro-openedx-extensions `_ project and build the package per the project instructions +* Copy the ``mitxpro-openedx-extensions-$VERSION.tar.gz`` file into devstack's ``edx-platform`` directory +* In devstack, run ``make lms-shell`` and within that shell ``pip install mitxpro-openedx-extensions-$VERSION.tar.gz`` + + * To update to a new development version without having to actually bump the package version, simply ``pip uninstall -y mitxpro-openedx-extensions``, then install again + +Configure MITx Online as a OAuth provider for Open edX +###################################################### + +In MITx Online: + +* go to ``/admin/oauth2_provider/application/`` and create a new application with these settings selected: + + * ``Redirect uris``: ``http://:18000/auth/complete/mitxpro-oauth2/`` + + * **[OSX users]** You will need redirect uris for both the local edX host alias and for ``host.docker.internal``. This value should be:: + + http://edx.odl.local:18000/auth/complete/mitxpro-oauth2/ + http://host.docker.internal:18000/auth/complete/mitxpro-oauth2/ + + * **[Linux users]** You will need redirect uris for both the local edX host alias and for the gateway IP of the docker-compose networking setup for MITx Online as found via ``docker network inspect mitx-online_default``:: + + http://edx.odl.local:18000/auth/complete/mitxpro-oauth2/ + http://:18000/auth/complete/mitxpro-oauth2/ + + + NOTE:``GATEWAY_IP`` should be something like ``172.19.0.1``. + + * ``Client type``: "Confidential" + * ``Authorization grant type``: "Authorization code" + * ``Skip authorization``: checked + * Other values are arbitrary but be sure to fill them all out. Save the client id and secret for later + +In Open edX (derived from instructions `here `_): + +* ``make lms-shell`` into the LMS container and ensure the following settings: + .. code-block:: yaml + :caption: /edx/etc/lms.yml + + FEATURES: + ALLOW_PUBLIC_ACCOUNT_CREATION: true + ENABLE_COMBINED_LOGIN_REGISTRATION: true + ENABLE_THIRD_PARTY_AUTH: true + ENABLE_OAUTH2_PROVIDER: true + SKIP_EMAIL_VALIDATION: true + REGISTRATION_EXTRA_FIELDS: + country: hidden + THIRD_PARTY_AUTH_BACKENDS: + - social_auth_mitxpro.backends.MITxProOAuth2 + +* ``make lms-restart`` to pick up the configuration changes +* Login to django-admin, go to ``http://:18000/admin/third_party_auth/oauth2providerconfig/``, and create a new config: + + * Select the default example site + * The slug field **MUST** match the the backend's name, which for us is ``mitxpro-oauth2`` + * Client Id should be the client id from the MITx Online Django Oauth Toolkit Application + * Check the following checkboxes: + + * Skip hinted login dialog + * Skip registration form + * Sync learner profile data + * Enable SSO id verification + + * In "Other settings", put: + + .. code-block:: json + + { + "AUTHORIZATION_URL": "http://:8013/oauth2/authorize/", + "ACCESS_TOKEN_URL": "http://:8013/oauth2/token/", + "API_ROOT": "http://:8013/" + } + + * ``LOCAL_MITX_ONLINE_ALIAS`` should be your ``/etc/hosts`` alias for the mitxonline app + * ``EXTERNAL_MITX_ONLINE_HOST`` will depend on your OS, but it needs to be resolvable within the edx container + + * Linux users: The gateway IP of the docker-compose networking setup for mitxonline as found via ``docker network inspect mitx-online_default`` + * OSX users: Use ``host.docker.internal`` + + + +Configure Open edX to support OAuth2 authentication from MITx Online +#################################################################### + +* In Open edX: + + * go to ``/admin/oauth2_provider/application/`` and verify that an application named 'edx-oauth-app' exists with these settings: + + * ``Redirect uris``: ``http://mitxonline.odl.local:8013/login/_private/complete`` + * ``Client type``: "Confidential" + * ``Authorization grant type``: "Authorization code" + * ``Skip authorization``: checked + * Other values are arbitrary but be sure to fill them all out. Save the client id and secret for later + +* In MITx Online: + + * Set ``OPENEDX_API_CLIENT_ID`` to the client id + * Set ``OPENEDX_API_CLIENT_SECRET`` to the client secret + +Configure Logout +################ + +* In Open edX, configure ``settings.IDA_LOGOUT_URI_LIST`` to be a list including the full url to ``://[:]/logout`` in MITx Online + + * For devstack, this means modifying the value in ``edx-platform/lms/envs/devstack.py`` to include ``http://mitxonline.odl.local:8013/logout`` + * For production, this setting can go in ``lms.env.json`` under the key ``IDA_LOGOUT_URI_LIST`` as a JSON array of with that string in it + +* MITx Online: + + * Set ``LOGOUT_REDIRECT_URL`` to the full path to the edx ``/logout`` view. + +For local development this will be ``http://:18000/logout`` + + +Configure Open edX user and token for use with MITx Online management commands +############################################################################## + +* In Open edX, create a staff user and then under ``/admin/oauth2_provider/accesstoken/`` add access token. The value of said token needs to match the value set for the ``OPENEDX_SERVICE_WORKER_API_TOKEN`` key in the MITx Online app. diff --git a/docs/source/ecommerce/assets/ecommerce-architecture.mm b/docs/source/ecommerce/assets/ecommerce-architecture.mm new file mode 100644 index 0000000000..fff9123bdc --- /dev/null +++ b/docs/source/ecommerce/assets/ecommerce-architecture.mm @@ -0,0 +1,14 @@ +flowchart TB + product_subsystem(Product Subsystem) + basket_subsystem(Basket Subsystem) + order_subsystem(Order Subsystem) + discount_subsystem(Discount Subsystem) + payment_subsystem(Payment Subsystem) + + cybersource(CyberSource) + + basket_subsystem --> order_subsystem + discount_subsystem --> order_subsystem + order_subsystem & basket_subsystem & discount_subsystem --> product_subsystem + order_subsystem <--> payment_subsystem + payment_subsystem <--> cybersource diff --git a/docs/source/ecommerce/index.rst b/docs/source/ecommerce/index.rst new file mode 100644 index 0000000000..6b006ed085 --- /dev/null +++ b/docs/source/ecommerce/index.rst @@ -0,0 +1,8 @@ +Ecommerce +========= + +.. toctree:: + :maxdepth: 2 + + overview + subsystems/index diff --git a/docs/source/ecommerce/overview.rst b/docs/source/ecommerce/overview.rst new file mode 100644 index 0000000000..916ec66478 --- /dev/null +++ b/docs/source/ecommerce/overview.rst @@ -0,0 +1,31 @@ +Overview +========= + +Goals +***** + +We will be creating a robust ecommerce implementation, incorporating learnings from the last several years to implement it in a scalable and reusable way. A good reference point is this guide on Pythonic SOLID Principle. We should also strongly strive towards keeping coupling between the subsystems proposed here to a minimum or at least limited in surface area. +Core + +The core of the ecommerce system should be simple enough to configure and operate but support enough functionality to serve a majority of our use cases. Users should be able to see programs or course runs, select them for purchase, and make a payment. + +Prior Art +********* + +We have a few implementations of ecommerce we’ve created over the years: + +MicroMasters +^^^^^^^^^^^^ +The MicroMasters implementation is highly specialized, particularly around financial aid programs where each learner gets custom pricing. Incorporating this level of complexity into the core of the ecommerce system is not something we want to do, but we should carve out some options to extend the system in the future without implementing it in the core system. + +xPro +^^^^ + +xPro ecommerce was implemented based on our experiences implementing ecommerce in MicroMasters. A good amount of planning went into this implementation, although it also has some specializations we wouldn’t be using in MITx Online such as a vouchers system. We will probably borrow heavily from the core designs that were proved out here. + +Core Systems +************ + +Ecommerce is actually a combination of 3 discernable subsystems that often get muddled together: products, orders, and payment. See the high-level diagram below to understand the pieces of data and operations that happen. + +.. mermaid:: assets/ecommerce-architecture.mm diff --git a/docs/source/ecommerce/subsystems/basket.rst b/docs/source/ecommerce/subsystems/basket.rst new file mode 100644 index 0000000000..a63398f79b --- /dev/null +++ b/docs/source/ecommerce/subsystems/basket.rst @@ -0,0 +1,30 @@ +Basket Subsystem +================ + +This tracks products intended to be purchased, often providing some additional state such as which runs under a program a user is purchasing. + +A simple schema for this would be: + +.. code-block:: python + + class Basket(TimestampedModel): + """Represents a User's basket.""" + + user = models.OneToOneField(settings.AUTH_USER_MODEL) + + class BasketItem(TimestampedModel): + """Represents one or more products in a user's basket.""" + + product = models.ForeignKey(Product) + basket = models.ForeignKey(Basket) + quantity = models.PositiveIntegerField() + +APIs +^^^^ + +- ``GET /api/v0/basket/`` -> get the current basket state +- ``POST /api/v0/basket/`` -> update the basket state + +Notes +^^^^^ +The implementation of this would use the discount subsystem to calculate the discounted prices, those values would be returned in the API for the frontend to use for display purposes. diff --git a/docs/source/ecommerce/subsystems/discount.rst b/docs/source/ecommerce/subsystems/discount.rst new file mode 100644 index 0000000000..3b0f8b09e8 --- /dev/null +++ b/docs/source/ecommerce/subsystems/discount.rst @@ -0,0 +1,101 @@ +Discount Subsystem +================== + +Discounts will need to be provided on occasion, these give the user a reduced price for some or all products. Treating this as a discount system and not necessarily a coupon system (e.g. a coupon is a kind of discount) is a good way to frame this approach. + +The discount system would support discounts of multiple types. We’ve done discounts a lot of different ways before so we need to balance out flexibility against keeping complexity down. Each discount would be associated with a certain Product. + +A discount may optionally be pre-associated with a User so that it can be automatically applied on checkout. + +Discounts should only be computed on the backend, some of our ecommerce implementations have computed the discount on the frontend and we want to avoid this going forward. + +Discount Types +^^^^^^^^^^^^^^ +Discount types would track how the discounted price is computed, some examples/ideas: +``percent-off``: a percentage off the original price +``dollars-off``: a fixed dollar reduction in price (e.g. $30 off) +``fixed-price``: the price is discounted to the fixed price (e.g. a product would cost $100 regardless of what the original price was) + +Redemption Types +^^^^^^^^^^^^^^^^ +There may be a few different ways we want to track discount usage, for example: + +``one-time``: the discount can only be used once by anyone +``one-time-per-user``: the discount can be used once per user +``unlimited``: the discount can be used any number of times + +Data Models +^^^^^^^^^^^ +.. code-block:: python + + class Discount(TimestampedModel): + """Discount model""" + amount = models.DecimalField( + decimal_places=5, + max_digits=20, + ) + automatic = models.BooleanField(default=False) + discount_type = models.CharField( + choices=DISCOUNT_TYPES, max_length=30 + ) + redemption_type = models.CharField( + choices=REDEMPTION_TYPES, max_length=30 + ) + max_redemptions = models.PositiveIntegerField(null=True) + + class UserDiscount(TimestampedModel): + """pre-assignment for a discount to a user""" + discount = models.ForeignKey(Discount) + user = models.ForeignKey(User) + +Implementation Proposal +^^^^^^^^^^^^^^^^^^^^^^^ + +Rather than codifying the discount logic in a complicated computation function, discount types can be implemented by abstracting the logic around discounts into a registry-driven discount factory like this: + +.. code-block:: python + + import abc + from dataclasses import dataclass + + + @dataclass + class DiscountType(abc.ABC): + _CLASSES = {} + + discount: Discount + + # see https://www.python.org/dev/peps/pep-0487/ + def __init_subclass__(cls, *, discount_type, **kwargs): + super().__init_subclass__(**kwargs) + + if discount_type in _CLASSES: + raise TypeError(f"{discount_type} already defined for DiscountType") + + cls.discount_type = discount_type + cls._CLASSES[discount_type] = cls + + @classmethod + def for_discount(cls, discount: Discount): + DiscountTypeCls = cls._CLASSES[discount.discount_type] + + return DiscountTypeCls(discount) + + def get_product_price(self, product: Product): + return self.get_product_version_price(product.latest_version) + + @abc.abstractmethod + def get_product_version_price(self, product_version: ProductVersion): + ... + + class PercentDiscount(DiscountType, discount_type=Discount.PERCENT_DISCOUNT): + + def get_product_version_price(self, product_version: ProductVersion): + return product_version.price * self.discount.amount + + class FixedPriceDiscount(DiscountType, discount_type=Discount.PERCENT_DISCOUNT): + + def get_product_version_price(self, product_version: ProductVersion): + return self.discount.amount # the amount here is the fixed price + +With this implementation, prices before ordering would use get_product_price, whereas the receipt service would use get_product_version_price on the purchased versions. This makes it far more scalable to introduce new discount types without having to refactor existing code. diff --git a/docs/source/ecommerce/subsystems/index.rst b/docs/source/ecommerce/subsystems/index.rst new file mode 100644 index 0000000000..fd4f0d3a3f --- /dev/null +++ b/docs/source/ecommerce/subsystems/index.rst @@ -0,0 +1,12 @@ +Subsystems +========== + +.. toctree:: + :maxdepth: 2 + + product + basket + order + discount + payment + reporting diff --git a/docs/source/ecommerce/subsystems/order.rst b/docs/source/ecommerce/subsystems/order.rst new file mode 100644 index 0000000000..0e8690df64 --- /dev/null +++ b/docs/source/ecommerce/subsystems/order.rst @@ -0,0 +1,30 @@ +Order Subsystem +=============== + +Orders represent a payment for some kind of product(s), these products will typically be either Programs or Course Runs. An order is marked as unfulfilled initially and then marked as fulfilled once a payment is completed. An order can fail or be refunded. + +Data Model +^^^^^^^^^^ +.. code-block:: python + + class Order(TimestampedModel): + """An order containing information for a purchase.""" + status = models.CharField() + purchaser = models.ForeignKey(settings.AUTH_USER_MODEL) + total_price_paid = models.DecimalField() + + class Line(TimestampedModel): + """A line in an Order.""" + + order = models.ForeignKey(Order) + product_version = models.ForeignKey(ProductVersion) + quantity = models.PositiveIntegerField() + + class Transaction(TimestampedModel): + """A transaction on an order, generally a payment but can also cover refunds""" + order = models.ForeignKey(Order) + amount = models.DecimalField( + decimal_places=5, + max_digits=20, + ) + data = models.JSONField() diff --git a/docs/source/ecommerce/subsystems/payment.rst b/docs/source/ecommerce/subsystems/payment.rst new file mode 100644 index 0000000000..ef6b97a9fc --- /dev/null +++ b/docs/source/ecommerce/subsystems/payment.rst @@ -0,0 +1,4 @@ +Payment Subsystem +================= + +The payment subsystem takes unfulfilled orders, takes the user through payment completion, and finally marks the order as fulfilled. We historically and for the foreseeable future use CyberSource, but this should be strongly decoupled from the rest of the ecommerce system and made pluggable for future flexibility. This system would also be responsible for any webhooks/callbacks that the payment processor makes to us. diff --git a/docs/source/ecommerce/subsystems/product.rst b/docs/source/ecommerce/subsystems/product.rst new file mode 100644 index 0000000000..5f4b7d157b --- /dev/null +++ b/docs/source/ecommerce/subsystems/product.rst @@ -0,0 +1,28 @@ +Product Subsystem +================= + +The product subsystem is responsible for tracking all product-related data. Purchasable products are typically Programs and Course Runs. Pricing information is tracked as immutable data for the sake of historically accurate pricing for orders. + +Data Models +^^^^^^^^^^^ + +.. code-block:: python + + @reversion.register(exclude=("content_type", "object_id", "created_on", "updated_on")) + class Product(TimestampedModel): + """ + Representation of a purchasable product. There is a GenericForeignKey to a Course or Program. + """ + content_type = models.ForeignKey(ContentType) + object_id = models.PositiveIntegerField() + content_object = GenericForeignKey("content_type", "object_id") + +This will utilize `django-reversion` to version product data. + +APIs +^^^^ + +The API that would be primarily needed would be one to read back product data. It is presumed that data entry is done through django-admin: + +- ``GET /api/v0/products/`` -> returns a paginated list of products +- ``GET /api/v0/products/1/`` -> returns a single product diff --git a/docs/source/ecommerce/subsystems/reporting.rst b/docs/source/ecommerce/subsystems/reporting.rst new file mode 100644 index 0000000000..9aa05dc621 --- /dev/null +++ b/docs/source/ecommerce/subsystems/reporting.rst @@ -0,0 +1,17 @@ +Reporting Subsystem +=================== + +OL marketing and finance team need to include financial data in their reporting, so they can evaluate and plan campaigns, reconcile accounts, and payout royalties. As a guiding principle, we want to be able to report on any data that is collected by the ecommerce system, but avoid supplementing the data with external considerations (such as marketing costs, or payout rates). + +Use Cases +********* + +MicroMasters +^^^^^^^^^^^^ + +Most of MicroMasters programs and courses function the same as any other MITx course, they wouldn’t otherwise be treated any differently except for marketing purposes, which are out of scope for this document if not MITx Online altogether. + +DEDP +^^^^ + +The DEDP program as it is implemented currently is a special case when it comes to ecommerce - DEDP currently supports financial aid. This essentially amounts to a discount to a lower fixed rate. There are a few tiers of financial aid discounts available, but this is easily handled with a couple of preconfigured discounts. We’d need a UI somewhere that enables staff to assign a discount to a particular learner. That discount would then automatically apply to courses within that program, the same as any other program-scoped discount. diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000000..4919248235 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,21 @@ +.. MITx Online Documentation documentation master file, created by + sphinx-quickstart on Tue Dec 7 10:41:39 2021. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to MITx Online Documentation's documentation! +===================================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + authentication + configuration/index + ecommerce/index + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`search` diff --git a/pants b/pants new file mode 100755 index 0000000000..f5b2b80e68 --- /dev/null +++ b/pants @@ -0,0 +1,372 @@ +#!/usr/bin/env bash +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +# =============================== NOTE =============================== +# This ./pants bootstrap script comes from the pantsbuild/setup +# project. It is intended to be checked into your code repository so +# that other developers have the same setup. +# +# Learn more here: https://www.pantsbuild.org/docs/installation +# ==================================================================== + +set -eou pipefail + +# NOTE: To use an unreleased version of Pants from the pantsbuild/pants main branch, +# locate the main branch SHA, set PANTS_SHA= in the environment, and run this script as usual. +# +# E.g., PANTS_SHA=725fdaf504237190f6787dda3d72c39010a4c574 ./pants --version + +PYTHON_BIN_NAME="${PYTHON:-unspecified}" + +# Set this to specify a non-standard location for this script to read the Pants version from. +# NB: This will *not* cause Pants itself to use this location as a config file. +# You can use PANTS_CONFIG_FILES or --pants-config-files to do so. +PANTS_TOML=${PANTS_TOML:-pants.toml} + +PANTS_BIN_NAME="${PANTS_BIN_NAME:-$0}" + +PANTS_SETUP_CACHE="${PANTS_SETUP_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/pants/setup}" +# If given a relative path, we fix it to be absolute. +if [[ "$PANTS_SETUP_CACHE" != /* ]]; then + PANTS_SETUP_CACHE="${PWD}/${PANTS_SETUP_CACHE}" +fi + +PANTS_BOOTSTRAP="${PANTS_SETUP_CACHE}/bootstrap-$(uname -s)-$(uname -m)" + +_PEX_VERSION=2.1.42 +_PEX_URL="https://github.com/pantsbuild/pex/releases/download/v${_PEX_VERSION}/pex" +_PEX_EXPECTED_SHA256="69d6b1b1009b00dd14a3a9f19b72cff818a713ca44b3186c9b12074b2a31e51f" + +VIRTUALENV_VERSION=20.4.7 +VIRTUALENV_REQUIREMENTS=$( +cat << EOF +virtualenv==${VIRTUALENV_VERSION} --hash sha256:2b0126166ea7c9c3661f5b8e06773d28f83322de7a3ff7d06f0aed18c9de6a76 +filelock==3.0.12 --hash sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836 +six==1.16.0 --hash sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 +distlib==0.3.2 --hash sha256:23e223426b28491b1ced97dc3bbe183027419dfc7982b4fa2f05d5f3ff10711c +appdirs==1.4.4 --hash sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128 +importlib-resources==5.1.4; python_version < "3.7" --hash sha256:e962bff7440364183203d179d7ae9ad90cb1f2b74dcb84300e88ecc42dca3351 +importlib-metadata==4.5.0; python_version < "3.8" --hash sha256:833b26fb89d5de469b24a390e9df088d4e52e4ba33b01dc5e0e4f41b81a16c00 +zipp==3.4.1; python_version < "3.10" --hash sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098 +typing-extensions==3.10.0.0; python_version < "3.8" --hash sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84 +EOF +) + +COLOR_RED="\x1b[31m" +COLOR_GREEN="\x1b[32m" +COLOR_YELLOW="\x1b[33m" +COLOR_RESET="\x1b[0m" + +function log() { + echo -e "$@" 1>&2 +} + +function die() { + (($# > 0)) && log "${COLOR_RED}$*${COLOR_RESET}" + exit 1 +} + +function green() { + (($# > 0)) && log "${COLOR_GREEN}$*${COLOR_RESET}" +} + +function warn() { + (($# > 0)) && log "${COLOR_YELLOW}$*${COLOR_RESET}" +} + +function tempdir { + mkdir -p "$1" + mktemp -d "$1"/pants.XXXXXX +} + +function get_exe_path_or_die { + local exe="$1" + if ! command -v "${exe}"; then + die "Could not find ${exe}. Please ensure ${exe} is on your PATH." + fi +} + +function get_pants_config_value { + local config_key="$1" + local optional_space="[[:space:]]*" + local prefix="^${config_key}${optional_space}=${optional_space}" + local raw_value + raw_value="$(sed -ne "/${prefix}/ s#${prefix}##p" "${PANTS_TOML}")" + echo "${raw_value}" | tr -d \"\' && return 0 + return 0 +} + +function get_python_major_minor_version { + local python_exe="$1" + "$python_exe" <&1 > /dev/null)" == "pyenv: python${version}"* ]]; then + continue + fi + if [[ -n "$(check_python_exe_compatible_version "${interpreter_path}")" ]]; then + echo "${interpreter_path}" && return 0 + fi + done +} + +function determine_python_exe { + local pants_version="$1" + set_supported_python_versions "${pants_version}" + local requirement_str="For \`pants_version = \"${pants_version}\"\`, Pants requires Python ${supported_message} to run." + + local python_exe + if [[ "${PYTHON_BIN_NAME}" != 'unspecified' ]]; then + python_exe="$(get_exe_path_or_die "${PYTHON_BIN_NAME}")" || exit 1 + if [[ -z "$(check_python_exe_compatible_version "${python_exe}")" ]]; then + die "Invalid Python interpreter version for ${python_exe}. ${requirement_str}" + fi + else + python_exe="$(determine_default_python_exe)" + if [[ -z "${python_exe}" ]]; then + die "No valid Python interpreter found. ${requirement_str} Please check that a valid interpreter is installed and on your \$PATH." + fi + fi + echo "${python_exe}" +} + +function compute_sha256 { + local python="$1" + local path="$2" + + "$python" <&2 || exit 1 + fi + echo "${bootstrapped}" +} + +function scrub_PEX_env_vars { + # Ensure the virtualenv PEX runs as shrink-wrapped. + # See: https://github.com/pantsbuild/setup/issues/105 + if [[ -n "${!PEX_@}" ]]; then + warn "Scrubbing ${!PEX_@}" + unset "${!PEX_@}" + fi +} + +function bootstrap_virtualenv { + local python="$1" + local bootstrapped="${PANTS_BOOTSTRAP}/virtualenv-${VIRTUALENV_VERSION}/virtualenv.pex" + if [[ ! -f "${bootstrapped}" ]]; then + ( + green "Creating the virtualenv PEX." + pex_path="$(bootstrap_pex "${python}")" || exit 1 + mkdir -p "${PANTS_BOOTSTRAP}" + local staging_dir + staging_dir=$(tempdir "${PANTS_BOOTSTRAP}") + cd "${staging_dir}" + echo "${VIRTUALENV_REQUIREMENTS}" > requirements.txt + ( + scrub_PEX_env_vars + "${python}" "${pex_path}" -r requirements.txt -c virtualenv -o virtualenv.pex + ) + mkdir -p "$(dirname "${bootstrapped}")" + mv -f "${staging_dir}/virtualenv.pex" "${bootstrapped}" + rm -rf "${staging_dir}" + ) 1>&2 || exit 1 + fi + echo "${bootstrapped}" +} + +function find_links_url { + local pants_version="$1" + local pants_sha="$2" + echo -n "https://binaries.pantsbuild.org/wheels/pantsbuild.pants/${pants_sha}/${pants_version/+/%2B}/index.html" +} + +function get_version_for_sha { + local sha="$1" + + # Retrieve the Pants version associated with this commit. + local pants_version + pants_version="$(curl --fail -sL "https://raw.githubusercontent.com/pantsbuild/pants/${sha}/src/python/pants/VERSION")" + + # Construct the version as the release version from src/python/pants/VERSION, plus the string `+gitXXXXXXXX`, + # where the XXXXXXXX is the first 8 characters of the SHA. + echo "${pants_version}+git${sha:0:8}" +} + +function bootstrap_pants { + local pants_version="$1" + local python="$2" + local pants_sha="${3:-}" + + local pants_requirement="pantsbuild.pants==${pants_version}" + local maybe_find_links + if [[ -z "${pants_sha}" ]]; then + maybe_find_links="" + else + maybe_find_links="--find-links=$(find_links_url "${pants_version}" "${pants_sha}")" + fi + local python_major_minor_version + python_major_minor_version="$(get_python_major_minor_version "${python}")" + local target_folder_name="${pants_version}_py${python_major_minor_version}" + local bootstrapped="${PANTS_BOOTSTRAP}/${target_folder_name}" + + if [[ ! -d "${bootstrapped}" ]]; then + ( + green "Bootstrapping Pants using ${python}" + local staging_dir + staging_dir=$(tempdir "${PANTS_BOOTSTRAP}") + local virtualenv_path + virtualenv_path="$(bootstrap_virtualenv "${python}")" || exit 1 + green "Installing ${pants_requirement} into a virtual environment at ${bootstrapped}" + ( + scrub_PEX_env_vars + # shellcheck disable=SC2086 + "${python}" "${virtualenv_path}" --no-download "${staging_dir}/install" && \ + "${staging_dir}/install/bin/pip" install -U pip && \ + "${staging_dir}/install/bin/pip" install ${maybe_find_links} --progress-bar off "${pants_requirement}" + ) && \ + ln -s "${staging_dir}/install" "${staging_dir}/${target_folder_name}" && \ + mv "${staging_dir}/${target_folder_name}" "${bootstrapped}" && \ + green "New virtual environment successfully created at ${bootstrapped}." + ) 1>&2 || exit 1 + fi + echo "${bootstrapped}" +} + +# Ensure we operate from the context of the ./pants buildroot. +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +pants_version="$(determine_pants_version)" +python="$(determine_python_exe "${pants_version}")" +pants_dir="$(bootstrap_pants "${pants_version}" "${python}" "${PANTS_SHA:-}")" || exit 1 + +pants_python="${pants_dir}/bin/python" +pants_binary="${pants_dir}/bin/pants" +pants_extra_args="" +if [[ -n "${PANTS_SHA:-}" ]]; then + pants_extra_args="${pants_extra_args} --python-repos-repos=$(find_links_url "$pants_version" "$PANTS_SHA")" +fi + +# shellcheck disable=SC2086 +exec "${pants_python}" "${pants_binary}" ${pants_extra_args} \ + --pants-bin-name="${PANTS_BIN_NAME}" --pants-version=${pants_version} "$@" diff --git a/pants-plugins/mitol/__init__.py b/pants-plugins/mitol/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pants-plugins/mitol/docs/register.py b/pants-plugins/mitol/docs/register.py new file mode 100644 index 0000000000..7d3b060023 --- /dev/null +++ b/pants-plugins/mitol/docs/register.py @@ -0,0 +1,92 @@ +from pants.backend.python.target_types import ConsoleScript +from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.pex import ( + Pex, + PexProcess, + PexRequest, + PexRequirements, +) +from pants.core.util_rules.distdir import DistDir +from pants.engine.fs import Digest, Workspace, MergeDigests, PathGlobs +from pants.engine.process import FallibleProcessResult +from pants.engine.rules import Get, MultiGet, goal_rule, collect_rules, rule +from pants.engine.goal import Goal, GoalSubsystem +from pants.engine.target import Target, Tags, COMMON_TARGET_FIELDS, Targets, Sources, StringField, FieldSet +from dataclasses import dataclass +import logging + +logger = logging.getLogger(__name__) + +class DocsGoalSubsystem(GoalSubsystem): + name = "docs" + help = "Generate documentation" + + +class Docs(Goal): + subsystem_cls = DocsGoalSubsystem + + +class SphinxSources(StringField): + alias = "source_directory" + + +@dataclass(frozen=True) +class SphinxDocsFieldSet(FieldSet): + required_fields = (SphinxSources,) + source_directory: SphinxSources + +class SphinxDocs(Target): + alias = "sphinx_docs" + core_fields = (*COMMON_TARGET_FIELDS, SphinxSources) + help = "Define a sphinx docs source" + + +@goal_rule +async def build_docs(targets: Targets, dist_dir: DistDir, workspace: Workspace) -> Docs: + + pex = await Get( + Pex, + PexRequest( + output_filename="sphinx-build.pex", + internal_only=True, + requirements=PexRequirements(["sphinx", "insipid-sphinx-theme", "sphinxcontrib-mermaid"]), + interpreter_constraints=InterpreterConstraints([">=3.6"]), + main=ConsoleScript("sphinx-build"), + ) + ) + digests = [] + + for target in targets: + if not SphinxDocsFieldSet.is_applicable(target): + continue + source_dir = f"{target.address.spec_path}/{target.get(SphinxSources).value}" + output_path = "sphinx" + + digest = await Get(Digest, PathGlobs([f"{source_dir}/**/*"])) + result = await Get( + FallibleProcessResult, + PexProcess( + pex, + argv=[source_dir, output_path], + description="Generate sphinx docs", + input_digest=digest, + output_directories=[output_path] + ) + ) + digests.append(result.output_digest) + logger.info(result.stdout.decode()) + logger.info(result.stderr.decode()) + + merged_digest = await Get( + Digest, + MergeDigests(digests) + ) + + workspace.write_digest(merged_digest, path_prefix=str(dist_dir.relpath)) + return Docs(exit_code=0) + +def target_types(): + return [SphinxDocs] + +def rules(): + return collect_rules() diff --git a/pants.toml b/pants.toml new file mode 100644 index 0000000000..564a171981 --- /dev/null +++ b/pants.toml @@ -0,0 +1,12 @@ +[GLOBAL] +backend_packages = [ + "pants.backend.python", + "pants.backend.python.lint.black", + # local plugins + "mitol.docs" +] +pants_version = "2.8.1rc1" +pythonpath = ["%(buildroot)s/pants-plugins"] + +[anonymous-telemetry] +enabled = false