diff --git a/backend-services/.dockerignore b/backend-services/.dockerignore index 207875a..f275cfa 100644 --- a/backend-services/.dockerignore +++ b/backend-services/.dockerignore @@ -19,6 +19,4 @@ !tests/__init__.py !tests/data_service/__init__.py !tests/data_service/operations/__init__.py -!tests/data_service/operations/helpers.py -!tests/data_service/operations/test_bookmark.py !tox.ini diff --git a/backend-services/README.md b/backend-services/README.md index 58250e8..e5fbd22 100644 --- a/backend-services/README.md +++ b/backend-services/README.md @@ -1,4 +1,4 @@ -# Wallet Backend Services +# Trusted Compute Data Services An HTTP server backend for the Nautilus Wallet powered by [FastAPI][fastapi] and the [uvicorn ASGI server][uvicorn]. @@ -51,7 +51,7 @@ eval "$(pyenv init -)" ``` Make sure to log out (and back in of course) or, alternatively, restart your -machine. Once you are back in the `backend-services` sub-directory of the +machine. Once you are back in the `backend-services` subdirectory of the project run ```shell @@ -71,11 +71,11 @@ poetry install Make sure the following environment variables have been set in your local `.env` file: -- `WALLET_DB_CONNECTION_STRING` -- `WALLET_DB_DATABASE_NAME` -- `WALLET_BOOKMARK_DB_COLLECTION_NAME` +- `PRIMARY_ORIGIN` +- `VAULT_DB_CONNECTION_STRING` +- `VAULT_DB_NAME` -For examples you may consult the [python-dotenv] documentation. Once this is done, a local instance of the server may be started on `localhost:8000` by running +For examples, you may consult the [python-dotenv] documentation. Once this is done, a local instance of the server may be started on `localhost:8000` by running ```shell poetry run uvicorn web_asgi.main:app diff --git a/backend-services/src/data_service/operations/datapool.py b/backend-services/src/data_service/operations/datapool.py index 9b2280c..03a73b4 100644 --- a/backend-services/src/data_service/operations/datapool.py +++ b/backend-services/src/data_service/operations/datapool.py @@ -2,12 +2,12 @@ from odmantic import ObjectId from common.types import WalletAddress -from data_service.schema.actions import CreateDatapool, DeleteDatapool +from data_service.schema.actions import CreateDatapool, UpdateDataPool, DeleteDatapool from data_service.schema.entities import Datapool, DatapoolList from data_service.schema.types import Engine -async def create_datapool(engine: Engine, params: CreateDatapool) -> Datapool: +async def create_datapool(engine: Engine, params: CreateDatapool) -> None: """ Create a new datapool. """ @@ -15,6 +15,7 @@ async def create_datapool(engine: Engine, params: CreateDatapool) -> Datapool: creator_wallet_id=params.creator_wallet_id, name=params.name, description=params.description, + datapool_schema=params.datapool_schema, datapool_hash=params.datapool_hash, smart_contract_id=params.smart_contract_id, smart_contract_address=params.smart_contract_address, @@ -23,7 +24,27 @@ async def create_datapool(engine: Engine, params: CreateDatapool) -> Datapool: created=params.created, ) await engine.save(new_datapool) - return new_datapool + + +async def update_datapool(engine: Engine, params: UpdateDataPool) -> None: + """ + Update existing datapool + """ + existing_datapool = await engine.find_one( + Datapool, + (UpdateDataPool.application_id == params.application_id), + ) + + if existing_datapool is None: + raise HTTPException(404) + + if existing_datapool: + existing_datapool.sealed_data = (params.sealed_data,) + existing_datapool.contribution_token_id = (params.contribution_token_id,) + existing_datapool.ref_contributors = (params.ref_contributors,) + await engine.save(existing_datapool) + else: + pass async def delete_datapool(engine: Engine, params: DeleteDatapool) -> None: diff --git a/backend-services/src/data_service/operations/dataschema.py b/backend-services/src/data_service/operations/dataschema.py deleted file mode 100644 index b180966..0000000 --- a/backend-services/src/data_service/operations/dataschema.py +++ /dev/null @@ -1,38 +0,0 @@ -from fastapi import HTTPException -from odmantic import ObjectId - -from data_service.schema.actions import CreateDataschema, DeleteDataschema -from data_service.schema.entities import Dataschema, DataschemaList -from data_service.schema.types import Engine - - -async def create_dataschema(engine: Engine, params: CreateDataschema) -> Dataschema: - """ - Create a new dataschema. - """ - new_dataschema = Dataschema( - name=params.name, data_schema=params.data_schema, created=params.created - ) - await engine.save(new_dataschema) - return new_dataschema - - -async def delete_dataschema(engine: Engine, params: DeleteDataschema) -> None: - """ - Delete a specified dataschema. - """ - # XXX: assumes `params.id` is a 24 character hex string - id_to_delete = ObjectId(params.delete_id) - existing_dataschema = await engine.find_one( - Dataschema, Dataschema.id == id_to_delete - ) - if existing_dataschema is None: - raise HTTPException(404) - await engine.delete(existing_dataschema) - - -async def find_by_pool(engine: Engine, data_schema_id: str) -> DataschemaList: - """ - Retrieve a list of all dataschemas for a given user from the database. - """ - return await engine.find(Dataschema, Dataschema.data_schema_id == data_schema_id) diff --git a/backend-services/src/data_service/operations/dataset.py b/backend-services/src/data_service/operations/dataset.py deleted file mode 100644 index eadf7fd..0000000 --- a/backend-services/src/data_service/operations/dataset.py +++ /dev/null @@ -1,51 +0,0 @@ -from fastapi import HTTPException -from odmantic import ObjectId - -from common.types import WalletAddress -from data_service.schema.actions import CreateDataset, DeleteDataset -from data_service.schema.entities import Dataset, DatasetList -from data_service.schema.types import Engine - - -async def create_dataset(engine: Engine, params: CreateDataset) -> Dataset: - """ - Create a new dataset. - """ - new_dataset = Dataset( - wallet_id=params.wallet_id, - data_pool_id=params.data_pool_id, - data_schema_id=params.data_schema_id, - name=params.name, - description=params.description, - num_of_rows=params.num_of_rows, - data_pool_position=params.data_pool_position, - created=params.created, - ) - await engine.save(new_dataset) - return new_dataset - - -async def delete_dataset(engine: Engine, params: DeleteDataset) -> None: - """ - Delete a specified dataset. - """ - # XXX: assumes `params.id` is a 24 character hex string - id_to_delete = ObjectId(params.delete_id) - existing_dataset = await engine.find_one(Dataset, Dataset.id == id_to_delete) - if existing_dataset is None: - raise HTTPException(404) - await engine.delete(existing_dataset) - - -async def datasets(engine: Engine, wallet_id: WalletAddress) -> DatasetList: - """ - Retrieve a list of all datasets for a given user from the database. - """ - return await engine.find(Dataset, Dataset.wallet_id == wallet_id) - - -async def find_by_pool(engine: Engine, data_pool_id: str) -> DatasetList: - """ - Retrieve a list of all datasets for a given user from the database. - """ - return await engine.find(Dataset, Dataset.data_pool_id == data_pool_id) diff --git a/backend-services/src/data_service/operations/wasm.py b/backend-services/src/data_service/operations/wasm.py new file mode 100644 index 0000000..643c73e --- /dev/null +++ b/backend-services/src/data_service/operations/wasm.py @@ -0,0 +1,35 @@ +from fastapi import HTTPException +from odmantic import ObjectId + +from data_service.schema.actions import CreateWasmBinary, DeleteWasmBinary +from data_service.schema.entities import WasmBinary, WasmBinaryList +from data_service.schema.types import Engine + + +async def create_wasm_binary(engine: Engine, params: CreateWasmBinary) -> None: + """ + Store a new WASM Binary. + """ + new_wasm_binary = WasmBinary(name=params.name, wasm_binary=params.wasm_binary) + await engine.save(new_wasm_binary) + + +async def get_wasm_binary(engine: Engine, name: str) -> WasmBinaryList: + """ + Retrieve a WASM BInary. + """ + return await engine.find(WasmBinary, WasmBinary.name == name) + + +async def delete_wasm_binary(engine: Engine, params: DeleteWasmBinary) -> None: + """ + Delete a specified WASM Binary. + """ + # XXX: assumes `params.id` is a 24 character hex string + id_to_delete = ObjectId(params.delete_id) + existing_wasm_binary = await engine.find_one( + WasmBinary, WasmBinary.id == id_to_delete + ) + if existing_wasm_binary is None: + raise HTTPException(404) + await engine.delete(existing_wasm_binary) diff --git a/backend-services/src/data_service/schema/actions.py b/backend-services/src/data_service/schema/actions.py index 46c72fb..50880eb 100644 --- a/backend-services/src/data_service/schema/actions.py +++ b/backend-services/src/data_service/schema/actions.py @@ -1,57 +1,34 @@ -from datetime import datetime - from pydantic import BaseModel, validator from common.types import WalletAddress -class CreateDataset(BaseModel): +class CreateDatapool(BaseModel): """ - Datset creation parameters. + Datapool creation parameters. """ - wallet_id: WalletAddress - data_pool_id: str - data_schema_id: str + application_id: str + creator_wallet_id: WalletAddress name: str description: str - num_of_rows: int - data_pool_position: int - created: datetime - - -class DeleteDataset(BaseModel): - """ - Dataset deletion parameters. - """ - - delete_id: str - - @validator("delete_id") - @classmethod - def valid_object_id_hex_representation(cls: type, v: str) -> str: - int(v, 16) - if len(v) != 24: - raise AssertionError( - f"expected a 24 character hexadecimal string but '{v}' has length {len(v)}" - ) - return v + datapool_schema: str + sealed_data: str + ref_drt_id: list[str] + contribution_token_id: str + append_token_id: str + ref_contributors: list[str] -class CreateDatapool(BaseModel): +class UpdateDataPool(BaseModel): """ - Datpool creation parameters. + Datapool update parameters """ - creator_wallet_id: WalletAddress - name: str - description: str - datapool_hash: str - smart_contract_id: str - smart_contract_address: str + application_id: str sealed_data: str - total_rows: int - created: datetime + contribution_token_id: str + ref_contributors: str class DeleteDatapool(BaseModel): @@ -72,21 +49,12 @@ def valid_object_id_hex_representation(cls: type, v: str) -> str: return v -class CreateDataschema(BaseModel): - """ - Schema creation parameters. - """ - +class CreateWasmBinary(BaseModel): name: str - data_schema: str - created: datetime - + wasm_binary: str -class DeleteDataschema(BaseModel): - """ - Datapool deletion parameters. - """ +class DeleteWasmBinary(BaseModel): delete_id: str @validator("delete_id") diff --git a/backend-services/src/data_service/schema/entities.py b/backend-services/src/data_service/schema/entities.py index ddcfdb4..af7b517 100644 --- a/backend-services/src/data_service/schema/entities.py +++ b/backend-services/src/data_service/schema/entities.py @@ -1,4 +1,3 @@ -from datetime import datetime from typing import TypeAlias from odmantic import Model @@ -6,51 +5,41 @@ from common.types import WalletAddress -class Dataset(Model): +class Datapool(Model): """ An address on the ledger dataseted by the user. """ - wallet_id: WalletAddress - data_pool_id: str - data_schema_id: str + application_id: str + creator_wallet_id: WalletAddress name: str description: str - num_of_rows: int - data_pool_position: int - created: datetime + datapool_schema: str + sealed_data: str + ref_drt_id: list[str] + contribution_token_id: str + append_token_id: str + ref_contributors: list[str] -DatasetList: TypeAlias = list[Dataset] +DatapoolList: TypeAlias = list[Datapool] -class Datapool(Model): - """ - An address on the ledger dataseted by the user. - """ - - creator_wallet_id: WalletAddress +class Drt(Model): + asset_id: str name: str description: str - datapool_hash: str - smart_contract_id: str - smart_contract_address: str - sealed_data: str - total_rows: int - created: datetime + url_binary: str + price: float + amount_created: int -DatapoolList: TypeAlias = list[Datapool] +DrtList: TypeAlias = list[Drt] -class Dataschema(Model): - """ - A JSON schema for a dataset of datapool. - """ - +class WasmBinary(Model): name: str - data_schema: str - created: datetime + wasm_binary: str -DataschemaList: TypeAlias = list[Dataschema] +WasmBinaryList: TypeAlias = list[WasmBinary] diff --git a/backend-services/src/web_asgi/main.py b/backend-services/src/web_asgi/main.py index 38e6a87..b75061d 100644 --- a/backend-services/src/web_asgi/main.py +++ b/backend-services/src/web_asgi/main.py @@ -1,5 +1,5 @@ """ -Entry point for the Nautilus Wallet web server. +Entry point for Trusted Compute web server. """ from fastapi import FastAPI, status @@ -8,23 +8,28 @@ from odmantic import AIOEngine from common.types import WalletAddress -from data_service.operations.datapool import create_datapool, datapools, delete_datapool -from data_service.operations.dataschema import create_dataschema -from data_service.operations.dataset import create_dataset, datasets -from data_service.operations.dataset import delete_dataset as data_delete_dataset +from data_service.operations.datapool import ( + create_datapool, + update_datapool, + datapools, + delete_datapool, +) +from data_service.operations.wasm import ( + create_wasm_binary, + get_wasm_binary, + delete_wasm_binary, +) from data_service.schema.actions import ( CreateDatapool, - CreateDataschema, - CreateDataset, + UpdateDataPool, DeleteDatapool, - DeleteDataset, + CreateWasmBinary, + DeleteWasmBinary, ) from data_service.schema.entities import ( Datapool, DatapoolList, - Dataschema, - Dataset, - DatasetList, + WasmBinaryList, ) from web_asgi.settings import AppSettings @@ -49,27 +54,6 @@ ) -@app.get("/datasets", response_model=DatasetList, status_code=status.HTTP_200_OK) -async def get_datasets(wallet_id: WalletAddress) -> DatasetList: - return await datasets(mongo_engine, wallet_id) - - -@app.post( - "/dataset/create", response_model=Dataset, status_code=status.HTTP_201_CREATED -) -async def post_dataset_create(request: CreateDataset) -> Dataset: - return await create_dataset(mongo_engine, request) - - -@app.delete( - "/dataset", - response_model=None, - status_code=status.HTTP_204_NO_CONTENT, -) -async def delete_dataset(request: DeleteDataset) -> None: - await data_delete_dataset(mongo_engine, request) - - @app.get("/datapools", response_model=DatapoolList, status_code=status.HTTP_200_OK) async def get_datapools(wallet_id: WalletAddress) -> DatapoolList: return await datapools(mongo_engine, wallet_id) @@ -82,6 +66,13 @@ async def post_datapool_create(request: CreateDatapool) -> Datapool: return await create_datapool(mongo_engine, request) +@app.post( + "/datapool/update", response_model=Datapool, status_code=status.HTTP_201_CREATED +) +async def post_datapool_update(request: UpdateDataPool) -> Datapool: + return await update_datapool(mongo_engine, request) + + @app.delete( "/datapool", response_model=None, @@ -91,8 +82,20 @@ async def post_delete_datapool(request: DeleteDatapool) -> None: await delete_datapool(mongo_engine, request) -@app.post( - "/dataschema/create", response_model=Dataschema, status_code=status.HTTP_201_CREATED +@app.get("/wasm", response_model=WasmBinaryList, status_code=status.HTTP_200_OK) +async def get_wasm_binary(name: str) -> WasmBinaryList: + return await datapools(mongo_engine, name) + + +@app.post("/wasm", response_model=None, status_code=status.HTTP_201_CREATED) +async def post_wasm_binary_post(request: CreateWasmBinary) -> None: + return await create_wasm_binary(mongo_engine, request) + + +@app.delete( + "/wasm", + response_model=None, + status_code=status.HTTP_204_NO_CONTENT, ) -async def post_dataschema_create(request: CreateDataschema) -> Dataschema: - return await create_dataschema(mongo_engine, request) +async def post_delete_wasm_binary(request: DeleteWasmBinary) -> None: + await delete_wasm_binary(mongo_engine, request) diff --git a/backend-services/tests/data_service/operations/helpers.py b/backend-services/tests/data_service/operations/helpers.py deleted file mode 100644 index 3e09665..0000000 --- a/backend-services/tests/data_service/operations/helpers.py +++ /dev/null @@ -1,5 +0,0 @@ -from data_service.settings import MongoSettings - -mock_settings = MongoSettings( - vault_db_dataset_collection="dataset", -) diff --git a/backend-services/tests/data_service/operations/test_bookmark.py b/backend-services/tests/data_service/operations/test_bookmark.py deleted file mode 100644 index 8e19634..0000000 --- a/backend-services/tests/data_service/operations/test_bookmark.py +++ /dev/null @@ -1,99 +0,0 @@ -from datetime import datetime -from unittest.mock import AsyncMock - -import pytest -from motor import motor_asyncio -from odmantic import AIOEngine, ObjectId -from pytest_mock import MockerFixture - -from common.types import WalletAddress -from data_service.operations.dataset import create_dataset, delete_dataset -from data_service.schema.actions import CreateDataset, DeleteDataset -from data_service.schema.entities import Dataset - - -@pytest.mark.asyncio -async def test_create_dataset_success(mocker: MockerFixture) -> None: - mocker.patch("motor.motor_asyncio.AsyncIOMotorClient") - test_create_dataset = CreateDataset( - wallet_id=WalletAddress("test_wallet_id"), - data_pool_id="x0x0x0x0x0x0x0x0x0x0x0", - data_schema_id="x0x0x0x0x0x0x0x0x0x0x0", - name="test_name", - description="test description of datasete", - num_of_rows=500, - data_pool_position=0, - created=datetime.now(), - ) - - mock_save = AsyncMock(return_value=test_create_dataset) - mocker.patch.object(AIOEngine, "save", mock_save) - engine = AIOEngine(client=motor_asyncio.AsyncIOMotorClient()) - - returned_dataset = await create_dataset(engine, test_create_dataset) - mock_save.assert_awaited_once_with(returned_dataset) - - -@pytest.mark.asyncio -async def test_delete_dataset_success(mocker: MockerFixture) -> None: - hex_string_id = "a" * 24 - dataset_to_delete = Dataset.parse_obj( - { - "id": ObjectId(hex_string_id), - "wallet_id": "test_wallet_id", - "data_pool_id": "x0x0x0x0x0x0x0x0x0x0x0", - "data_schema_id": "x0x0x0x0x0x0x0x0x0x0x0", - "name": "test_name1", - "description": "test description of dataset", - "num_of_rows": 500, - "data_pool_position": 0, - "created": datetime.now(), - } - ) - mocker.patch("motor.motor_asyncio.AsyncIOMotorClient") - - mock_find_one = AsyncMock(return_value=dataset_to_delete) - mock_delete = AsyncMock(return_value=None) - mocker.patch.object(AIOEngine, "find_one", mock_find_one) - mocker.patch.object(AIOEngine, "delete", mock_delete) - - engine = AIOEngine(client=motor_asyncio.AsyncIOMotorClient()) - - params = DeleteDataset(delete_id=hex_string_id) - - assert await delete_dataset(engine, params) is None - mock_find_one.assert_awaited_once_with( - Dataset, Dataset.id == ObjectId(params.delete_id) - ) - mock_delete.assert_awaited_once_with(dataset_to_delete) - - -@pytest.mark.asyncio -async def test_get_datasets_success(mocker: MockerFixture) -> None: - stored_docs = [ - { - "id": ObjectId(b"a" * 12), - "wallet_id": "test_wallet_id", - "data_pool_id": "x0x0x0x0x0x0x0x0x0x0x0", - "data_schema_id": "x0x0x0x0x0x0x0x0x0x0x0", - "name": "test_name1", - "description": "test description of dataset", - "num_of_rows": 500, - "data_pool_position": 0, - "created": datetime.now(), - }, - { - "id": ObjectId(b"b" * 12), - "wallet_id": "test_wallet_id2", - "data_pool_id": "x0x0x0x0x0x0x0x0x0x0x0", - "data_schema_id": "x0x0x0x0x0x0x0x0x0x0x0", - "name": "test_name2", - "description": "test description of dataset again", - "num_of_rows": 200, - "data_pool_position": 0, - "created": datetime.now(), - }, - ] - mock_find = AsyncMock(return_value=stored_docs) - mocker.patch("motor.motor_asyncio.AsyncIOMotorClient") - mocker.patch.object(AIOEngine, "find", mock_find)