diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ff2a3943..55ceb3751 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ # Changelog +# Version 3.58.1 (2023-12-15) +## Added +* Support to export all projects and all model runs to `export_v2` for a `dataset` and a `slice` +## Notebooks +* Update exports v2 notebook to include methods that return `ExportTask` # Version 3.58.0 (2023-12-11) ## Added diff --git a/CONTRIB.md b/CONTRIB.md index 77122ed79..70c2f79b1 100644 --- a/CONTRIB.md +++ b/CONTRIB.md @@ -12,6 +12,7 @@ review the following considerations before proceeding: improvements to functions within the schema/ package. * Please ensure that any new libraries are compliant with the Apache license that governs the Labelbox SDK. * Ensure that you update any relevant docstrings and comments within your code +* Ensure that any new python components like classes, methods etc that need to feature in labelbox documentation have entries in the file [index.rst](https://github.com/Labelbox/labelbox-python/blob/develop/docs/source/index.rst). Also make sure you run `make html` locally in the `docs` folder to check if the documentation correctly updated according to the docstrings in the code added. ## Repository Organization diff --git a/docs/source/conf.py b/docs/source/conf.py index a10fb6392..31abf7a12 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -21,7 +21,7 @@ copyright = '2021, Labelbox' author = 'Labelbox' -release = '3.58.0' +release = '3.58.1' # -- General configuration --------------------------------------------------- diff --git a/examples/annotation_import/conversational.ipynb b/examples/annotation_import/conversational.ipynb index 5fc7c0472..5dcce6f2a 100644 --- a/examples/annotation_import/conversational.ipynb +++ b/examples/annotation_import/conversational.ipynb @@ -362,9 +362,9 @@ "ontology_builder = lb.OntologyBuilder(\n", " tools=[\n", " lb.Tool(tool=lb.Tool.Type.NER,name=\"ner\"),\n", - " ],\n", - " classifications=[\n", - " lb.Classification(\n", + " ], \n", + " classifications=[ \n", + " lb.Classification( \n", " class_type=lb.Classification.Type.TEXT,\n", " scope=lb.Classification.Scope.INDEX,\n", " name=\"text_convo\"),\n", @@ -587,11 +587,11 @@ { "metadata": {}, "source": [ - "# Upload label for this data row in project\n", + "# Upload label for this data row in project \n", "upload_job = lb.LabelImport.create_from_objects(\n", - " client = client,\n", - " project_id = project.uid,\n", - " name=\"label_import_job\"+str(uuid.uuid4()),\n", + " client = client, \n", + " project_id = project.uid, \n", + " name=\"label_import_job\"+str(uuid.uuid4()), \n", " labels=label)\n", "\n", "upload_job.wait_until_done()\n", diff --git a/examples/basics/export_data.ipynb b/examples/basics/export_data.ipynb index 8e1c19ad8..95cf7f708 100644 --- a/examples/basics/export_data.ipynb +++ b/examples/basics/export_data.ipynb @@ -31,7 +31,7 @@ "metadata": {}, "source": [ "# Export data\n", - "How to export data for projects, datasets, slices, and models, with examples for each type of v2 export along with details on optional parameters and filters." + "How to export data for projects, datasets, slices, data rows and models, with examples for each type of v2 export along with details on optional parameters and filters." ], "cell_type": "markdown" }, @@ -50,7 +50,8 @@ "source": [ "import labelbox as lb\n", "import urllib.request\n", - "from PIL import Image" + "from PIL import Image\n", + "import time" ], "cell_type": "code", "outputs": [], @@ -125,6 +126,13 @@ "outputs": [], "execution_count": null }, + { + "metadata": {}, + "source": [ + "#### Export V2 Method" + ], + "cell_type": "markdown" + }, { "metadata": {}, "source": [ @@ -160,6 +168,100 @@ "outputs": [], "execution_count": null }, + { + "metadata": {}, + "source": [ + "#### Stream Task Export Method\n", + "The return type of this method is an ExportTask, instead of a Task. This is just a wrapper around Task, and most of its features are also present in ExportTask.\n", + "This allows streaming of task results and errors." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "# Set the export params to include/exclude certain fields. \n", + "export_params= {\n", + " \"attachments\": True,\n", + " \"metadata_fields\": True,\n", + " \"data_row_details\": True,\n", + " \"project_details\": True,\n", + " \"label_details\": True,\n", + " \"performance_details\": True,\n", + " \"interpolated_frames\": True\n", + "}\n", + "\n", + "# Note: Filters follow AND logic, so typically using one filter is sufficient.\n", + "filters= {\n", + " \"last_activity_at\": [\"2000-01-01 00:00:00\", \"2050-01-01 00:00:00\"],\n", + " \"label_created_at\": [\"2000-01-01 00:00:00\", \"2050-01-01 00:00:00\"],\n", + " # \"data_row_ids\": [\"\", \"\"],\n", + " # \"batch_ids\": [\"\", \"\"],\n", + "}\n", + "\n", + "client.enable_experimental = True\n", + "\n", + "export_task = project.export(params=export_params, filters=filters)\n", + "export_task.wait_till_done()" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "# Provide results with JSON converter\n", + "# Returns streamed JSON output strings from export task results/errors, one by one\n", + "\n", + "# Callback used for JSON Converter\n", + "def json_stream_handler(output: lb.JsonConverterOutput):\n", + " print(output.json_str)\n", + "\n", + "\n", + "if export_task.has_errors():\n", + " export_task.get_stream(\n", + " \n", + " converter=lb.JsonConverter(),\n", + " stream_type=lb.StreamType.ERRORS\n", + " ).start(stream_handler=lambda error: print(error))\n", + "\n", + "if export_task.has_result():\n", + " export_json = export_task.get_stream(\n", + " converter=lb.JsonConverter(),\n", + " stream_type=lb.StreamType.RESULT\n", + " ).start(stream_handler=json_stream_handler)\n", + "\n", + "print(\"file size: \", export_task.get_total_file_size(stream_type=lb.StreamType.RESULT))\n", + "print(\"line count: \", export_task.get_total_lines(stream_type=lb.StreamType.RESULT))" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "# Uncomment to get stream results as a written file\n", + "\n", + "# Provide results with file converter\n", + "\n", + "# if export_task.has_errors():\n", + "# export_task.get_stream(\n", + "# converter=lb.FileConverter(file_path=\"./errors.txt\"),\n", + "# stream_type=lb.StreamType.ERRORS\n", + "# ).start()\n", + "\n", + "# if export_task.has_result(): \n", + "# export_task.get_stream(\n", + "# converter=lb.FileConverter(file_path=\"./result.txt\"),\n", + "# stream_type=lb.StreamType.RESULT\n", + "# ).start()" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, { "metadata": {}, "source": [ @@ -202,6 +304,13 @@ "outputs": [], "execution_count": null }, + { + "metadata": {}, + "source": [ + "#### Export V2 Method" + ], + "cell_type": "markdown" + }, { "metadata": {}, "source": [ @@ -236,6 +345,98 @@ "outputs": [], "execution_count": null }, + { + "metadata": {}, + "source": [ + "#### Stream Task Export Method\n", + "The return type of this method is an ExportTask, instead of a Task. This is just a wrapper around Task, and most of its features are also present in ExportTask.\n", + "This allows streaming of task results and errors." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "# Set the export params to include/exclude certain fields.\n", + "export_params= {\n", + " \"attachments\": True,\n", + " \"metadata_fields\": True,\n", + " \"data_row_details\": True,\n", + " \"project_details\": True,\n", + " \"label_details\": True,\n", + " \"performance_details\": True,\n", + " \"interpolated_frames\": True,\n", + " # \"project_ids\": [\"\", \"\"],\n", + " # \"model_run_ids\": [\"\", \"\"] \n", + "}\n", + "\n", + "# Note: Filters follow AND logic, so typically using one filter is sufficient.\n", + "filters= {\n", + " \"last_activity_at\": [\"2000-01-01 00:00:00\", \"2050-01-01 00:00:00\"]\n", + "}\n", + "\n", + "client.enable_experimental = True\n", + "\n", + "export_task = dataset.export(params=export_params, filters=filters)\n", + "export_task.wait_till_done()" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "# Provide results with JSON converter\n", + "# Returns streamed JSON output strings from export task results/errors, one by one\n", + "\n", + "# Callback used for JSON Converter\n", + "def json_stream_handler(output: lb.JsonConverterOutput):\n", + " print(output.json_str)\n", + "\n", + "\n", + "if export_task.has_errors():\n", + " export_task.get_stream(\n", + " converter=lb.JsonConverter(),\n", + " stream_type=lb.StreamType.ERRORS\n", + " ).start(stream_handler=lambda error: print(error))\n", + "\n", + "if export_task.has_result():\n", + " export_json = export_task.get_stream(\n", + " converter=lb.JsonConverter(),\n", + " stream_type=lb.StreamType.RESULT\n", + " ).start(stream_handler=json_stream_handler)\n", + "\n", + "print(\"file size: \", export_task.get_total_file_size(stream_type=lb.StreamType.RESULT))\n", + "print(\"line count: \", export_task.get_total_lines(stream_type=lb.StreamType.RESULT))" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "# Uncomment to get stream results as a written file\n", + "\n", + "# Provide results with file converter\n", + "\n", + "# if export_task.has_errors():\n", + "# export_task.get_stream(\n", + "# converter=lb.FileConverter(file_path=\"./errors.txt\"),\n", + "# stream_type=lb.StreamType.ERRORS\n", + "# ).start()\n", + "\n", + "# if export_task.has_result(): \n", + "# export_task.get_stream(\n", + "# converter=lb.FileConverter(file_path=\"./result.txt\"),\n", + "# stream_type=lb.StreamType.RESULT\n", + "# ).start()" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, { "metadata": {}, "source": [ @@ -261,6 +462,13 @@ "outputs": [], "execution_count": null }, + { + "metadata": {}, + "source": [ + "#### Export V2 Method" + ], + "cell_type": "markdown" + }, { "metadata": {}, "source": [ @@ -290,6 +498,93 @@ "outputs": [], "execution_count": null }, + { + "metadata": {}, + "source": [ + "#### Stream Task Export Method\n", + "The return type of this method is an ExportTask, instead of a Task. This is just a wrapper around Task, and most of its features are also present in ExportTask.\n", + "This allows streaming of task results and errors." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "# Set the export params to include/exclude certain fields.\n", + "export_params = {\n", + " \"attachments\": True,\n", + " \"metadata_fields\": True,\n", + " \"data_row_details\": True,\n", + " \"project_details\": True,\n", + " \"label_details\": True,\n", + " \"performance_details\": True,\n", + " \"interpolated_frames\": True,\n", + " # \"project_ids\": [\"\", \"\"],\n", + " # \"model_run_ids\": [\"\", \"\"]\n", + "}\n", + "\n", + "\n", + "client.enable_experimental = True\n", + "\n", + "export_task = catalog_slice.export(params=export_params)\n", + "export_task.wait_till_done()" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "# Provide results with JSON converter\n", + "# Returns streamed JSON output strings from export task results/errors, one by one\n", + "\n", + "# Callback used for JSON Converter\n", + "def json_stream_handler(output: lb.JsonConverterOutput):\n", + " print(output.json_str)\n", + "\n", + "if export_task.has_errors():\n", + " export_task.get_stream(\n", + " converter=lb.JsonConverter(),\n", + " stream_type=lb.StreamType.ERRORS\n", + " ).start(stream_handler=lambda error: print(error))\n", + "\n", + "if export_task.has_result():\n", + " export_json = export_task.get_stream(\n", + " converter=lb.JsonConverter(),\n", + " stream_type=lb.StreamType.RESULT\n", + " ).start(stream_handler=json_stream_handler)\n", + "\n", + "print(\"file size: \", export_task.get_total_file_size(stream_type=lb.StreamType.RESULT))\n", + "print(\"line count: \", export_task.get_total_lines(stream_type=lb.StreamType.RESULT))" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "# Uncomment to get stream results as a written file\n", + "\n", + "# Provide results with file converter\n", + "\n", + "# if export_task.has_errors():\n", + "# export_task.get_stream(\n", + "# converter=lb.FileConverter(file_path=\"./errors.txt\"),\n", + "# stream_type=lb.StreamType.ERRORS\n", + "# ).start()\n", + "\n", + "# if export_task.has_result(): \n", + "# export_task.get_stream(\n", + "# converter=lb.FileConverter(file_path=\"./result.txt\"),\n", + "# stream_type=lb.StreamType.RESULT\n", + "# ).start()" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, { "metadata": {}, "source": [ @@ -321,6 +616,13 @@ "outputs": [], "execution_count": null }, + { + "metadata": {}, + "source": [ + "#### Export V2 Method" + ], + "cell_type": "markdown" + }, { "metadata": {}, "source": [ @@ -346,6 +648,208 @@ "outputs": [], "execution_count": null }, + { + "metadata": {}, + "source": [ + "#### Stream Task Export Method\n", + "The return type of this method is an ExportTask, instead of a Task. This is just a wrapper around Task, and most of its features are also present in ExportTask.\n", + "This allows streaming of task results and errors." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "# Set the export params to include/exclude certain fields.\n", + "export_params = {\n", + " \"attachments\": True,\n", + " \"metadata_fields\": True,\n", + " \"data_row_details\": True,\n", + " \"interpolated_frames\": True,\n", + " \"predictions\": True\n", + "}\n", + "\n", + "client.enable_experimental = True\n", + "\n", + "export_task = model_run.export(params=export_params)\n", + "export_task.wait_till_done()" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "# Provide results with JSON converter\n", + "# Returns streamed JSON output strings from export task results/errors, one by one\n", + "\n", + "# Callback used for JSON Converter\n", + "def json_stream_handler(output: lb.JsonConverterOutput):\n", + " print(output.json_str)\n", + "\n", + "if export_task.has_errors():\n", + " export_task.get_stream(\n", + " converter=lb.JsonConverter(),\n", + " stream_type=lb.StreamType.ERRORS\n", + " ).start(stream_handler=lambda error: print(error))\n", + "\n", + "if export_task.has_result():\n", + " export_json = export_task.get_stream(\n", + " converter=lb.JsonConverter(),\n", + " stream_type=lb.StreamType.RESULT\n", + " ).start(stream_handler=json_stream_handler)\n", + "\n", + "print(\"file size: \", export_task.get_total_file_size(stream_type=lb.StreamType.RESULT))\n", + "print(\"line count: \", export_task.get_total_lines(stream_type=lb.StreamType.RESULT))" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "# Uncomment to get stream results as a written file\n", + "\n", + "# Provide results with file converter\n", + "# if export_task.has_errors():\n", + "# export_task.get_stream(\n", + "# converter=lb.FileConverter(file_path=\"./errors.txt\"),\n", + "# stream_type=lb.StreamType.ERRORS\n", + "# ).start()\n", + "\n", + "# if export_task.has_result(): \n", + "# export_task.get_stream(\n", + "# converter=lb.FileConverter(file_path=\"./result.txt\"),\n", + "# stream_type=lb.StreamType.RESULT\n", + "# ).start()" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "## Export Data Row\n", + "For complete details on the supported filters and parameters, including how they are used and what information is included, please see the [Export overview](https://docs.labelbox.com/reference/label-export#optional-parameters-and-filters) developer guide.\n", + "\n", + "### Parameters\n", + "When exporting data rows, you can apply the same parameters as exporting from a project.\n", + "\n", + "### Filters\n", + "No filters are applicable to export data rows. All the data rows specified in the export task are included." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "# Insert the global key of the data row you wish to export\n", + "DATA_ROW_GLOBAL_KEY = \"\"" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "#### Export V2 Method" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "# Set the export params to include/exclude certain fields.\n", + "export_params= {\n", + " \"attachments\": True,\n", + " \"metadata_fields\": True,\n", + " \"data_row_details\": True,\n", + " \"project_details\": True,\n", + " \"label_details\": True,\n", + " \"performance_details\": True,\n", + " \"interpolated_frames\": True\n", + "}\n", + "\n", + "# Provide a list of data row global keys\n", + "export_task = lb.DataRow.export_v2(client=client, global_keys=[DATA_ROW_GLOBAL_KEY], params=export_params)\n", + "export_task.wait_till_done()\n", + "\n", + "if export_task.errors:\n", + " print(export_task.errors)\n", + "\n", + "export_json = export_task.result\n", + "print(\"results: \", export_json)" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "#### Stream Task Export Method\n", + "The return type of this method is an ExportTask, instead of a Task. This is just a wrapper around Task, and most of its features are also present in ExportTask.\n", + "This allows streaming of task results and errors." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "# Set the export params to include/exclude certain fields.\n", + "export_params= {\n", + " \"attachments\": True,\n", + " \"metadata_fields\": True,\n", + " \"data_row_details\": True,\n", + " \"project_details\": True,\n", + " \"label_details\": True,\n", + " \"performance_details\": True,\n", + " \"interpolated_frames\": True\n", + "}\n", + "\n", + "client.enable_experimental = True\n", + "\n", + "# Provide a list of data row global keys\n", + "export_task = lb.DataRow.export(client=client, global_keys=[DATA_ROW_GLOBAL_KEY], params=export_params)\n", + "export_task.wait_till_done()" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "# Provide results with JSON converter\n", + "# Returns streamed JSON output strings from export task results/errors, one by one\n", + "\n", + "# Callback used for JSON Converter\n", + "def json_stream_handler(output: lb.JsonConverterOutput):\n", + " print(output.json_str)\n", + "\n", + "if export_task.has_errors():\n", + " export_task.get_stream(\n", + " converter=lb.JsonConverter(),\n", + " stream_type=lb.StreamType.ERRORS\n", + " ).start(stream_handler=lambda error: print(error))\n", + "\n", + "if export_task.has_result():\n", + " export_json = export_task.get_stream(\n", + " converter=lb.JsonConverter(),\n", + " stream_type=lb.StreamType.RESULT\n", + " ).start(stream_handler=json_stream_handler)\n", + "\n", + "print(\"file size: \", export_task.get_total_file_size(stream_type=lb.StreamType.RESULT))\n", + "print(\"line count: \", export_task.get_total_lines(stream_type=lb.StreamType.RESULT))" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, { "metadata": {}, "source": [ @@ -388,19 +892,7 @@ "image\n" ], "cell_type": "code", - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAACgAAAAarCAAAAADsRK9dAAAa60lEQVR4nO3dyZLbRgBEQVCh//9l+SDJ4gxJrI3eKvNkhS0TxFxeVGPIZQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACDXo/UFAAAE+vX/PzWosR/1XxIAIN2vt/9YiwAEAKitQfQ9E4AAAJX9WvlTDQIQAKCuxvufAAQAqOyl/6oHod8CBgCo6G3tVS4yCyAAQD3Nj3+XxQIIAFDPx/yrm2QCEACgjrX1r2qT/az5YgAAuU6f/j79xTKdaAEEAKhho//eRtnq3zmfcQIQAOC74pvbjvnv9YX2TYYnLlAAAgB88dJdBXppV8t9e53zvzC8dcECEADg2Q0f1Lc35Z5f5drnxaxfrwAEAHjyKbwuRNOhlnuc+Duf/zcn/iUAQJaV7jpdTW0++3ntcn0TCADAX2utdrbjGn33x9rLCkAAgD/WW62Lb3Hbb+VyBSAAwG9bhXeqANtl4+dXFoAAAMuy7Em1WTZAAQgAsCz76u54AXbZjAIQAGDptNQu+vSeBCAAwO7+O5qJfWalAAQA2K/Povvkw9UKQACAu7qu01wUgAAAB0LtSNN12n8CEADgUKjt/4977T8BCADEOxhqe//zbvtPAAIA6e4JtV9d9N/7i3hUvgoAgL6cCLWNgOoi/f54e6kCEACIdqrWPhZUT+23LIsABAB4cbbYXhuqu/ZblkUAAgB8cyHavkZUn/W3LO9jTwACAKmuVdv/FdVv/C2LAAQAeHI53B69x9+yLG9rTwACAJEGSLdCXnPP5wACAIly+u8NAQgABErqv9f3KgABgDxJ/feGZwABgDRx+fc9+CyAAECYuP57eccCEADIktd/LwQgABAlsv++vWkBCAAkiey/7/wSCACQIzf/vjSfBRAAiJHbf18JQAAgRXL/fXnvAhAACJHcf1/f/c9mVwEAUFF2/n1lAQQAEui/pzsgAAGAAPrv+R4IQABgfvpvWZ7uggAEAKan/377ex8EIAAwO/3315874ZtAAIDJ6b/vLIAAAGEEIAAwNwPgCwEIAExN/70SgAAAYQQgADAzA+AbAhAAIIwABAAmZgB8RwACAPPSf2/5IGgAYFby7wMLIAAwKf33iQAEAOak/z4SgAAAYQQgADAlA+BnAhAAmJH+WyEAAYAJ6b81AhAAmI/+WyUAAYDp6L91AhAAmI3+2yAAAQDCCEAAYDIGwC0CEACYi/7bJAABAMIIQABgKgbAbQIQACCMAAQAZmIA3EEAAgCEEYAAwEQMgHsIQABgHvpvFwEIAExD/+3zaH0BAABFqL/dBCAAMAH1d4QABACGJ/+OEYAAwODk31ECEAAYmfo7QQACAOOSf6cIQABgVPLvJAEIAIxJ/p0mAAGAEcm/C3wTCAAwIP13hQUQABiO/LvGAggAjEb/XWQBBADGIv8uswACAEPRf9dZAAGADsm8OwlAAKAr0u9+joABgJ7ovwoEIADQEf1XgwAEAPqh/6oQgABAN/RfHQIQACCMAAQAemEArEQAAgCEEYAAQCcMgLUIQACAMAIQAOiDAbAaAQgAdEH/1SMAAQDCCEAAoAcGwIoEIABAGAEIAHTAAFiTAAQACCMAAQDCCEAAoD0nwFUJQACAMAIQACCMAAQACCMAAYDmPAJYlwAEAAgjAAEAwghAAKA1J8CVCUAAgDACEABozABYmwAEAAgjAAGAtgyA1QlAAIAwAhAAaMoAWJ8ABABa0n8NCEAAgDACEABoyADYggAEANrRf00IQACgGf3XhgAEAAgjAAGAVgyAjQhAAIAwAhAAIIwABAAacQLcigAEAAgjAAEAwghAAIAwAhAAaMMjgM0IQACAMAIQACCMAAQACCMAAQDCCEAAoAm/A9KOAAQACCMAAQDCCEAAgDACEABowSOADQlAAIAwAhAAIIwABAAIIwABgAY8AtiSAAQACCMAAQDCCEAAoD4nwE0JQACAMAIQAKjOANiWAAQAatN/jQlAAKAy/deaAAQA6tJ/zQlAAKAq/deeAAQAatJ/HRCAAEBF+q8HAhAAqEf/dUEAAgCEEYAAQDUGwD4IQACgFv3XCQEIABBGAAIAlRgAeyEAAYA69F83BCAAUIX+64cABABq0H8dEYAAQAX6rycCEAC4n/7rigAEAG6n//oiAAGAu+m/zghAAOBm+q83AhAAuJf+644ABABupf/6IwABgDvpvw4JQADgRvqvRwIQALiP/uuSAAQAbqP/+iQAAYC76L9OPVpfAAAwKfnXLQEIANxB/nXMETAAcAP91zMBCACUp/+6JgABgOL0X98EIABQmv7rnAAEAAgjAAGAwgyAvROAAABhBCAAUJYBsHsCEAAgjAAEAIoyAPZPAAIAhBGAAABhBCAAQBgBCACU5BHAAQhAAIAwAhAAIIwABAAKcgI8AgEIABBGAAIAhBGAAEA5ToCHIAABAMIIQACgGAPgGAQgAEAYAQgAlGIAHIQABAAIIwABgEIMgKMQgAAAYQQgAEAYAQgAlOEEeBgCEAAgjAAEAAgjAAGAIpwAj0MAAgCEEYAAAGEEIABQghPggQhAAIAwAhAAIIwABAAIIwABgAI8AjgSAQgAEEYAAgCEEYAAAGEEIABwnUcAhyIAAQDCCEAA4DID4FgEIABAGAEIABBGAAIAVzkBHowABAAIIwABgIsMgKMRgADANfpvOAIQALhE/43n0foCAICRyb8RWQABgPP035AsgADAWfJvUAIQADhH/g1LAAIAZ8i/gQlAAOAw9Tc2AQgAHKP+hicAAYAj5N8EfAwMAHCA/puBAAQA9tN/UxCAAMBu+m8OAhAA2Ev/TUIAAgCEEYAAwE4GwFkIQACAMAIQANjHADgNAQgAEEYAAgC7GADnIQABAMIIQABgDwPgRAQgAEAYAQgA7GAAnIkABAAIIwABAMIIQABgmxPgqQhAAIAwAhAA2GQAnIsABAAIIwABAMIIQABgixPgyQhAAIAwAhAAIIwABAAIIwABgA0eAZyNAAQACCMAAQDCCEAAYJ0T4OkIQACAMAIQACCMAAQACCMAAYBVHgGcjwAEAAgjAAEAwghAAGCNE+AJCUAAgDACEAAgjAAEAFY4AZ6RAAQACCMAAQDCCEAA4DMnwFMSgAAAYQQgAEAYAQgAEEYAAgAfeQRwTgIQACCMAAQACCMAAQDCCEAAgDACEAAgjAAEAAgjAAEAwghAAIAwAhAAIIwABAAIIwABAMIIQACAMAIQACCMAAQAPnq0vgBuIQABAMIIQACAMAIQAPjMGfCUBCAAQBgBCACsMAHOSAACAIQRgADAGhPghAQgAEAYAQgArDIBzkcAAgCEEYAAwDoT4HQEIABAGAEIAGwwAc5GAAIAWxTgZAQgALBJAc5FAAIAhBGAAMA2E+BUBCAAsIMCnIkABAD2UIATEYAAAGEEIACwiwlwHgIQACCMAAQA9jEBTkMAAgA7KcBZCEAAgDACEADYywQ4CQEIAOymAOcgAAGA/RTgFAQgAHCAApyBAAQAjlCAExCAAMAhCnB8AhAAOEYBDk8AAgCEEYAAAGEEIABAGAEIABzkIcDRCUAAgDACEAAgjAAEAI5yBjw4AQgAEEYAAgCHmQDHJgABAMIIQADgOBPg0AQgAEAYAQgAnGACHJkABAAIIwABgDNMgAMTgAAAYQQgAHCKCXBcAhAAIIwABADOMQEOSwACAIQRgAAAYQQgAEAYAQgAnOQhwFEJQACAMAIQACCMAAQACCMAAQDCCEAA4Cy/BTIoAQgAEEYAAgCEEYAAAGEEIABAGAEIABBGAAIAhBGAAABhBCAAQBgBCAAQRgACAIQRgADAab4LbkwCEAAgjAAEAAgjAAEAwghAAIAwAhAAIMzP1hcAAEA1j2URgAAAOf58bo8jYACAEH8/t9ECCAAQ4d/HdgtAAIAAz9/a4ggYADjPd8GN4stPygIIADC7x+ofAQAO+dX6Atj00nuOgAEApva691kAAYBLTIB9exd7ngEEAJjVh6nPAggAXGMC7NTnzBOAAMBFCrBHa5EnAAGAqxRgd9YTTwACAJcpwL5sBZ4ABAAKkID92M47AQgAlKAAO7En7gQgAFCGBOzBrrYTgABAKRKwsb1hJwABgGIUYEv7s04AAgAlacAGjgadAAQAypKAlZzPOAEIAJQmAW93LeEEIABQnAK81eV+E4AAwA0k4G0K1JsABADuoADvUaTdBCAAcA8JWF6hcvtR5n8DAPCNmam4UrfUjwYAuI0RsKhi3SYAAYAbScByymWbAAQAbiUBCylYbZ4BBABuZW0qo+R99DMBAO5mBLyuaLNZAAGAuxmcLit7C/1AAIAKjICXFC42AQgAVCEBzysdbI6AAYAqHmanbvhRAADVWAFPKd5rAhAAqEgCHlc+1wQgAFCXBjxIAAIA45OAR9xQawIQAGhAA+51R6wJQACgCQm4yy2tJgABgEYk4LZ7Us3nAAIAjfhkwFYEIADQjALccNMNEoAAQDsKsAkBCAA0pABbEIAAQEsKcMVdN0cAAgBNKcD6BCAA0JYC/OS2OyMAAYDGFGBtAhAAaE0BvnXfbRGAAEBzCvCNG2+KAAQA2lOAL+68JQIQAOiAAvzm1hsiAAGAHijAL+69HW42ANCJX60voB83F5oFEADoxMMwVYkABAC6oQB/u/s+uM8AQFccBN/fZwIQAOhMegLen2cCEADoT3IDVqgzAQgAdCm1AWvEmQAEAHqV2IBV2kwAAgAdi2tAAQgAkNWAddJMAAIA3YtpwEplJgABgEHMn4G1wkwAAgAz+12Nj+c/dKtalwlAACBLtxVYL8sEIACQp8cIrFhlAhAASNZNCtaMsh8VXwsAoDePxDUs8T0DADzrYQWs2mQWQAAgXdwgJgABgHjtC7DuFQhAAID2BVhV2NsFAPig6ZOAdZNMAAIA/NGsASsXmQAEAPinTQNWLjLPAAIA/BPxuYACEADgWUACCkAAgK+mL8Dp3yAAwHF1HwWsHWQWQACAF3NvZAIQAODV1E8CCkAAgHcmLkABCADwVrUCrJ6aAhAA4L1pN0ABCADwwawFKAABAD6ZtAAnfVsAAGXc/4mA9XPMAggAsGLGtUwAAgCsmbAABSAAQEsNAlMAAgCEEYAAAKvmOwMWgAAAYQQgAEBDLfZFAQgAEEYAAgCsm+4hQAEIABBGAAIAtNNkXRSAAABhBCAAwIbZHgIUgAAAYQQgAEAYAQgAEEYAAgBsmewhQAEIABBGAAIAhBGAAACb5joDFoAAANvmKkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAG70H126NUI+OpM6AAAAAElFTkSuQmCC", - "text/plain": [ - "" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "execution_count": null } ] diff --git a/labelbox/__init__.py b/labelbox/__init__.py index ab617b4f1..a7263200e 100644 --- a/labelbox/__init__.py +++ b/labelbox/__init__.py @@ -1,6 +1,6 @@ name = "labelbox" -__version__ = "3.58.0" +__version__ = "3.58.1" from labelbox.client import Client from labelbox.schema.project import Project diff --git a/labelbox/schema/data_row.py b/labelbox/schema/data_row.py index 088d04efa..9b0667d5f 100644 --- a/labelbox/schema/data_row.py +++ b/labelbox/schema/data_row.py @@ -240,6 +240,8 @@ def export_v2( "model_run_ids": None, "project_ids": None, "interpolated_frames": False, + "all_projects": False, + "all_model_runs": False, }) validate_catalog_export_params(_params) @@ -303,6 +305,10 @@ def export_v2( _params.get('project_ids', None), "modelRunIds": _params.get('model_run_ids', None), + "allProjects": + _params.get('all_projects', False), + "allModelRuns": + _params.get('all_model_runs', False), }, "streamable": streamable } diff --git a/labelbox/schema/dataset.py b/labelbox/schema/dataset.py index b563c9f1f..9aa1ea5c5 100644 --- a/labelbox/schema/dataset.py +++ b/labelbox/schema/dataset.py @@ -658,6 +658,8 @@ def export_v2( "model_run_ids": None, "project_ids": None, "interpolated_frames": False, + "all_projects": False, + "all_model_runs": False, }) validate_catalog_export_params(_params) @@ -708,6 +710,10 @@ def export_v2( _params.get('project_ids', None), "modelRunIds": _params.get('model_run_ids', None), + "allProjects": + _params.get('all_projects', False), + "allModelRuns": + _params.get('all_model_runs', False), }, "streamable": streamable, } diff --git a/labelbox/schema/export_params.py b/labelbox/schema/export_params.py index 275e3f2b3..6da97b17b 100644 --- a/labelbox/schema/export_params.py +++ b/labelbox/schema/export_params.py @@ -32,6 +32,8 @@ class CatalogExportParams(DataRowParams): model_run_ids: Optional[List[str]] project_ids: Optional[List[str]] interpolated_frames: Optional[bool] + all_projects: Optional[bool] + all_model_runs: Optional[bool] class ModelRunExportParams(DataRowParams): diff --git a/labelbox/schema/identifiables.py b/labelbox/schema/identifiables.py index ab4c89b53..ba4265b17 100644 --- a/labelbox/schema/identifiables.py +++ b/labelbox/schema/identifiables.py @@ -29,6 +29,9 @@ def __init__(self, iterable, id_type: IdType): def __iter__(self): return iter(self._iterable) + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self._iterable})" + class UniqueIds(Identifiables): """ diff --git a/labelbox/schema/project.py b/labelbox/schema/project.py index 648b84a37..3bcd6333c 100644 --- a/labelbox/schema/project.py +++ b/labelbox/schema/project.py @@ -1185,10 +1185,26 @@ def set_labeling_parameter_overrides(self, data) -> bool: res = self.client.execute(query_str, {id_param: self.uid}) return res["project"]["setLabelingParameterOverrides"]["success"] + @overload + def update_data_row_labeling_priority( + self, + data_rows: DataRowIdentifiers, + priority: int, + ) -> bool: + pass + + @overload def update_data_row_labeling_priority( self, data_rows: List[str], priority: int, + ) -> bool: + pass + + def update_data_row_labeling_priority( + self, + data_rows, + priority: int, ) -> bool: """ Updates labeling parameter overrides to this project in bulk. This method allows up to 1 million data rows to be @@ -1198,25 +1214,31 @@ def update_data_row_labeling_priority( https://docs.labelbox.com/en/configure-editor/queue-system#reservation-system Args: - data_rows (iterable): An iterable of data row ids. + data_rows: a list of data row ids to update priorities for. This can be a list of strings or a DataRowIdentifiers object + DataRowIdentifier objects are lists of ids or global keys. A DataIdentifier object can be a UniqueIds or GlobalKeys class. priority (int): Priority for the new override. See above for more information. Returns: bool, indicates if the operation was a success. """ + if isinstance(data_rows, list): + data_rows = UniqueIds(data_rows) + warnings.warn("Using data row ids will be deprecated. Please use " + "UniqueIds or GlobalKeys instead.") + method = "createQueuePriorityUpdateTask" priority_param = "priority" project_param = "projectId" - data_rows_param = "dataRowIds" + data_rows_param = "dataRowIdentifiers" query_str = """mutation %sPyApi( $%s: Int! $%s: ID! - $%s: [ID!] + $%s: QueuePriorityUpdateDataRowIdentifiersInput ) { project(where: { id: $%s }) { %s( - data: { priority: $%s, dataRowIds: $%s } + data: { priority: $%s, dataRowIdentifiers: $%s } ) { taskId } @@ -1228,7 +1250,10 @@ def update_data_row_labeling_priority( query_str, { priority_param: priority, project_param: self.uid, - data_rows_param: data_rows + data_rows_param: { + "ids": [id for id in data_rows], + "idType": data_rows._id_type, + }, })["project"][method] task_id = res['taskId'] diff --git a/labelbox/schema/slice.py b/labelbox/schema/slice.py index 20e40f1c6..58d5d3b41 100644 --- a/labelbox/schema/slice.py +++ b/labelbox/schema/slice.py @@ -108,6 +108,8 @@ def export_v2( "model_run_ids": None, "project_ids": None, "interpolated_frames": False, + "all_projects": False, + "all_model_runs": False, }) validate_catalog_export_params(_params) @@ -146,6 +148,10 @@ def export_v2( _params.get('project_ids', None), "modelRunIds": _params.get('model_run_ids', None), + "allProjects": + _params.get('all_projects', False), + "allModelRuns": + _params.get('all_model_runs', False), }, "streamable": streamable, } diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 8fc514b29..8f6b00b48 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -173,13 +173,18 @@ def consensus_project_with_batch(consensus_project, initial_dataset, rand_gen, project = consensus_project dataset = initial_dataset - task = dataset.create_data_rows([{DataRow.row_data: image_url}] * 3) + data_rows = [] + for _ in range(3): + data_rows.append({ + DataRow.row_data: image_url, + DataRow.global_key: str(uuid.uuid4()) + }) + task = dataset.create_data_rows(data_rows) task.wait_till_done() assert task.status == "COMPLETE" data_rows = list(dataset.data_rows()) assert len(data_rows) == 3 - batch = project.create_batch( rand_gen(str), data_rows, # sample of data row objects @@ -745,13 +750,6 @@ def export_v2_test_helpers() -> Type[ExportV2Helpers]: return ExportV2Helpers() -@pytest.fixture(scope="session") -def is_adv_enabled(client) -> bool: - query_str = "query IsAdvEnabledPyApi { user { isAdvEnabled } }" - response = client.execute(query_str) - return bool(response['user']['isAdvEnabled']) - - IMAGE_URL = "https://storage.googleapis.com/diagnostics-demo-data/coco/COCO_train2014_000000000034.jpg" EXTERNAL_ID = "my-image" diff --git a/tests/integration/test_data_row_metadata.py b/tests/integration/test_data_row_metadata.py index 6b6f79206..3d2fc9224 100644 --- a/tests/integration/test_data_row_metadata.py +++ b/tests/integration/test_data_row_metadata.py @@ -227,7 +227,7 @@ def test_bulk_partial_delete_datarow_metadata(data_row, mdo): assert len(fields) == (len(metadata.fields) - 1) -def test_large_bulk_delete_datarow_metadata(big_dataset, mdo, is_adv_enabled): +def test_large_bulk_delete_datarow_metadata(big_dataset, mdo): metadata = [] data_row_ids = [dr.uid for dr in big_dataset.data_rows()] for data_row_id in data_row_ids: @@ -249,13 +249,11 @@ def test_large_bulk_delete_datarow_metadata(big_dataset, mdo, is_adv_enabled): data_row_id=data_row_id, fields=[SPLIT_SCHEMA_ID, CAPTURE_DT_SCHEMA_ID])) errors = mdo.bulk_delete(deletes) - if is_adv_enabled: - assert len(errors) == len(data_row_ids) - for error in errors: - assert error.fields == [CAPTURE_DT_SCHEMA_ID] - assert error.error == 'Schema did not exist' - else: - assert len(errors) == 0 + + assert len(errors) == len(data_row_ids) + for error in errors: + assert error.fields == [CAPTURE_DT_SCHEMA_ID] + assert error.error == 'Schema did not exist' for data_row_id in data_row_ids: fields = [f for f in mdo.bulk_export([data_row_id])[0].fields] @@ -308,17 +306,14 @@ def test_upsert_non_existent_schema_id(data_row, mdo): mdo.bulk_upsert([metadata]) -def test_delete_non_existent_schema_id(data_row, mdo, is_adv_enabled): +def test_delete_non_existent_schema_id(data_row, mdo): res = mdo.bulk_delete([ DeleteDataRowMetadata(data_row_id=data_row.uid, fields=[SPLIT_SCHEMA_ID]) ]) - if is_adv_enabled: - assert len(res) == 1 - assert res[0].fields == [SPLIT_SCHEMA_ID] - assert res[0].error == 'Schema did not exist' - else: - assert len(res) == 0 + assert len(res) == 1 + assert res[0].fields == [SPLIT_SCHEMA_ID] + assert res[0].error == 'Schema did not exist' def test_parse_raw_metadata(mdo): diff --git a/tests/integration/test_data_rows.py b/tests/integration/test_data_rows.py index 89e1a818e..326a93c27 100644 --- a/tests/integration/test_data_rows.py +++ b/tests/integration/test_data_rows.py @@ -10,11 +10,9 @@ from labelbox import DataRow from labelbox.exceptions import MalformedQueryException -from labelbox.schema.export_filters import DatarowExportFilters from labelbox.schema.task import Task from labelbox.schema.data_row_metadata import DataRowMetadataField, DataRowMetadataKind import labelbox.exceptions -from utils import INTEGRATION_SNAPSHOT_DIRECTORY SPLIT_SCHEMA_ID = "cko8sbczn0002h2dkdaxb5kal" TEST_SPLIT_ID = "cko8scbz70005h2dkastwhgqt" @@ -484,8 +482,7 @@ def create_data_row(data_rows): CUSTOM_TEXT_SCHEMA_NAME].uid -def test_create_data_rows_with_invalid_metadata(dataset, image_url, - is_adv_enabled): +def test_create_data_rows_with_invalid_metadata(dataset, image_url): fields = make_metadata_fields() # make the payload invalid by providing the same schema id more than once fields.append( @@ -496,14 +493,11 @@ def test_create_data_rows_with_invalid_metadata(dataset, image_url, DataRow.metadata_fields: fields }]) task.wait_till_done(timeout_seconds=60) - if is_adv_enabled: - assert task.status == "COMPLETE" - assert len(task.failed_data_rows) == 1 - assert f"A schemaId can only be specified once per DataRow : [{TEXT_SCHEMA_ID}]" in task.failed_data_rows[ - 0]["message"] - else: - assert task.status == "FAILED" - assert len(task.failed_data_rows) > 0 + + assert task.status == "COMPLETE" + assert len(task.failed_data_rows) == 1 + assert f"A schemaId can only be specified once per DataRow : [{TEXT_SCHEMA_ID}]" in task.failed_data_rows[ + 0]["message"] def test_create_data_rows_with_metadata_missing_value(dataset, image_url): @@ -815,7 +809,7 @@ def test_data_row_bulk_creation_with_unique_global_keys(dataset, sample_image): def test_data_row_bulk_creation_with_same_global_keys(dataset, sample_image, - snapshot, is_adv_enabled): + snapshot): global_key_1 = str(uuid.uuid4()) task = dataset.create_data_rows([{ DataRow.row_data: sample_image, @@ -826,48 +820,22 @@ def test_data_row_bulk_creation_with_same_global_keys(dataset, sample_image, }]) task.wait_till_done() - if is_adv_enabled: - assert task.status == "COMPLETE" - assert type(task.failed_data_rows) is list - assert len(task.failed_data_rows) == 1 - assert type(task.created_data_rows) is list - assert len(task.created_data_rows) == 1 - assert task.failed_data_rows[0][ - 'message'] == f"Duplicate global key: '{global_key_1}'" - assert task.failed_data_rows[0]['failedDataRows'][0][ - 'externalId'] == sample_image - assert task.created_data_rows[0]['externalId'] == sample_image - assert task.created_data_rows[0]['globalKey'] == global_key_1 - else: - assert task.status == "FAILED" - assert len(task.failed_data_rows) > 0 - assert len(list(dataset.data_rows())) == 0 - assert task.errors == "Data rows contain duplicate global keys" - - # Dynamic values, resetting to make snapshot - task.failed_data_rows[0]['failedDataRows'][0]['rowData'] = '' - task.failed_data_rows[0]['failedDataRows'][1]['rowData'] = '' - task.failed_data_rows[0]['failedDataRows'][0]['globalKey'] = '' - task.failed_data_rows[0]['failedDataRows'][1]['globalKey'] = '' - snapshot.snapshot_dir = INTEGRATION_SNAPSHOT_DIRECTORY - snapshot.assert_match( - json.dumps(task.failed_data_rows), - 'test_data_rows.test_data_row_bulk_creation_with_same_global_keys.failed_data_rows.json' - ) - - task = dataset.create_data_rows([{ - DataRow.row_data: sample_image, - DataRow.global_key: global_key_1 - }]) - task.wait_till_done() - assert task.status == "COMPLETE" - assert len(list(dataset.data_rows())) == 1 - assert list(dataset.data_rows())[0].global_key == global_key_1 + assert task.status == "COMPLETE" + assert type(task.failed_data_rows) is list + assert len(task.failed_data_rows) == 1 + assert type(task.created_data_rows) is list + assert len(task.created_data_rows) == 1 + assert task.failed_data_rows[0][ + 'message'] == f"Duplicate global key: '{global_key_1}'" + assert task.failed_data_rows[0]['failedDataRows'][0][ + 'externalId'] == sample_image + assert task.created_data_rows[0]['externalId'] == sample_image + assert task.created_data_rows[0]['globalKey'] == global_key_1 def test_data_row_delete_and_create_with_same_global_key( - client, dataset, sample_image, is_adv_enabled): + client, dataset, sample_image): global_key_1 = str(uuid.uuid4()) data_row_payload = { DataRow.row_data: sample_image, @@ -887,15 +855,10 @@ def test_data_row_delete_and_create_with_same_global_key( task = dataset.create_data_rows([data_row_payload]) task.wait_till_done() - if is_adv_enabled: - assert task.status == "COMPLETE" - assert len(task.failed_data_rows) == 1 - assert task.failed_data_rows[0][ - 'message'] == f"Duplicate global key: '{global_key_1}'" - else: - assert task.status == "FAILED" - assert len(task.failed_data_rows) > 0 - assert task.errors.startswith("Duplicate global keys found") + assert task.status == "COMPLETE" + assert len(task.failed_data_rows) == 1 + assert task.failed_data_rows[0][ + 'message'] == f"Duplicate global key: '{global_key_1}'" # delete datarow client.get_data_row(new_data_row_id).delete() @@ -934,7 +897,7 @@ def test_data_row_bulk_creation_sync_with_unique_global_keys( def test_data_row_bulk_creation_sync_with_same_global_keys( - dataset, sample_image, is_adv_enabled): + dataset, sample_image): global_key_1 = str(uuid.uuid4()) with pytest.raises(labelbox.exceptions.MalformedQueryException) as exc_info: @@ -946,22 +909,10 @@ def test_data_row_bulk_creation_sync_with_same_global_keys( DataRow.global_key: global_key_1 }]) - if is_adv_enabled: - # ADV will import the first data row but not the second (duplicate global key) - assert len(list(dataset.data_rows())) == 1 - assert list(dataset.data_rows())[0].global_key == global_key_1 - assert "Some data rows were not imported. Check error output here" in str( - exc_info.value) - else: - assert len(list(dataset.data_rows())) == 0 - - dataset.create_data_rows_sync([{ - DataRow.row_data: sample_image, - DataRow.global_key: global_key_1 - }]) - - assert len(list(dataset.data_rows())) == 1 - assert list(dataset.data_rows())[0].global_key == global_key_1 + assert len(list(dataset.data_rows())) == 1 + assert list(dataset.data_rows())[0].global_key == global_key_1 + assert "Some data rows were not imported. Check error output here" in str( + exc_info.value) @pytest.fixture @@ -995,10 +946,12 @@ def test_create_conversational_text(converstational_data_rows, data_row.row_data).json() == conversational_content['row_data'] -def test_invalid_media_type(dataset, conversational_content, is_adv_enabled): - for error_message, invalid_media_type in [[ - "Found invalid contents for media type: 'IMAGE'", 'IMAGE' - ], ["Found invalid media type: 'totallyinvalid'", 'totallyinvalid']]: +def test_invalid_media_type(dataset, conversational_content): + for _, __ in [["Found invalid contents for media type: 'IMAGE'", 'IMAGE'], + [ + "Found invalid media type: 'totallyinvalid'", + 'totallyinvalid' + ]]: # TODO: What error kind should this be? It looks like for global key we are # using malformed query. But for invalid contents in FileUploads we use InvalidQueryError with pytest.raises(labelbox.exceptions.InvalidQueryError): @@ -1006,16 +959,6 @@ def test_invalid_media_type(dataset, conversational_content, is_adv_enabled): **conversational_content, 'media_type': 'IMAGE' }]) - if is_adv_enabled: - # ADV does not take media type hint into account for async import requests - continue - - task = dataset.create_data_rows([{ - **conversational_content, 'media_type': invalid_media_type - }]) - task.wait_till_done() - assert task.errors == {'message': error_message} - def test_create_tiled_layer(dataset, tile_content): examples = [ @@ -1044,15 +987,12 @@ def test_create_data_row_with_attachments(dataset): assert len(attachments) == 1 -def test_create_data_row_with_media_type(dataset, image_url, is_adv_enabled): +def test_create_data_row_with_media_type(dataset, image_url): with pytest.raises(labelbox.exceptions.InvalidQueryError) as exc: dr = dataset.create_data_row( row_data={'invalid_object': 'invalid_value'}, media_type="IMAGE") - if is_adv_enabled: - assert "Media type validation failed, expected: 'image/*', was: application/json" in str( - exc.value) - else: - assert "Found invalid contents for media type: \'IMAGE\'" in str( - exc.value) + + assert "Media type validation failed, expected: 'image/*', was: application/json" in str( + exc.value) dataset.create_data_row(row_data=image_url, media_type="IMAGE") diff --git a/tests/integration/test_labeling_parameter_overrides.py b/tests/integration/test_labeling_parameter_overrides.py index 46b46b2b1..20cb6d000 100644 --- a/tests/integration/test_labeling_parameter_overrides.py +++ b/tests/integration/test_labeling_parameter_overrides.py @@ -1,5 +1,6 @@ import pytest from labelbox import DataRow +from labelbox.schema.identifiables import GlobalKeys, UniqueIds def test_labeling_parameter_overrides(consensus_project_with_batch): @@ -49,8 +50,21 @@ def test_set_labeling_priority(consensus_project_with_batch): data = [data_row.uid for data_row in data_rows] success = project.update_data_row_labeling_priority(data, 1) + lo = list(project.labeling_parameter_overrides()) assert success + assert len(lo) == 3 + assert {o.priority for o in lo} == {1, 1, 1} - updated_overrides = list(project.labeling_parameter_overrides()) - assert len(updated_overrides) == 3 - assert {o.priority for o in updated_overrides} == {1, 1, 1} + data = [data_row.uid for data_row in data_rows] + success = project.update_data_row_labeling_priority(UniqueIds(data), 2) + lo = list(project.labeling_parameter_overrides()) + assert success + assert len(lo) == 3 + assert {o.priority for o in lo} == {2, 2, 2} + + data = [data_row.global_key for data_row in data_rows] + success = project.update_data_row_labeling_priority(GlobalKeys(data), 3) + lo = list(project.labeling_parameter_overrides()) + assert success + assert len(lo) == 3 + assert {o.priority for o in lo} == {3, 3, 3} diff --git a/tests/integration/test_task.py b/tests/integration/test_task.py index b035b09ed..6def2dbbb 100644 --- a/tests/integration/test_task.py +++ b/tests/integration/test_task.py @@ -8,7 +8,7 @@ TEXT_SCHEMA_ID = "cko8s9r5v0001h2dk9elqdidh" -def test_task_errors(dataset, image_url, snapshot, is_adv_enabled): +def test_task_errors(dataset, image_url, snapshot): client = dataset.client task = dataset.create_data_rows([ { @@ -25,22 +25,11 @@ def test_task_errors(dataset, image_url, snapshot, is_adv_enabled): assert task in client.get_user().created_tasks() task.wait_till_done() - if is_adv_enabled: - assert len(task.failed_data_rows) == 1 - assert "A schemaId can only be specified once per DataRow : [cko8s9r5v0001h2dk9elqdidh]" in task.failed_data_rows[ - 0]['message'] - assert len( - task.failed_data_rows[0]['failedDataRows'][0]['metadata']) == 2 - else: - snapshot.snapshot_dir = INTEGRATION_SNAPSHOT_DIRECTORY - # RowData is dynamic, so we need to remove it from the snapshot - task.failed_data_rows[0]['failedDataRows'][0]['rowData'] = '' - snapshot.assert_match( - json.dumps(task.failed_data_rows), - 'test_task.test_task_errors.failed_data_rows.json') - assert task.errors is not None - snapshot.assert_match(json.dumps(task.errors), - 'test_task.test_task_errors.errors.json') + + assert len(task.failed_data_rows) == 1 + assert "A schemaId can only be specified once per DataRow : [cko8s9r5v0001h2dk9elqdidh]" in task.failed_data_rows[ + 0]['message'] + assert len(task.failed_data_rows[0]['failedDataRows'][0]['metadata']) == 2 def test_task_success_json(dataset, image_url, snapshot): diff --git a/tests/unit/test_unit_identifiables.py b/tests/unit/test_unit_identifiables.py index 9f834ef33..b52b3d4eb 100644 --- a/tests/unit/test_unit_identifiables.py +++ b/tests/unit/test_unit_identifiables.py @@ -13,3 +13,13 @@ def test_global_keys(): identifiables = GlobalKeys(ids) assert [i for i in identifiables] == ids assert identifiables._id_type == "GKEY" + + +def test_repr(): + ids = ["a", "b", "c"] + identifiables = GlobalKeys(ids) + assert repr(identifiables) == "GlobalKeys(['a', 'b', 'c'])" + + ids = ["a", "b", "c"] + identifiables = UniqueIds(ids) + assert repr(identifiables) == "UniqueIds(['a', 'b', 'c'])"