diff --git a/.github/workflows/github-projects.yml b/.github/workflows/github-projects.yml index ce658b79..9fbda814 100644 --- a/.github/workflows/github-projects.yml +++ b/.github/workflows/github-projects.yml @@ -1,14 +1,16 @@ -name: Add bugs to relevant GitHub Projects +name: Add issues to relevant GitHub Projects on: issues: types: + - labeled - opened - - labelled + - transferred + - edited jobs: add-to-project: - name: Add issue to project + name: Add issues to relevant GitHub Projects runs-on: ubuntu-latest steps: - uses: actions/add-to-project@main @@ -28,3 +30,75 @@ jobs: github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} labeled: 🎯 P0, 🎯 P1 label-operator: OR + - uses: actions/add-to-project@main + with: + project-url: https://github.com/orgs/openfoodfacts/projects/11 # Add issue to the openfoodfacts-design project + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + labeled: 🎨 Mockup available, 🎨 Mockup required + label-operator: OR + - uses: actions/add-to-project@main + with: + project-url: https://github.com/orgs/openfoodfacts/projects/36 # Add issue to the open pet food facts project + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + labeled: 🐾 Open Pet Food Facts + label-operator: OR + - uses: actions/add-to-project@main + with: + project-url: https://github.com/orgs/openfoodfacts/projects/43 # Add issue to the open products facts project + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + labeled: 📸 Open Products Facts + label-operator: OR + - uses: actions/add-to-project@main + with: + project-url: https://github.com/orgs/openfoodfacts/projects/37 # Add issue to the open beauty facts project + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + labeled: 🧴 Open Beauty Facts + label-operator: OR + - uses: actions/add-to-project@main + with: + project-url: https://github.com/orgs/openfoodfacts/projects/4 # Add issue to the packaging project + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + labeled: 📦 Packaging + label-operator: OR + - uses: actions/add-to-project@main + with: + project-url: https://github.com/orgs/openfoodfacts/projects/25 # Add issue to the documentation project + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + labeled: 📚 Documentation + label-operator: OR + - uses: actions/add-to-project@main + with: + project-url: https://github.com/orgs/openfoodfacts/projects/5 # Add issue to the folksonomy project + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + labeled: 🏷️ Folksonomy Project + label-operator: OR + - uses: actions/add-to-project@main + with: + project-url: https://github.com/orgs/openfoodfacts/projects/44 # Add issue to the data quality project + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + labeled: 🧽 Data quality + label-operator: OR + - uses: actions/add-to-project@main + with: + project-url: https://github.com/orgs/openfoodfacts/projects/82 # Add issue to the search project + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + labeled: 🔎 Search + label-operator: OR + - uses: actions/add-to-project@main + with: + project-url: https://github.com/orgs/openfoodfacts/projects/41 # Add issue to the producer platform project + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + labeled: 🏭 Producers Platform + label-operator: OR + - uses: actions/add-to-project@main + with: + project-url: https://github.com/orgs/openfoodfacts/projects/19 # Add issue to the infrastructure project + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + labeled: infrastructure + label-operator: OR + - uses: actions/add-to-project@main + with: + project-url: https://github.com/orgs/openfoodfacts/projects/92 # Add issue to the Nutri-Score project + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + labeled: 🚦 Nutri-Score + label-operator: OR diff --git a/genai-features/dataset/recent_changes.txt b/genai-features/dataset/recent_changes.txt new file mode 100644 index 00000000..ff3a4a96 --- /dev/null +++ b/genai-features/dataset/recent_changes.txt @@ -0,0 +1 @@ +http://static.openfoodfacts.org/data/openfoodfacts_recent_changes.jsonl.gz diff --git a/genai-features/notebooks/explore_recent_changes.ipynb b/genai-features/notebooks/explore_recent_changes.ipynb new file mode 100644 index 00000000..f1f16feb --- /dev/null +++ b/genai-features/notebooks/explore_recent_changes.ipynb @@ -0,0 +1,367 @@ +{ + "cells": [ + { + "cell_type": "code", + "id": "initial_id", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-10-02T05:33:00.083862Z", + "start_time": "2024-10-02T05:32:59.258183Z" + } + }, + "source": [ + "import pandas as pd\n", + "import json\n", + "import requests" + ], + "outputs": [], + "execution_count": 1 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-10-02T05:33:00.094035Z", + "start_time": "2024-10-02T05:33:00.092103Z" + } + }, + "cell_type": "code", + "source": [ + "n_sample = None\n", + "re_download = False" + ], + "id": "5471642089fb0a62", + "outputs": [], + "execution_count": 2 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-10-02T05:34:22.364615Z", + "start_time": "2024-10-02T05:33:00.106532Z" + } + }, + "cell_type": "code", + "source": [ + "data_url = 'http://static.openfoodfacts.org/data/openfoodfacts_recent_changes.jsonl.gz'\n", + "\n", + "if re_download:\n", + " import gzip\n", + " import shutil\n", + " # Download the data\n", + " data_path = '/Users/baslad01/data_dump'\n", + " file_path = f'{data_path}/openfoodfacts_recent_changes.jsonl.gz'\n", + " response = requests.get(data_url)\n", + " with open(file_path, 'wb') as file:\n", + " file.write(response.content)\n", + " \n", + " compressed_file_path = f'{data_path}/openfoodfacts_recent_changes.jsonl.gz'\n", + " uncompressed_file_path = f'{data_path}/openfoodfacts_recent_changes.jsonl'\n", + "\n", + " \n", + " with gzip.open(compressed_file_path, 'rb') as f_in:\n", + " with open(uncompressed_file_path, 'wb') as f_out:\n", + " shutil.copyfileobj(f_in, f_out)\n", + "\n", + "\n", + "data_path = '/Users/baslad01/data_dump'\n", + "file_path = f'{data_path}/openfoodfacts_recent_changes.jsonl'\n", + "key_words = ['vandal']\n", + "\n", + "filtered_data = []\n", + "\n", + "with open(file_path, 'r') as file:\n", + " for line in file:\n", + " try:\n", + " json_obj = json.loads(line)\n", + " if 'comment' in json_obj and any(kw.lower() in json_obj['comment'].lower() for kw in key_words):\n", + " filtered_data.append(json_obj)\n", + " except json.JSONDecodeError as e:\n", + " print(f\"Error decoding JSON: {e}\")\n", + "\n", + "df_recent_changes_filtered = pd.DataFrame(filtered_data)\n", + "df_recent_changes_filtered" + ], + "id": "56b61ebb99017c3d", + "outputs": [ + { + "data": { + "text/plain": [ + " _id userid code \\\n", + "0 {'$oid': '5bbcdc0a4ade5fdf2732e301'} sebleouf 3596654383769 \n", + "1 {'$oid': '5bbce3b24ade5f069444b8ce'} sebleouf 8010059016480 \n", + "2 {'$oid': '5bc064534ade5fee8676c08a'} sebleouf 9789045548647 \n", + "3 {'$oid': '5bc373984ade5f613e7f885c'} sebleouf 6922572400030 \n", + "4 {'$oid': '5bc4674c4ade5fa734766af5'} sebleouf 3515450030899 \n", + "... ... ... ... \n", + "1578 {'$oid': '66d886631ec3700d69da1183'} charlesnepote 0810554026773 \n", + "1579 {'$oid': '66d8868637b2e30ab8da1180'} charlesnepote 3760144210563 \n", + "1580 {'$oid': '66d886a906b365812fda1180'} charlesnepote 3760282062437 \n", + "1581 {'$oid': '66d8871362cae1c7fcda1180'} charlesnepote 8052282080203 \n", + "1582 {'$oid': '66d887396aabbb9bc9da1181'} charlesnepote 4056489774877 \n", + "\n", + " comment countries_tags diffs t rev \n", + "0 Suppression du produit :Vandalisme [en:france] {} 1539103754 8 \n", + "1 Suppression du produit :Vandalisme [en:france] {} 1539105714 8 \n", + "2 Suppression du produit :Vandalisme [en:belgium] {} 1539335251 6 \n", + "3 Suppression du produit :Vandalisme [en:belgium] {} 1539535765 6 \n", + "4 Deleting product:Vandalisme [en:algeria] {} 1539598156 5 \n", + "... ... ... ... ... ... \n", + "1578 Deleting product:Vandalism [en:france] {} 1725466211 7 \n", + "1579 Deleting product:Vandalism [en:france] {} 1725466246 7 \n", + "1580 Deleting product:Vandalism [en:france] {} 1725466281 7 \n", + "1581 Deleting product:Vandalism [en:france] {} 1725466387 10 \n", + "1582 Deleting product:Vandalism [en:france] {} 1725466425 9 \n", + "\n", + "[1583 rows x 8 columns]" + ], + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
_iduseridcodecommentcountries_tagsdiffstrev
0{'$oid': '5bbcdc0a4ade5fdf2732e301'}sebleouf3596654383769Suppression du produit :Vandalisme[en:france]{}15391037548
1{'$oid': '5bbce3b24ade5f069444b8ce'}sebleouf8010059016480Suppression du produit :Vandalisme[en:france]{}15391057148
2{'$oid': '5bc064534ade5fee8676c08a'}sebleouf9789045548647Suppression du produit :Vandalisme[en:belgium]{}15393352516
3{'$oid': '5bc373984ade5f613e7f885c'}sebleouf6922572400030Suppression du produit :Vandalisme[en:belgium]{}15395357656
4{'$oid': '5bc4674c4ade5fa734766af5'}sebleouf3515450030899Deleting product:Vandalisme[en:algeria]{}15395981565
...........................
1578{'$oid': '66d886631ec3700d69da1183'}charlesnepote0810554026773Deleting product:Vandalism[en:france]{}17254662117
1579{'$oid': '66d8868637b2e30ab8da1180'}charlesnepote3760144210563Deleting product:Vandalism[en:france]{}17254662467
1580{'$oid': '66d886a906b365812fda1180'}charlesnepote3760282062437Deleting product:Vandalism[en:france]{}17254662817
1581{'$oid': '66d8871362cae1c7fcda1180'}charlesnepote8052282080203Deleting product:Vandalism[en:france]{}172546638710
1582{'$oid': '66d887396aabbb9bc9da1181'}charlesnepote4056489774877Deleting product:Vandalism[en:france]{}17254664259
\n", + "

1583 rows × 8 columns

\n", + "
" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 3 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-10-02T05:39:14.162753Z", + "start_time": "2024-10-02T05:39:14.011972Z" + } + }, + "cell_type": "code", + "source": [ + "product_id = 875444\n", + "rev_id = 3\n", + "api_url = f\"https://world.openfoodfacts.org/api/v2/product/{product_id}?rev={rev_id}\"\n", + "# Get the product data\n", + "product_data = requests.get(api_url).json()\n", + "product_data" + ], + "id": "31a28bb96ee9ff2f", + "outputs": [ + { + "data": { + "text/plain": [ + "{'code': '875444', 'status': 0, 'status_verbose': 'product not found'}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 5 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-10-02T05:43:52.597713Z", + "start_time": "2024-10-02T05:43:52.593995Z" + } + }, + "cell_type": "code", + "source": "api_url", + "id": "669c29e99544240f", + "outputs": [ + { + "data": { + "text/plain": [ + "'https://world.openfoodfacts.org/api/v2/product/875444?rev=3'" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 6 + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": "", + "id": "797b55bb518c5c03" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ml_utils/ml_utils_cli/README.md b/ml_utils/ml_utils_cli/README.md new file mode 100644 index 00000000..719c697b --- /dev/null +++ b/ml_utils/ml_utils_cli/README.md @@ -0,0 +1,133 @@ +# ML CLI + +This is a command line interface that aims to provide a set of tools to help data scientists and machine learning engineers to deal with ML data annotation, data preprocessing and format conversion. + +This project started as a way to automate some of the tasks we do at Open Food Facts to manage data at different stages of the machine learning pipeline. + +The CLI currently is integrated with Label Studio (for data annotation), Ultralytics (for object detection) and Hugging Face (for model and dataset storage). It only works with some specific tasks (object detection only currently), but it's meant to be extended to other tasks in the future. + +It currently allows to: + +- create Label Studio projects +- upload images to Label Studio +- pre-annotate the tasks either with an existing object detection model run by Triton, or with Yolo-World (through Ultralytics) +- perform data quality checks on Label Studio +- export the data to Hugging Face Dataset or to local disk + +## Installation + +Python 3.9 or higher is required to run this CLI. +You need to install the CLI manually for now, there is no project published on Pypi. +To do so: + +We recommend to install the CLI in a virtual environment. First, create a virtual environment using conda: +```bash +conda create -n ml-cli python=3.12 +conda activate ml-cli +``` + +Then, clone the repository and install the requirements: + +```bash +git clone git@github.com:openfoodfacts/openfoodfacts-ai.git +``` + +```bash +python3 -m pip install -r requirements.txt +``` +or if you are using conda: +```bash +pip install -r requirements.txt +``` + +We assume in the following that you have installed the CLI in a virtual environment, and defined the following alias in your shell configuration file (e.g. `.bashrc` or `.zshrc`): + +```bash +alias ml-cli='${VIRTUALENV_DIR}/bin/python3 ${PROJECT_PATH}/main.py' +``` +or if you are using conda: +```bash +alias ml-cli='${CONDA_PREFIX}/bin/python3 ${PROJECT_PATH}/main.py' +``` + +with `${VIRTUALENV_DIR}` the path to the virtual environment where you installed the CLI and `${PROJECT_PATH}` the path to the root of the project, for example: +```bash +${PROJECT_PATH} = /home/user/openfoodfacts-ai/ml_utils/ml_utils_cli +``` + +## Usage + +### Label Studio integration + +To create a Label Studio project, you need to have a Label Studio instance running. Launching a Label Studio instance is out of the scope of this project, but you can follow the instructions on the [Label Studio documentation](https://labelstud.io/guide/install.html). + +By default, the CLI will use Open Food Facts Label Studio instance, but you can change the URL by setting the `--label-studio-url` CLI option. + +For all the commands that interact with Label Studio, you need to provide an API key using the `--api-key` CLI option. You can get an API key by logging in to the Label Studio instance and going to the Account & Settings page. + +#### Create a project + +Once you have a Label Studio instance running, you can create a project with the following command: + +```bash +ml-cli projects create --title my_project --api-key API_KEY --config-file label_config.xml +``` + +where `API_KEY` is the API key of the Label Studio instance (API key is available at Account page), and `label_config.xml` is the configuration file of the project. + +#### Create a dataset file + +If you have a list of images, for an object detection task, you can quickly create a dataset file with the following command: + +```bash +ml-cli projects create-dataset-file --input-file image_urls.txt --output-file dataset.json +``` + +where `image_urls.txt` is a file containing the URLs of the images, one per line, and `dataset.json` is the output file. + +#### Import data + +Next, import the generated data to a project with the following command: + +```bash +ml-cli projects import-data --project-id PROJECT_ID --dataset-path dataset.json +``` + +where `PROJECT_ID` is the ID of the project you created. + +#### Pre-annotate the data + +To accelerate annotation, you can pre-annotate the images with an object detection model. We support two pre-annotation backends: + +- Triton: you need to have a Triton server running with a model that supports object detection. The object detection model is expected to be a yolo-v8 model. You can set the URL of the Triton server with the `--triton-url` CLI option. + +- Ultralytics: you can use the [Yolo-World model from Ultralytics](https://github.com/ultralytics/ultralytics), Ultralytics should be installed in the same virtualenv. + +To pre-annotate the data with Triton, use the following command: + +```bash +ml-cli projects add-prediction --project-id PROJECT_ID --backend ultralytics --labels 'product' --labels 'price tag' --label-mapping '{"price tag": "price-tag"}' +``` + +where `labels` is the list of labels to use for the object detection task (you can add as many labels as you want). +For Ultralytics, you can also provide a `--label-mapping` option to map the labels from the model to the labels of the project. + +By default, for Ultralytics, the `yolov8x-worldv2.pt` model is used. You can change the model by setting the `--model-name` CLI option. + +#### Export the data + +Once the data is annotated, you can export it to a Hugging Face dataset or to local disk (Ultralytics format). To export it to disk, use the following command: + +```bash +ml-cli datasets export --project-id PROJECT_ID --from ls --to ultralytics --output-dir output --label-names 'product,price-tag' +``` + +where `output` is the directory where the data will be exported. Currently, label names must be provided, as the CLI does not support exporting label names from Label Studio yet. + +To export the data to a Hugging Face dataset, use the following command: + +```bash +ml-cli datasets export --project-id PROJECT_ID --from ls --to huggingface --repo-id REPO_ID --label-names 'product,price-tag' +``` + +where `REPO_ID` is the ID of the Hugging Face repository where the dataset will be uploaded (ex: `openfoodfacts/food-detection`). \ No newline at end of file diff --git a/ml_utils/ml_utils_cli/cli/annotate.py b/ml_utils/ml_utils_cli/cli/annotate.py new file mode 100644 index 00000000..4fbd3dd2 --- /dev/null +++ b/ml_utils/ml_utils_cli/cli/annotate.py @@ -0,0 +1,102 @@ +import random +import string + +from cli.triton.object_detection import ObjectDetectionResult +from openfoodfacts.utils import get_logger +from ultralytics.engine.results import Results + +logger = get_logger(__name__) + + +def format_annotation_results_from_triton( + objects: list[ObjectDetectionResult], image_width: int, image_height: int +): + """Format annotation results from a Triton object detection model into + Label Studio format.""" + annotation_results = [] + for object_ in objects: + bbox = object_.bounding_box + category_name = object_.label + # These are relative coordinates (between 0.0 and 1.0) + y_min, x_min, y_max, x_max = bbox + # Make sure the coordinates are within the image boundaries, + # and convert them to percentages + y_min = min(max(0, y_min), 1.0) * 100 + x_min = min(max(0, x_min), 1.0) * 100 + y_max = min(max(0, y_max), 1.0) * 100 + x_max = min(max(0, x_max), 1.0) * 100 + x = x_min + y = y_min + width = x_max - x_min + height = y_max - y_min + + id_ = generate_id() + annotation_results.append( + { + "id": id_, + "type": "rectanglelabels", + "from_name": "label", + "to_name": "image", + "original_width": image_width, + "original_height": image_height, + "image_rotation": 0, + "value": { + "rotation": 0, + "x": x, + "y": y, + "width": width, + "height": height, + "rectanglelabels": [category_name], + }, + }, + ) + return annotation_results + + +def format_annotation_results_from_ultralytics( + results: Results, + labels: list[str], + label_mapping: dict[str, str] | None = None, +) -> list[dict]: + annotation_results = [] + orig_height, orig_width = results.orig_shape + boxes = results.boxes + classes = boxes.cls.tolist() + for i, xyxyn in enumerate(boxes.xyxyn): + # Boxes found. + if len(xyxyn) > 0: + xyxyn = xyxyn.tolist() + x1 = xyxyn[0] * 100 + y1 = xyxyn[1] * 100 + x2 = xyxyn[2] * 100 + y2 = xyxyn[3] * 100 + width = x2 - x1 + height = y2 - y1 + label_id = int(classes[i]) + label_name = labels[label_id] + if label_mapping: + label_name = label_mapping.get(label_name, label_name) + annotation_results.append( + { + "id": generate_id(), + "type": "rectanglelabels", + "from_name": "label", + "to_name": "image", + "original_width": orig_width, + "original_height": orig_height, + "image_rotation": 0, + "value": { + "rotation": 0, + "x": x1, + "y": y1, + "width": width, + "height": height, + "rectanglelabels": [label_name], + }, + }, + ) + return annotation_results + + +def generate_id(length: int = 10) -> str: + return "".join(random.choices(string.ascii_letters + string.digits, k=length)) diff --git a/ml_utils/ml_utils_cli/cli/apps/datasets.py b/ml_utils/ml_utils_cli/cli/apps/datasets.py index 34650363..b5e6fd69 100644 --- a/ml_utils/ml_utils_cli/cli/apps/datasets.py +++ b/ml_utils/ml_utils_cli/cli/apps/datasets.py @@ -1,6 +1,7 @@ import json import random import shutil +import typing from pathlib import Path from typing import Annotated, Optional @@ -132,9 +133,9 @@ def export( Optional[str], typer.Option(help="Hugging Face Datasets repository ID to convert"), ] = None, - category_names: Annotated[ + label_names: Annotated[ Optional[str], - typer.Option(help="Category names to use, as a comma-separated list"), + typer.Option(help="Label names to use, as a comma-separated list"), ] = None, project_id: Annotated[ Optional[int], typer.Option(help="Label Studio Project ID") @@ -150,21 +151,33 @@ def export( help="if True, don't use HF images and download images from the server" ), ] = False, + train_ratio: Annotated[ + float, + typer.Option( + help="Train ratio for splitting the dataset, if the split name is not " + "provided (typically, if the source is Label Studio)" + ), + ] = 0.8, ): """Export Label Studio annotation, either to Hugging Face Datasets or local files (ultralytics format).""" from cli.export import ( export_from_hf_to_ultralytics, + export_from_ls_to_hf, export_from_ls_to_ultralytics, - export_to_hf, ) from label_studio_sdk.client import LabelStudio if (to == ExportDestination.hf or from_ == ExportSource.hf) and repo_id is None: raise typer.BadParameter("Repository ID is required for export/import with HF") - if to == ExportDestination.hf and category_names is None: - raise typer.BadParameter("Category names are required for HF export") + if label_names is None: + if to == ExportDestination.hf: + raise typer.BadParameter("Label names are required for HF export") + if from_ == ExportSource.ls: + raise typer.BadParameter( + "Label names are required for export from LS source" + ) if from_ == ExportSource.ls: if project_id is None: @@ -176,67 +189,29 @@ def export( raise typer.BadParameter("Output directory is required for Ultralytics export") if from_ == ExportSource.ls: + ls = LabelStudio(base_url=label_studio_url, api_key=api_key) + label_names = typing.cast(str, label_names) + label_names_list = label_names.split(",") if to == ExportDestination.hf: - ls = LabelStudio(base_url=label_studio_url, api_key=api_key) - category_names_list = category_names.split(",") - export_to_hf(ls, repo_id, category_names_list, project_id) + repo_id = typing.cast(str, repo_id) + export_from_ls_to_hf( + ls, repo_id, label_names_list, typing.cast(int, project_id) + ) elif to == ExportDestination.ultralytics: export_from_ls_to_ultralytics( - ls, output_dir, category_names_list, project_id + ls, + typing.cast(Path, output_dir), + label_names_list, + typing.cast(int, project_id), + train_ratio=train_ratio, ) elif from_ == ExportSource.hf: if to == ExportDestination.ultralytics: export_from_hf_to_ultralytics( - repo_id, output_dir, download_images=download_images + typing.cast(str, repo_id), + typing.cast(Path, output_dir), + download_images=download_images, ) else: raise typer.BadParameter("Unsupported export format") - - -@app.command() -def create_dataset_file( - input_file: Annotated[ - Path, - typer.Option(help="Path to a list of image URLs", exists=True), - ], - output_file: Annotated[ - Path, typer.Option(help="Path to the output JSON file", exists=False) - ], -): - """Create a Label Studio object detection dataset file from a list of - image URLs.""" - from urllib.parse import urlparse - - import tqdm - from cli.sample import format_object_detection_sample_to_ls - from openfoodfacts.images import extract_barcode_from_url, extract_source_from_url - from openfoodfacts.utils import get_image_from_url - - logger.info("Loading dataset: %s", input_file) - - with output_file.open("wt") as f: - for line in tqdm.tqdm(input_file.open("rt"), desc="images"): - url = line.strip() - if not url: - continue - - extra_meta = {} - image_id = Path(urlparse(url).path).stem - if ".openfoodfacts.org" in url: - barcode = extract_barcode_from_url(url) - extra_meta["barcode"] = barcode - off_image_id = Path(extract_source_from_url(url)).stem - extra_meta["off_image_id"] = off_image_id - image_id = f"{barcode}-{off_image_id}" - - image = get_image_from_url(url, error_raise=False) - - if image is None: - logger.warning("Failed to load image: %s", url) - continue - - label_studio_sample = format_object_detection_sample_to_ls( - image_id, url, image.width, image.height, extra_meta - ) - f.write(json.dumps(label_studio_sample) + "\n") diff --git a/ml_utils/ml_utils_cli/cli/apps/projects.py b/ml_utils/ml_utils_cli/cli/apps/projects.py index 2dd1f3ae..5a10381f 100644 --- a/ml_utils/ml_utils_cli/cli/apps/projects.py +++ b/ml_utils/ml_utils_cli/cli/apps/projects.py @@ -1,10 +1,17 @@ +import enum import json +import typing from pathlib import Path from typing import Annotated, Optional import typer from openfoodfacts.utils import get_logger +from PIL import Image +from ..annotate import ( + format_annotation_results_from_triton, + format_annotation_results_from_ultralytics, +) from ..config import LABEL_STUDIO_DEFAULT_URL app = typer.Typer() @@ -145,41 +152,188 @@ def annotate_from_prediction( ) +class PredictorBackend(enum.Enum): + triton = "triton" + ultralytics = "ultralytics" + + @app.command() def add_prediction( api_key: Annotated[str, typer.Option(envvar="LABEL_STUDIO_API_KEY")], project_id: Annotated[int, typer.Option(help="Label Studio Project ID")], model_name: Annotated[ - str, typer.Option(help="Name of the object detection model to run") - ], + str, + typer.Option( + help="Name of the object detection model to run (for Triton server) or " + "of the Ultralytics zero-shot model to run." + ), + ] = "yolov8x-worldv2.pt", triton_uri: Annotated[ - str, typer.Option(help="URI (host+port) of the Triton Inference Server") - ], + Optional[str], + typer.Option(help="URI (host+port) of the Triton Inference Server"), + ] = None, + backend: Annotated[ + PredictorBackend, + typer.Option( + help="Prediction backend: either use a Triton server to perform " + "the prediction or uses Ultralytics." + ), + ] = PredictorBackend.ultralytics, + labels: Annotated[ + Optional[list[str]], + typer.Option( + help="List of class labels to use for Yolo model. If you're using Yolo-World or other " + "zero-shot models, this is the list of label names that are going to be provided to the " + "model. In such case, you can use `label_mapping` to map the model's output to the " + "actual class names expected by Label Studio." + ), + ] = None, + label_mapping: Annotated[ + Optional[str], + typer.Option(help="Mapping of model labels to class names, as a JSON string"), + ] = None, label_studio_url: str = LABEL_STUDIO_DEFAULT_URL, - threshold: float = 0.5, + threshold: Annotated[ + Optional[float], + typer.Option( + help="Confidence threshold for selecting bounding boxes. The default is 0.5 " + "for Triton backend and 0.1 for Ultralytics backend." + ), + ] = None, + max_det: Annotated[int, typer.Option(help="Maximum numbers of detections")] = 300, + dry_run: Annotated[ + bool, + typer.Option( + help="Launch in dry run mode, without uploading annotations to Label Studio" + ), + ] = False, ): """Add predictions as pre-annotations to Label Studio tasks, for an object detection model running on Triton Inference Server.""" import tqdm - from cli.sample import format_annotation_results_from_triton from cli.triton.object_detection import ObjectDetectionModelRegistry from label_studio_sdk.client import LabelStudio from openfoodfacts.utils import get_image_from_url + label_mapping_dict = None + if label_mapping: + label_mapping_dict = json.loads(label_mapping) + + if dry_run: + logger.info("** Dry run mode enabled **") + + logger.info( + "backend: %s, model_name: %s, labels: %s, threshold: %s, label mapping: %s", + backend, + model_name, + labels, + threshold, + label_mapping, + ) ls = LabelStudio(base_url=label_studio_url, api_key=api_key) - model = ObjectDetectionModelRegistry.load(model_name) + + model: ObjectDetectionModelRegistry | "YOLO" + + if backend == PredictorBackend.ultralytics: + from ultralytics import YOLO + + if labels is None: + raise typer.BadParameter("Labels are required for Ultralytics backend") + + if threshold is None: + threshold = 0.1 + + model = YOLO(model_name) + model.set_classes(labels) + elif backend == PredictorBackend.triton: + if triton_uri is None: + raise typer.BadParameter("Triton URI is required for Triton backend") + + if threshold is None: + threshold = 0.5 + + model = ObjectDetectionModelRegistry.load(model_name) + else: + raise typer.BadParameter(f"Unsupported backend: {backend}") for task in tqdm.tqdm(ls.tasks.list(project=project_id), desc="tasks"): if task.total_predictions == 0: - image = get_image_from_url(task.data["image_url"], error_raise=True) - output = model.detect_from_image(image, triton_uri=triton_uri) - results = output.select(threshold=threshold) - logger.info("Adding prediction to task: %s", task.id) - label_studio_result = format_annotation_results_from_triton( - results, image.width, image.height + image_url = task.data["image_url"] + image = typing.cast( + Image.Image, + get_image_from_url(image_url, error_raise=True), ) - ls.predictions.create( - task=task.id, - result=label_studio_result, + if backend == PredictorBackend.ultralytics: + results = model.predict( + image, + conf=threshold, + max_det=max_det, + )[0] + labels = typing.cast(list[str], labels) + label_studio_result = format_annotation_results_from_ultralytics( + results, labels, label_mapping_dict + ) + else: + output = model.detect_from_image(image, triton_uri=triton_uri) + results = output.select(threshold=threshold) + logger.info("Adding prediction to task: %s", task.id) + label_studio_result = format_annotation_results_from_triton( + results, image.width, image.height + ) + if dry_run: + logger.info("image_url: %s", image_url) + logger.info("result: %s", label_studio_result) + else: + ls.predictions.create( + task=task.id, + result=label_studio_result, + ) + + +@app.command() +def create_dataset_file( + input_file: Annotated[ + Path, + typer.Option(help="Path to a list of image URLs", exists=True), + ], + output_file: Annotated[ + Path, typer.Option(help="Path to the output JSON file", exists=False) + ], +): + """Create a Label Studio object detection dataset file from a list of + image URLs.""" + from urllib.parse import urlparse + + import tqdm + from cli.sample import format_object_detection_sample_to_ls + from openfoodfacts.images import extract_barcode_from_url, extract_source_from_url + from openfoodfacts.utils import get_image_from_url + + logger.info("Loading dataset: %s", input_file) + + with output_file.open("wt") as f: + for line in tqdm.tqdm(input_file.open("rt"), desc="images"): + url = line.strip() + if not url: + continue + + extra_meta = {} + image_id = Path(urlparse(url).path).stem + if ".openfoodfacts.org" in url: + barcode = extract_barcode_from_url(url) + extra_meta["barcode"] = barcode + off_image_id = Path(extract_source_from_url(url)).stem + extra_meta["off_image_id"] = off_image_id + image_id = f"{barcode}-{off_image_id}" + + image = get_image_from_url(url, error_raise=False) + + if image is None: + logger.warning("Failed to load image: %s", url) + continue + + label_studio_sample = format_object_detection_sample_to_ls( + image_id, url, image.width, image.height, extra_meta ) + f.write(json.dumps(label_studio_sample) + "\n") diff --git a/ml_utils/ml_utils_cli/cli/export.py b/ml_utils/ml_utils_cli/cli/export.py index 3237c81c..59afb269 100644 --- a/ml_utils/ml_utils_cli/cli/export.py +++ b/ml_utils/ml_utils_cli/cli/export.py @@ -1,15 +1,17 @@ import functools import logging import pickle +import random import tempfile +import typing from pathlib import Path import datasets import tqdm +from cli.sample import HF_DS_FEATURES, format_object_detection_sample_to_hf from label_studio_sdk.client import LabelStudio from openfoodfacts.images import download_image - -from cli.sample import HF_DS_FEATURES, format_object_detection_sample_to_hf +from PIL import Image logger = logging.getLogger(__name__) @@ -21,7 +23,7 @@ def _pickle_sample_generator(dir: Path): yield pickle.load(f) -def export_to_hf( +def export_from_ls_to_hf( ls: LabelStudio, repo_id: str, category_names: list[str], @@ -61,6 +63,7 @@ def export_from_ls_to_ultralytics( output_dir: Path, category_names: list[str], project_id: int, + train_ratio: float = 0.8, ): """Export annotations from a Label Studio project to the Ultralytics format. @@ -72,18 +75,33 @@ def export_from_ls_to_ultralytics( data_dir = output_dir / "data" data_dir.mkdir(parents=True, exist_ok=True) + split_warning_displayed = False + # NOTE: before, all images were sent to val, the last split + label_dir = data_dir / "labels" + images_dir = data_dir / "images" for split in ["train", "val"]: - split_labels_dir = data_dir / "labels" / split - split_labels_dir.mkdir(parents=True, exist_ok=True) - split_images_dir = data_dir / "images" / split - split_images_dir.mkdir(parents=True, exist_ok=True) + (label_dir / split).mkdir(parents=True, exist_ok=True) + (images_dir / split).mkdir(parents=True, exist_ok=True) for task in tqdm.tqdm( ls.tasks.list(project=project_id, fields="all"), desc="tasks", ): - split = task.data["split"] + split = task.data.get("split") + + if split is None: + if not split_warning_displayed: + logger.warning( + "Split information not found, assigning randomly. " + "To avoid this, set the `split` field in the task data." + ) + split_warning_displayed = True + split = "train" if random.random() < train_ratio else "val" + + elif split not in ["train", "val"]: + raise ValueError("Invalid split name: %s", split) + if len(task.annotations) > 1: logger.warning("More than one annotation found, skipping") continue @@ -92,45 +110,66 @@ def export_from_ls_to_ultralytics( continue annotation = task.annotations[0] - image_id = task.data["image_id"] - - image_url = task.data["image_url"] - download_output = download_image(image_url, return_bytes=True) - if download_output is None: - logger.error("Failed to download image: %s", image_url) + if annotation["was_cancelled"] is True: + logger.debug("Annotation was cancelled, skipping") continue - _, image_bytes = download_output + if "image_id" not in task.data: + raise ValueError( + "`image_id` field not found in task data. " + "Make sure the task data contains the `image_id` " + "field, which should be a unique identifier for the image." + ) + if "image_url" not in task.data: + raise ValueError( + "`image_url` field not found in task data. " + "Make sure the task data contains the `image_url` " + "field, which should be the URL of the image." + ) + image_id = task.data["image_id"] + image_url = task.data["image_url"] - with (split_images_dir / f"{image_id}.jpg").open("wb") as f: - f.write(image_bytes) + has_valid_annotation = False + with (label_dir / split / f"{image_id}.txt").open("w") as f: + if not any( + annotation_result["type"] == "rectanglelabels" + for annotation_result in annotation["result"] + ): + continue - with (split_labels_dir / f"{image_id}.txt").open("w") as f: for annotation_result in annotation["result"]: - if annotation_result["type"] != "rectanglelabels": - raise ValueError( - "Invalid annotation type: %s" % annotation_result["type"] - ) - - value = annotation_result["value"] - x_min = value["x"] / 100 - y_min = value["y"] / 100 - width = value["width"] / 100 - height = value["height"] / 100 - category_name = value["rectanglelabels"][0] - category_id = category_names.index(category_name) - - # Save the labels in the Ultralytics format: - # - one label per line - # - each line is a list of 5 elements: - # - category_id - # - x_center - # - y_center - # - width - # - height - x_center = x_min + width / 2 - y_center = y_min + height / 2 - f.write(f"{category_id} {x_center} {y_center} {width} {height}\n") + if annotation_result["type"] == "rectanglelabels": + value = annotation_result["value"] + x_min = value["x"] / 100 + y_min = value["y"] / 100 + width = value["width"] / 100 + height = value["height"] / 100 + category_name = value["rectanglelabels"][0] + category_id = category_names.index(category_name) + + # Save the labels in the Ultralytics format: + # - one label per line + # - each line is a list of 5 elements: + # - category_id + # - x_center + # - y_center + # - width + # - height + x_center = x_min + width / 2 + y_center = y_min + height / 2 + f.write(f"{category_id} {x_center} {y_center} {width} {height}\n") + has_valid_annotation = True + + if has_valid_annotation: + download_output = download_image(image_url, return_bytes=True) + if download_output is None: + logger.error("Failed to download image: %s", image_url) + continue + + _, image_bytes = typing.cast(tuple[Image.Image, bytes], download_output) + + with (images_dir / split / f"{image_id}.jpg").open("wb") as f: + f.write(image_bytes) with (output_dir / "data.yaml").open("w") as f: f.write("path: data\n") diff --git a/ml_utils/ml_utils_cli/cli/sample.py b/ml_utils/ml_utils_cli/cli/sample.py index bbea020f..62901af6 100644 --- a/ml_utils/ml_utils_cli/cli/sample.py +++ b/ml_utils/ml_utils_cli/cli/sample.py @@ -5,8 +5,6 @@ import datasets from openfoodfacts.images import download_image -from cli.triton.object_detection import ObjectDetectionResult - logger = logging.getLogger(__name__) @@ -56,51 +54,6 @@ def format_annotation_results_from_hf( return annotation_results -def format_annotation_results_from_triton( - objects: list[ObjectDetectionResult], image_width: int, image_height: int -): - """Format annotation results from a Triton object detection model into - Label Studio format.""" - annotation_results = [] - for object_ in objects: - bbox = object_.bounding_box - category_name = object_.label - # These are relative coordinates (between 0.0 and 1.0) - y_min, x_min, y_max, x_max = bbox - # Make sure the coordinates are within the image boundaries, - # and convert them to percentages - y_min = min(max(0, y_min), 1.0) * 100 - x_min = min(max(0, x_min), 1.0) * 100 - y_max = min(max(0, y_max), 1.0) * 100 - x_max = min(max(0, x_max), 1.0) * 100 - x = x_min - y = y_min - width = x_max - x_min - height = y_max - y_min - - id_ = "".join(random.choices(string.ascii_letters + string.digits, k=10)) - annotation_results.append( - { - "id": id_, - "type": "rectanglelabels", - "from_name": "label", - "to_name": "image", - "original_width": image_width, - "original_height": image_height, - "image_rotation": 0, - "value": { - "rotation": 0, - "x": x, - "y": y, - "width": width, - "height": height, - "rectanglelabels": [category_name], - }, - }, - ) - return annotation_results - - def format_object_detection_sample_from_hf(hf_sample: dict, split: str) -> dict: hf_meta = hf_sample["meta"] objects = hf_sample["objects"] diff --git a/ml_utils/ml_utils_cli/config_files/product-detection.xml b/ml_utils/ml_utils_cli/config_files/product-detection.xml new file mode 100644 index 00000000..edb31a40 --- /dev/null +++ b/ml_utils/ml_utils_cli/config_files/product-detection.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/ml_utils/ml_utils_cli/main.py b/ml_utils/ml_utils_cli/main.py index f479a721..9cbe568d 100644 --- a/ml_utils/ml_utils_cli/main.py +++ b/ml_utils/ml_utils_cli/main.py @@ -7,7 +7,7 @@ from cli.config import LABEL_STUDIO_DEFAULT_URL from openfoodfacts.utils import get_logger -app = typer.Typer() +app = typer.Typer(pretty_exceptions_show_locals=False) logger = get_logger() diff --git a/ml_utils/ml_utils_cli/requirements.txt b/ml_utils/ml_utils_cli/requirements.txt index 845bf080..8330ab76 100644 --- a/ml_utils/ml_utils_cli/requirements.txt +++ b/ml_utils/ml_utils_cli/requirements.txt @@ -5,4 +5,5 @@ more_itertools==10.3.0 datasets==2.20.0 tritonclient==2.46.0 imagehash==4.3.1 -protobuf==5.28.3 \ No newline at end of file +protobuf==5.28.3 +ultralytics diff --git a/nutrition-detector/dataset-generation/2_create_project.py b/nutrition-detector/dataset-generation/2_create_project.py index bec9c4b7..c438111a 100644 --- a/nutrition-detector/dataset-generation/2_create_project.py +++ b/nutrition-detector/dataset-generation/2_create_project.py @@ -11,14 +11,16 @@ def create_project( - api_key: Annotated[str, typer.Argument(envvar="LABEL_STUDIO_API_KEY")] + api_key: Annotated[str, typer.Argument(envvar="LABEL_STUDIO_API_KEY")], + label_config_path: Path = typer.Argument( + file_okay=True, dir_okay=False, exists=True + ), + title: str = typer.Option(help="Project title"), ): ls = LabelStudio(base_url=LABEL_STUDIO_URL, api_key=api_key) - label_config = Path("./label_config.xml").read_text() + label_config = Path(label_config_path).read_text() - project = ls.projects.create( - title="Nutrition table token annotation", label_config=label_config - ) + project = ls.projects.create(title=title, label_config=label_config) logger.info(f"Project created: {project}") diff --git a/nutrition-detector/dataset-generation/8_update_errors.py b/nutrition-detector/dataset-generation/8_update_errors.py new file mode 100644 index 00000000..7839843d --- /dev/null +++ b/nutrition-detector/dataset-generation/8_update_errors.py @@ -0,0 +1,30 @@ +from typing import Annotated + +import tqdm +import typer +from label_studio_sdk import Client +from openfoodfacts.utils import get_logger + +logger = get_logger(level="DEBUG") + +LABEL_STUDIO_URL = "https://annotate.openfoodfacts.org" + + +def update_checked_field( + api_key: Annotated[str, typer.Argument(envvar="LABEL_STUDIO_API_KEY")], + project_id: int = 42, + view_id: int = 62, +): + ls = Client(url=LABEL_STUDIO_URL, api_key=api_key) + ls.check_connection() + + project = ls.get_project(project_id) + tasks = project.get_tasks(view_id=view_id) + logger.info(f"Found {len(tasks)} tasks with errors in the project") + for task in tqdm.tqdm(tasks, desc="tasks"): + data = task["data"] + project.update_task(task["id"], data={**data}) + + +if __name__ == "__main__": + typer.run(update_checked_field) diff --git a/nutrition-detector/dataset-generation/9_add_checked_field.py b/nutrition-detector/dataset-generation/9_add_checked_field.py new file mode 100644 index 00000000..051ff71d --- /dev/null +++ b/nutrition-detector/dataset-generation/9_add_checked_field.py @@ -0,0 +1,37 @@ +from typing import Annotated + +import typer +from label_studio_sdk import Task +from label_studio_sdk.client import LabelStudio +from openfoodfacts.utils import get_logger + +logger = get_logger(level="DEBUG") + +LABEL_STUDIO_URL = "https://annotate.openfoodfacts.org" + + +def add_checked_field( + api_key: Annotated[str, typer.Option(envvar="LABEL_STUDIO_API_KEY")], + view_id: Annotated[int, typer.Option(help="Label Studio view ID")], + project_id: Annotated[int, typer.Option(help="Label Studio project ID")], + label_studio_url: str = LABEL_STUDIO_DEFAULT_URL, +): + + ls = LabelStudio(base_url=label_studio_url, api_key=api_key) + + task: Task + for task in ls.tasks.list(project=project_id, fields="all", view=view_id): + if task.annotations and "checked" not in task.data: + last_annotation_results = task.annotations[-1]["result"] + for annotation_result in last_annotation_results: + if ( + annotation_result["type"] == "choices" + and "checked" in annotation_result["value"]["choices"] + ): + logger.info(f"Updating task {task['id']} with checked field") + ls.tasks.update(task.id, data={**task.data, "checked": True}) + break + + +if __name__ == "__main__": + typer.run(add_checked_field) diff --git a/object_detection/crop_detection/Makefile b/object_detection/crop_detection/Makefile new file mode 100644 index 00000000..20ff3eef --- /dev/null +++ b/object_detection/crop_detection/Makefile @@ -0,0 +1,22 @@ +MODEL_URL = "https://huggingface.co/openfoodfacts/crop-detection/resolve/main/weights/best_saved_model/best_float16.tflite?download=true" +MODEL_PATH = models/yolov8n_float16.tflite + +.PHONY: * + +init: hello install load-model + +hello: + @echo "🍋Welcome to the Crop Detection project.🍋" + +install: + @echo "Install dependencies." + pip install -r requirements.txt + +load-model: + @echo "Load model from the HF repository 🤗: https://huggingface.co/openfoodfacts/crop-detection" + @if [ ! -f "${MODEL_PATH}" ]; then \ + echo "Model not found. Downloading from HF repository 🤗..."; \ + wget -O "${MODEL_PATH}" "${MODEL_URL}" ; \ + else \ + echo "Model already exists in models/"; \ + fi \ No newline at end of file diff --git a/object_detection/crop_detection/README.md b/object_detection/crop_detection/README.md new file mode 100644 index 00000000..554d7138 --- /dev/null +++ b/object_detection/crop_detection/README.md @@ -0,0 +1,114 @@ +# :lemon: Crop detection :lemon: + + +When contributors use the mobile app, they are asked to take pictures of the product, then to crop it. But this stage is +fastidious, especially when contributors need to add several pictures of the same product. + +To assist users during the process, we create a crop-detection model desin to detect the product edges. We fine-tuned **Yolov8n** on images extracted from the Open Food Facts database. + + +

+ Image 1 + Image 2 +

+ +*Product image before and after automatic cropping.* + +## Dev +You shall generate a new environment before installing new dependencies. Using Conda: + +```bash +conda create -n crop-detection python=3.11 +``` + +Then, prepare your local environment with the following command: + +```bash +make init +``` + +If you just want to load the model, use: + +```bash +make load-model +``` + +This command load the float16.tflite version of the Yolov8n from the [Crop-Detection repository](https://huggingface.co/openfoodfacts/crop-detection) on HuggingFace. + + +## Run crop-detection + +We use Tensorflow Lite to perform the crop-detection inference on image. After `make init`, you can use the CLI to run the model on your computer: + +```bash +python -m cli --help +``` + +## Model training + +### Data pipeline + +To train Yolov8, we extracted product images from the Open Food Facts AWS Bucket. This solution enables us to download a large batch of images without the complexity of using the OFF API, mainly due to the number of requests limit. + +To understand how to reproduce the images extraction, check the Product Opener [documentation](https://openfoodfacts.github.io/openfoodfacts-server/api/aws-images-dataset/), you'll find a code snippet that was actually used to download a batch of images. + +However, all images are not equal for our use case. We're seeking for images of products that needs cropping, whereas most of images in the database are already cropped... + +Therefore, we filtered the images on 2 criteria: + +* The image editor shouldn't be **Yuka** +* We pick images before 2020. + +We used DuckDB coupled with the JSONL dump to filtered codes respecting these 2 criteria. We generate a `.txt` file to store all product barcodes corresponding to our search. + +```sql +CREATE TABLE object_detection AS +SELECT code, last_image_dates_tags, correctors_tags +FROM read_ndjson('openfoodfacts-products.jsonl.gz') +; + +COPY( + SELECT code + FROM object_detection + WHERE (last_image_dates_tags[-1]::integer) < '2020' + AND list_aggregate(correctors_tags, 'string_agg', '|') NOT LIKE '%yuka%' + ) TO 'best_image_codes.txt' (DELIMITER ' ', HEADER FALSE) +; +``` + +We then generate the set of images using the command `load_images_from_aws.sh`. + +### Annotation on Label-Studio + +We used Label-Studio for the annotation. You can find the annotated images at https://annotate.openfoodfacts.org/projects/50/data. + +We also pre-annotated the images using [Yolo-World](https://huggingface.co/spaces/stevengrove/YOLO-World), an object detection model using custom labels. + +You'll find the code to pre-annotate, upload and download the data in `ml_utils/ml_utils_cli/cli`. + +### Training + +The model training was done using the Ultralytics library. Learn more by check the [official documentation](https://docs.ultralytics.com/modes/train/). We used Lightning AI to run the training job using GPUs (L4) + +```bash +yolo detect train \ + data=data/data.yaml \ + model=models/yolov8n.pt \ + epochs=200 \ + imgsz=640 \ + batch=64 +``` + +### Export to TFLite + +To export is as easy as the training with Ultralytics: + +```bash +yolo export model=weights/best.pt format=tflite +``` + +## Links + +* Demo: https://huggingface.co/spaces/openfoodfacts/crop-detection +* Model repo: https://huggingface.co/openfoodfacts/crop-detection +* Label Studio: https://annotate.openfoodfacts.org/projects/50/data \ No newline at end of file diff --git a/object_detection/crop_detection/assets/cropped.jpg b/object_detection/crop_detection/assets/cropped.jpg new file mode 100644 index 00000000..ec5e5a26 Binary files /dev/null and b/object_detection/crop_detection/assets/cropped.jpg differ diff --git a/object_detection/crop_detection/assets/product.jpg b/object_detection/crop_detection/assets/product.jpg new file mode 100644 index 00000000..85e06914 Binary files /dev/null and b/object_detection/crop_detection/assets/product.jpg differ diff --git a/object_detection/crop_detection/best_images/.gitkeep b/object_detection/crop_detection/best_images/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/object_detection/crop_detection/cli/__main__.py b/object_detection/crop_detection/cli/__main__.py new file mode 100644 index 00000000..a815e62f --- /dev/null +++ b/object_detection/crop_detection/cli/__main__.py @@ -0,0 +1,4 @@ +from .inference_yolo_tflite import main + +if __name__ == "__main__": + main() diff --git a/object_detection/crop_detection/cli/inference_yolo_tflite.py b/object_detection/crop_detection/cli/inference_yolo_tflite.py new file mode 100644 index 00000000..8dd6ade4 --- /dev/null +++ b/object_detection/crop_detection/cli/inference_yolo_tflite.py @@ -0,0 +1,203 @@ +from typing import Tuple, Dict, Optional +import argparse +from PIL import Image +import logging + +import numpy as np +from cv2 import dnn +import tflite_runtime.interpreter as tflite + + +MODEL_PATH = "models/yolov8n_float16.tflite" +DETECTION_THRESHOLD = 0.8 +NMS_THRESHOLD = 0.45 + +LOGGER = logging.getLogger(__name__) +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) + + +def parse(): + parser = argparse.ArgumentParser( + description="""Detect product boundary box. + Return a dictionnary with the score and the box relative coordinates (between [0, 1]). + If --save-path is indicated, save the cropped image to the specified path. + """, + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--image-path", + type=str, + required=True, + help="Path to the image to be processed.", + ) + parser.add_argument( + "--model-path", + type=str, + default=MODEL_PATH, + help="Path to the .tflite model." + ) + parser.add_argument( + "--threshold", + type=float, + default=DETECTION_THRESHOLD, + help="Detection score threshold.", + ) + parser.add_argument( + "--nms-threshold", + type=float, + default=NMS_THRESHOLD, + help="Non-Maximum Suppression threshold." + ) + parser.add_argument( + "--debug", + action=argparse.BooleanOptionalAction, + default=None, + help="Set debug mode.", + ) + parser.add_argument( + "--save-path", + type=str, + required=False, + help="Path to save the cropped image." + ) + return parser.parse_args() + + +def main(): + args = parse() + if args.debug: + LOGGER.setLevel(logging.DEBUG) + detected_object = run_inference( + image_path=args.image_path, + threshold=args.threshold, + nms_threshold=args.nms_threshold, + interpreter=tflite.Interpreter(model_path=args.model_path), + ) + print(detected_object) + if args.save_path: + LOGGER.info(f"Saving cropped image to {args.save_path}") + save_detected_object( + image_path=args.image_path, + detected_object=detected_object, + output_path=args.save_path, + ) + + +def _preprocess_image(image: Image, input_shape: Tuple[int, int]) -> np.ndarray: + """Preprocess the image to match the model input size.""" + max_size = max(image.size) + squared_image = Image.new("RGB", (max_size, max_size), color="black") + squared_image.paste(image, (0, 0)) + resized_squared_image = squared_image.resize(input_shape) + # Normalize the pixel values (scale between 0 and 1 if needed) + normalized_image = np.array(resized_squared_image) / 255.0 + # Expand dimensions to match the model input shape [1, height, width, 3] + input_tensor = np.expand_dims(normalized_image, axis=0).astype(np.float32) + return input_tensor + + +def run_inference( + image_path: str, + threshold: float, + nms_threshold: float, + interpreter: tflite.Interpreter, +) -> Dict: + # Init Tensorflow Lite + interpreter.allocate_tensors() + input_details = interpreter.get_input_details() + output_details = interpreter.get_output_details() + LOGGER.debug(f"Input details: {input_details}") + LOGGER.debug(f"Output details: {output_details}") + + # Get shape required by the model + input_shape = input_details[0]["shape"][1:3].tolist() + + image = Image.open(image_path) + width, height = image.size + + # We keep image ratio after resizing to 640 + image_ratio = width / height + if image_ratio > 1: + scale_x = input_shape[0] + scale_y = input_shape[1] / image_ratio + else: + scale_x = input_shape[0] * image_ratio + scale_y = input_shape[1] + + # Prepare image for Yolo + input_tensor = _preprocess_image(image, input_shape) + assert list(input_tensor.shape) == list(input_details[0]["shape"]) + LOGGER.debug(f"Input tensor shape: {input_tensor.shape}") + + # Inference + interpreter.set_tensor(input_details[0]["index"], input_tensor) + interpreter.invoke() + output_tensor = interpreter.get_tensor(output_details[0]["index"]) + LOGGER.debug(f"Output tensor shape: {output_tensor.shape}") + result_tensor = np.squeeze(output_tensor, axis=0).T + + # Post-process the result + boxes = result_tensor[:, :4] + scores = result_tensor[:, 4] + detected_objects = [] + for i, score in enumerate(scores): + if score > threshold: + # YOLOv8 typically outputs (cx, cy, w, h) normalized in the range [0, 1] + cx, cy, w, h = boxes[i] + + # Convert to corner coordinates + xmin = (cx - w / 2) * input_shape[0] + ymin = (cy - h / 2) * input_shape[1] + xmax = (cx + w / 2) * input_shape[0] + ymax = (cy + h / 2) * input_shape[1] + + # Bounding box coordinates are normalized to the input image size + scaled_xmin = max(0.0, min(1.0, xmin / scale_x)) + scaled_ymin = max(0.0, min(1.0, ymin / scale_y)) + scaled_xmax = max(0.0, min(1.0, xmax / scale_x)) + scaled_ymax = max(0.0, min(1.0, ymax / scale_y)) + + detected_objects.append( + { + "score": scores[i], + "box": [scaled_xmin, scaled_ymin, scaled_xmax, scaled_ymax], + } + ) + + # Apply Non-Maximum Suppression to overlapping boxes + raw_detected_boxes = [ + detected_object["box"] for detected_object in detected_objects + ] + raw_detected_scores = [ + detected_object["score"] for detected_object in detected_objects + ] + nms_indices = dnn.NMSBoxes( + bboxes=raw_detected_boxes, + scores=raw_detected_scores, + score_threshold=threshold, + nms_threshold=nms_threshold, + ) + # We only look for one box per image + detected_object = detected_objects[nms_indices[0]] + + return detected_object + + +def save_detected_object( + image_path: str, detected_object: Dict, output_path: Optional[str] = None +) -> None: + if not output_path: + print(output_path) + raise ValueError("Output path is required.") + image = Image.open(image_path) + width, height = image.size + xmin, ymin, xmax, ymax = detected_object["box"] + xmin = int(xmin * width) + ymin = int(ymin * height) + xmax = int(xmax * width) + ymax = int(ymax * height) + image = image.crop((xmin, ymin, xmax, ymax)) + image.save(output_path) diff --git a/object_detection/crop_detection/commands/load_images_from_aws.sh b/object_detection/crop_detection/commands/load_images_from_aws.sh new file mode 100755 index 00000000..d9c73c67 --- /dev/null +++ b/object_detection/crop_detection/commands/load_images_from_aws.sh @@ -0,0 +1,41 @@ +# Number of images to download +n=1000 +# Directory to store images +folder="images" +# Codes of filtered images generated with DuckDB +codes="best_image_codes.txt" +# Image urls +urls_file="urls.txt" + +# Base URLS +bucket_url="https://openfoodfacts-images.s3.eu-west-3.amazonaws.com/" +off_urls="https://images.openfoodfacts.org/images/products/" + +# Create the folder to store images and URLs +mkdir -p "$folder" +touch "$folder/$urls_file" + +# Pre-filter the data once to avoid repeated decompression +zcat data_keys.gz | grep -v ".400.jpg" | grep "1.jpg" > temp_data_keys.txt + +# Process each code from best_image_codes.txt +shuf -n "$n" "$codes" | while read -r code; do + # Format the code into the required pattern 260/012/901/5091 + formatted_code=$(echo "$code" | sed 's|^\(...\)\(...\)\(...\)\(.*\)$|\1\/\2\/\3\/\4|') + + # Search for matching entries in data_keys.gz and pick one random image + grep "$formatted_code" temp_data_keys.txt | shuf -n 1 | while read -r url; do + # Construct the filename by stripping the bucket URL and formatting + filename=$(echo "$url" | sed "s|$bucket_url||" | tr '/' '_' | sed 's|data_||') + + # Download the image using wget + wget -O "$folder/$filename" "$bucket_url$url" + + # Add image url + image_url=$(echo "$url" | sed 's|data/||') + echo "$off_urls$image_url" >> "$folder/$urls_file" + done +done + +# Clean +rm temp_filtered_data.txt diff --git a/object_detection/crop_detection/data/.gitkeep b/object_detection/crop_detection/data/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/object_detection/crop_detection/images/.gitkeep b/object_detection/crop_detection/images/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/object_detection/crop_detection/models/.gitkeep b/object_detection/crop_detection/models/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/object_detection/crop_detection/requirements.txt b/object_detection/crop_detection/requirements.txt new file mode 100644 index 00000000..3f934a7d --- /dev/null +++ b/object_detection/crop_detection/requirements.txt @@ -0,0 +1,4 @@ +label-studio-sdk==1.0.2 +tflite-runtime==2.14.0 +typer==0.12.3 +ultralytics==8.2.94