Add filters in DID search

.github/actions/setup/action.yml

@@ -2,35 +2,6 @@ name: "Setup"
 runs:
   using: "composite"
   steps:
-    - name: Install node
-      uses: actions/setup-node@v4
-      with:
-        node-version: '18.x'
-    - name: Install Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.11'
-        architecture: 'x64'
-    - name: Setup pip cache
-      uses: actions/cache@v2
-      with:
-        path: ~/.cache/pip
-        key: pip-3.11-${{ hashFiles('package.json') }}
-        restore-keys: |
-          pip-3.11-
-          pip-
-    - name: Get yarn cache directory path
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-      shell: bash
-    - name: Setup yarn cache
-      uses: actions/cache@v2
-      id: yarn-cache
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          yarn-
     - name: Install Python dependencies
       run: python -m pip install -r requirements.txt
       shell: bash

.github/actions/test/action.yml

@@ -6,7 +6,7 @@ runs:
       run: jlpm run eslint:check
       shell: bash
     - name: Run Jest
-      run: jlpm jest
+      run: jlpm jest -u
       shell: bash
     - name: Run Pytest
       run: pytest rucio_jupyterlab/tests/

.github/workflows/build-and-publish-tagged.yml

@@ -5,24 +5,28 @@ on:
     tags: 'v*'
 
 jobs:
-  build:
-    name: Build and Publish to PyPI
+
+  Test:
+    uses: ./.github/workflows/test.yml
+
+  Build:
+    needs: [Test]
     runs-on: ubuntu-latest
-    environment:
-      name: pypi
-      url: https://pypi.org/p/rucio-jupyterlab
-    permissions:
-      id-token: write
     steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup
-      - uses: ./.github/actions/test
-      - uses: ./.github/actions/build-ext
-      - uses: ./.github/actions/post-test
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Base Setup
+        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+
+      - name: Custom Setup
+        uses: ./.github/actions/setup
+
       - name: Build sdist
         run: |
           pip install build
           python -m build --sdist
+
       - name: Publish distribution to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:

.github/workflows/test.yml

@@ -8,12 +8,34 @@ on:
   workflow_dispatch:
 
 jobs:
-  build:
-    runs-on: ubuntu-latest
+  Test:
+    name: Test
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.11", "3.12"]
+        node-version: ["18.x", "20.x"]
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup
-      - uses: ./.github/actions/test
-      - uses: ./.github/actions/build-ext
-      - uses: ./.github/actions/post-test
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Base Setup
+        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+
+      - name: Setup firefox
+        uses: browser-actions/setup-firefox@latest  
+
+      - name: Custom Setup
+        uses: ./.github/actions/setup
+
+      - name: Test
+        uses: ./.github/actions/test
+
+      - name: Build
+        uses: ./.github/actions/build-ext
+
+      - name: Post Test
+        env:
+            JLAB_BROWSER_TYPE: firefox
+        uses: ./.github/actions/post-test
 

.gitignore

@@ -123,3 +123,7 @@ dmypy.json
 
 # Yarn cache
 .yarn/
+
+# jest files
+src/__tests__/__snapshots__/
+junit.xml

jest.config.js

@@ -23,6 +23,6 @@ module.exports = {
     '!src/**/.ipynb_checkpoints/*'
   ],
   coverageReporters: ['lcov', 'text'],
-  testRegex: 'src/.*/.*.spec.ts[x]?$',
+  testRegex: 'src/\_\_tests\_\_/.*\.tsx?$',
   transformIgnorePatterns: [`/node_modules/(?!${esModules}).+`]
-};
+};

package.json

@@ -102,6 +102,7 @@
     "@types/mocha": "^10.0.6",
     "@types/react": "^18.0.26",
     "@types/react-addons-linked-state-mixin": "^0.14.22",
+    "@types/react-test-renderer": "^18.3.0",
     "@typescript-eslint/eslint-plugin": "^7.2.0",
     "@typescript-eslint/parser": "^7.2.0",
     "css-loader": "^6.7.1",
@@ -113,6 +114,7 @@
     "mkdirp": "^3.0.1",
     "npm-run-all": "^4.1.5",
     "prettier": "^3.0.0",
+    "react-test-renderer": "^18.3.1",
     "rimraf": "^5.0.1",
     "source-map-loader": "^1.0.2",
     "style-loader": "^3.3.1",

rucio_jupyterlab/_version.py

@@ -1,4 +1,4 @@
 # This file is auto-generated by Hatchling. As such, do not:
 #   - modify
 #   - track in version control e.g. be sure to add to .gitignore
-__version__ = VERSION = '0.10.0'
+__version__ = VERSION = '1.0.0'

rucio_jupyterlab/handlers/did_search.py

@@ -28,14 +28,20 @@ def __init__(self, namespace, rucio):
         self.rucio = rucio
         self.db = get_db()  # pylint: disable=invalid-name
 
-    def search_did(self, scope, name, search_type, limit):
+    def search_did(self, scope, name, search_type, filters, limit):
         wildcard_enabled = self.rucio.instance_config.get('wildcard_enabled', False)
 
         if ('*' in name or '%' in name) and not wildcard_enabled:
             raise WildcardDisallowedException()
 
-        dids = self.rucio.search_did(scope, name, search_type, limit)
+        dids = self.rucio.search_did(scope, name, search_type, filters, limit)
 
+        for did in dids:
+            if did['did_type'] is None:  # JSON plugin was used lacking data
+                metadata = self.rucio.get_metadata(scope, did['name'])[0]
+                did['did_type'] = f"DIDType.{metadata['did_type']}"
+                did['bytes'] = metadata['bytes']
+                did['length'] = metadata['length']
 
         def mapper(entry, _):
             return {
@@ -54,13 +60,14 @@ def get(self):
         namespace = self.get_query_argument('namespace')
         search_type = self.get_query_argument('type', 'collection')
         did = self.get_query_argument('did')
+        filters = self.get_query_argument('filters', default=None)
         rucio = self.rucio.for_instance(namespace)
 
         (scope, name) = did.split(':')
         handler = DIDSearchHandlerImpl(namespace, rucio)
 
         try:
-            dids = handler.search_did(scope, name, search_type, ROW_LIMIT)
+            dids = handler.search_did(scope, name, search_type, filters, ROW_LIMIT)
             self.finish(json.dumps(dids))
         except RucioAuthenticationException:
             self.set_status(401)

rucio_jupyterlab/rucio/rucio.py

@@ -8,6 +8,7 @@
 # - Muhammad Aditya Hilmy, <[email protected]>, 2020
 
 import logging
+import re
 import time
 import json
 from urllib.parse import urlencode, quote
@@ -16,6 +17,129 @@
 from .authenticators import RucioAuthenticationException, authenticate_userpass, authenticate_x509, authenticate_oidc
 
 
+def parse_did_filter_from_string_fe(input_string, name='*', type='collection', omit_name=False):
+    """
+    Parse DID filter string for the filter engine (fe).
+
+    Should adhere to the following conventions:
+    - ';' represents the logical OR operator
+    - ',' represents the logical AND operator
+    - all operators belong to set of (<=, >=, ==, !=, >, <, =)
+    - there should be no duplicate key+operator criteria.
+
+    One sided and compound inequalities are supported.
+
+    Sanity checking of input is left to the filter engine.
+
+    :param input_string: String containing the filter options.
+    :param name: DID name.
+    :param type: The type of the did: all(container, dataset, file), collection(dataset or container), dataset, container.
+    :param omit_name: omit addition of name to filters.
+    :return: list of dictionaries with each dictionary as a separate OR expression.
+    """
+    # lookup table unifying all comprehended operators to a nominal suffix.
+    # note that the order matters as the regex engine is eager, e.g. don't want to evaluate '<=' as '<' and '='.
+    operators_suffix_LUT = dict({
+        '≤': 'lte',
+        '≥': 'gte',
+        '==': '',
+        '≠': 'ne',
+        '>': 'gt',
+        '<': 'lt',
+        '=': ''
+    })
+
+    # lookup table mapping operator opposites, used to reverse compound inequalities.
+    operator_opposites_LUT = {
+        'lt': 'gt',
+        'lte': 'gte'
+    }
+    operator_opposites_LUT.update({op2: op1 for op1, op2 in operator_opposites_LUT.items()})
+
+    filters = []
+    if input_string:
+        or_groups = list(filter(None, input_string.split(';')))     # split <input_string> into OR clauses
+        for or_group in or_groups:
+            or_group = or_group.strip()
+            and_groups = list(filter(None, or_group.split(',')))    # split <or_group> into AND clauses
+            and_group_filters = {}
+            for and_group in and_groups:
+                and_group = and_group.strip()
+                # tokenise this AND clause using operators as delimiters.
+                tokenisation_regex = "({})".format('|'.join(operators_suffix_LUT.keys()))
+                and_group_split_by_operator = list(filter(None, re.split(tokenisation_regex, and_group)))
+                if len(and_group_split_by_operator) == 3:       # this is a one-sided inequality or expression
+                    key, operator, value = [token.strip() for token in and_group_split_by_operator]
+
+                    # substitute input operator with the nominal operator defined by the LUT, <operators_suffix_LUT>.
+                    operator_mapped = operators_suffix_LUT.get(operator)
+
+                    filter_key_full = key = "'{}'".format(key)
+                    if operator_mapped is not None:
+                        if operator_mapped:
+                            filter_key_full = "{}.{}".format(key, operator_mapped)
+                    else:
+                        raise ValueError("{} operator not understood.".format(operator_mapped))
+
+                    if filter_key_full in and_group_filters:
+                        raise ValueError(filter_key_full)
+                    else:
+                        if not is_numeric(value):
+                            value = "'{}'".format(value)
+                        and_group_filters[filter_key_full] = value
+                elif len(and_group_split_by_operator) == 5:     # this is a compound inequality
+                    value1, operator1, key, operator2, value2 = [token.strip() for token in and_group_split_by_operator]
+
+                    # substitute input operator with the nominal operator defined by the LUT, <operators_suffix_LUT>.
+                    operator1_mapped = operator_opposites_LUT.get(operators_suffix_LUT.get(operator1))
+                    operator2_mapped = operators_suffix_LUT.get(operator2)
+
+                    key = "'{}'".format(key)
+                    filter_key1_full = filter_key2_full = key
+                    if operator1_mapped is not None and operator2_mapped is not None:
+                        if operator1_mapped:    # ignore '' operator (maps from equals)
+                            filter_key1_full = "{}.{}".format(key, operator1_mapped)
+                        if operator2_mapped:    # ignore '' operator (maps from equals)
+                            filter_key2_full = "{}.{}".format(key, operator2_mapped)
+                    else:
+                        raise ValueError("{} operator not understood.".format(operator_mapped))
+
+                    if filter_key1_full in and_group_filters:
+                        raise ValueError(filter_key1_full)
+                    else:
+                        if not is_numeric(value1):
+                            value1 = "'{}'".format(value1)
+                        and_group_filters[filter_key1_full] = value1
+                    if filter_key2_full in and_group_filters:
+                        raise ValueError(filter_key2_full)
+                    else:
+                        if not is_numeric(value2):
+                            value2 = "'{}'".format(value2)
+                        and_group_filters[filter_key2_full] = value2
+                else:
+                    raise ValueError(and_group)
+
+            # add name key to each AND clause if it hasn't already been populated from the filter and <omit_name> not set.
+            if not omit_name and 'name' not in and_group_filters:
+                and_group_filters['name'] = name
+
+            filters.append(and_group_filters)
+    else:
+        if not omit_name:
+            filters.append({
+                'name': name
+            })
+    return filters, type
+
+
+def is_numeric(value):
+    try:
+        float(value)
+        return True
+    except ValueError:
+        return False
+
+
 class RucioAPI:
     rucio_auth_token_cache = dict()
 
@@ -55,16 +179,20 @@ def get_rses(self, rse_expression=None):
 
         return results
 
-    def search_did(self, scope, name, search_type='collection', limit=None):
+    def search_did(self, scope, name, search_type='collection', filters=None, limit=None):
         token = self._get_auth_token()
         headers = {'X-Rucio-Auth-Token': token}
 
         scope = quote(scope)
-        urlencoded_params = urlencode({
+        params = {
             'type': search_type,
             'long': '1',
             'name': name
-        })
+        }
+        if filters:
+            filters, _ = parse_did_filter_from_string_fe(filters, name=name)
+            params['filters'] = filters
+        urlencoded_params = urlencode(params)
 
         response = requests.get(url=f'{self.base_url}/dids/{scope}/dids/search?{urlencoded_params}', headers=headers, verify=self.rucio_ca_cert)
 
@@ -80,6 +208,23 @@ def search_did(self, scope, name, search_type='collection', limit=None):
 
         return results
 
+    def get_metadata(self, scope, name):
+        token = self._get_auth_token()
+        headers = {'X-Rucio-Auth-Token': token}
+
+        scope = quote(scope)
+        name = quote(name)
+
+        response = requests.get(url=f'{self.base_url}/dids/{scope}/{name}/meta', headers=headers, verify=self.rucio_ca_cert)
+
+        if response.text == '':
+            return []
+
+        lines = response.text.rstrip('\n').splitlines()
+        results = [json.loads(l) for l in lines]
+
+        return results
+
     def get_files(self, scope, name):
         token = self._get_auth_token()
         headers = {'X-Rucio-Auth-Token': token}