Merge pull request #92 from theablefew/esmarkowski/add-machinelearnin…

…g-models

Add MachineLearning Models

.github/workflows/spec.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         os: ['ubuntu']
-        ruby: ['3.1', '2.7']
+        ruby: ['3.1', '3.2']
         elasticsearch: ['7.x-SNAPSHOT', '8.12.2']
 
     steps:
@@ -68,4 +68,4 @@ jobs:
         - name: Install dependencies
           run: bundle install
         - name: Run tests
-          run: bundle exec rspec
+          run: bundle exec rspec --tag ~type:integration

containers/Dockerfile.opensearch

@@ -2,7 +2,6 @@
 
 # Base image
 FROM opensearchproject/opensearch:2.12.0
-
 # Environment variables
 # disables bootstrap checks that are enabled when network.host is set to a non-loopback address
 ENV discovery.type=single-node
@@ -12,8 +11,10 @@ ENV plugins.security.disabled=true
 # along with the memlock settings below, disables swapping
 ENV bootstrap.memory_lock=true
 # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
-ENV OPENSEARCH_JAVA_OPTS="-Xms512m -Xmx512m"
+ENV OPENSEARCH_JAVA_OPTS="-Xms3078m -Xmx3078m"
 # disables execution of install_demo_configuration.sh bundled with security plugin, which installs demo certificates and security configurations to OpenSearch
 ENV DISABLE_INSTALL_DEMO_CONFIG=true
 # disables security plugin entirely in OpenSearch by setting plugins.security.disabled: true in opensearch.yml
-ENV DISABLE_SECURITY_PLUGIN=true
+# ENV DISABLE_SECURITY_PLUGIN=true
+
+

docker-compose.yml

@@ -10,11 +10,16 @@ services:
       - "9200:9200"
     environment:
       - discovery.type=single-node
+    networks:
+      - elasticsearch-net
+
 
   opensearch:
-    build:
-      context: .
-      dockerfile: containers/Dockerfile.opensearch
+    image: opensearchproject/opensearch:latest
+    container_name: opensearch-node1
+    # build:
+    #   context: .
+    #   dockerfile: containers/Dockerfile.opensearch
     ulimits:
       memlock:
         soft: -1
@@ -24,29 +29,37 @@ services:
         hard: 65536
     environment:
       - discovery.type=single-node
+      - node.name=opensearch-node1
+      - plugins.security.disabled=true
+      - bootstrap.memory_lock=true
+      - "OPENSEARCH_JAVA_OPTS=-Xms3078m -Xmx3078m"
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=A3s0p3nS3cUr1tY
     # volumes:
     #   - opensearch-data1:/usr/share/opensearch/data
     ports:
       - 9200:9200
       - 9600:9600 # required for Performance Analyzer
-    # networks:
-    #   - opensearch-net
+    networks:
+      - opensearch-net
+
+  opensearch-dashboards:
+    image: opensearch-dashboards-no-security # Make sure the version of opensearch-dashboards matches the version of opensearch installed on other nodes
+    container_name: opensearch-dashboards
+    ports:
+      - 5601:5601 # Map host port 5601 to container port 5601
+    expose:
+      - "5601" # Expose port 5601 for web access to OpenSearch Dashboards
+    environment:
+      - OPENSEARCH_HOSTS="http://opensearch-node1:9200"
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=A3s0p3nS3cUr1tY
+      - plugins.security.disabled=true
+    networks:
+      - opensearch-net
 
-  # opensearch-dashboards:
-  #   image: opensearchproject/opensearch-dashboards:1.2.0
-  #   container_name: opensearch-dashboards
-  #   ports:
-  #     - 5601:5601
-  #   expose:
-  #     - "5601"
-  #   environment:
-  #     - 'OPENSEARCH_HOSTS=["http://opensearch-node1:9200"]'
-  #     - "DISABLE_SECURITY_DASHBOARDS_PLUGIN=true" # disables security dashboards plugin in OpenSearch Dashboards
-  #   networks:
-  #     - opensearch-net
 
 # volumes:
   # opensearch-data1:
 
-# networks:
-  # opensearch-net:
+networks:
+  opensearch-net:
+  elasticsearch-net:

lib/elasticsearch/api/actions/machine_learning/models/delete_model.rb

@@ -0,0 +1,33 @@
+module Elasticsearch
+  module API
+    module MachineLearning
+	    module Models
+	      module Actions
+	        # Returns a model.
+	        #
+	        # @option arguments [String] :id The model id
+          # @option arguments [Hash] :headers Custom HTTP headers
+	        #
+	        # Example
+	        # delete_model(id: 109sdj0asl092)
+          # 
+          # DELETE /_plugins/_ml/models/<model_id>
+	        def delete_model(arguments = {})
+		        raise ArgumentError, "Required argument 'id' missing" unless arguments[:id]
+		        headers = arguments.delete(:headers) || {}
+
+            _id = arguments.delete(:id)
+
+		        method = Elasticsearch::API::HTTP_DELETE
+		        path   = "_plugins/_ml/models/#{Utils.__listify(_id)}"
+		        params = Utils.__validate_and_extract_params arguments, ParamsRegistry.get(__method__)
+
+            body = nil
+		        perform_request(method, path, params, body, headers).body
+	        end
+
+	      end
+	    end
+    end
+  end
+end

lib/elasticsearch/api/actions/machine_learning/models/deploy.rb

@@ -0,0 +1,31 @@
+module Elasticsearch
+  module API
+    module MachineLearning
+	    module Models
+	      module Actions
+	        # Register a model.
+	        #
+	        # @option arguments [String] :id The model id
+	        # @option arguments [Hash] :body The deploy options
+	        #
+          # 
+          # POST /_plugins/_ml/models/<model_id>/_deploy
+	        def deploy(arguments = {})
+            raise ArgumentError, "Required argument 'id' missing" unless arguments[:id]
+	          _id = arguments.delete(:id)
+	          arguments = arguments.clone
+            headers = arguments.delete(:headers) || {}
+
+	          method = Elasticsearch::API::HTTP_POST
+	          path   = "_plugins/_ml/models/#{Utils.__listify(_id)}/_deploy"
+	          params = Utils.__validate_and_extract_params arguments, ParamsRegistry.get(__method__)
+
+	          body = arguments[:body]
+	          perform_request(method, path, params, body, headers).body
+	        end
+
+	      end
+	    end
+    end
+  end
+end

lib/elasticsearch/api/actions/machine_learning/models/get_model.rb

@@ -0,0 +1,43 @@
+module Elasticsearch
+  module API
+    module MachineLearning
+	    module Models
+	      module Actions
+	        # Returns a model.
+	        #
+	        # @option arguments [String] :id The model id
+	        #
+	        # Example
+	        # get_model(id: 109sdj0asl092)
+	        #
+	        # Example
+	        # # Get all models
+	        # get_model
+          # 
+          # GET /_plugins/_ml/models/<model_id>
+	        def get_model(arguments = {})
+            _id = arguments.delete(:id)
+            headers = arguments.delete(:headers) || {}
+		        method = Elasticsearch::API::HTTP_GET
+		        path   = if _id
+		                     body = nil
+		                     "_ml/trained_models/#{Utils.__listify(_id)}"
+		                   else
+		                     body = {
+														"query": {
+															"match_all": {}
+														},
+														"size": 1000
+													}
+		                     '_ml/trained_models'
+		                   end
+		        params = Utils.__validate_and_extract_params arguments, ParamsRegistry.get(__method__)
+
+		        perform_request(method, path, params, body, headers).body
+	        end
+
+	      end
+	    end
+    end
+  end
+end

lib/elasticsearch/api/actions/machine_learning/models/get_status.rb

@@ -0,0 +1,31 @@
+module Elasticsearch
+  module API
+    module MachineLearning
+	    module Models
+	      module Actions
+	        # Register a model.
+	        #
+	        # @option arguments [String] :id The model id
+	        # @option arguments [Hash] :body The deploy options
+	        #
+          # 
+          # GET /_plugins/_ml/tasks/<task_id>
+	        def get_status(arguments = {})
+						raise ArgumentError, "Required argument 'task_id' missing" unless arguments[:task_id]
+	          _id = arguments.delete(:task_id)
+	          arguments = arguments.clone
+						headers = arguments.delete(:headers) || {}
+
+	          method = Elasticsearch::API::HTTP_GET
+	          path   = "_plugins/_ml/tasks/#{Utils.__listify(_id)}"
+	          params = Utils.__validate_and_extract_params arguments, ParamsRegistry.get(__method__)
+
+						body = nil
+	          perform_request(method, path, params, body, headers).body
+	        end
+
+	      end
+	    end
+    end
+  end
+end

lib/elasticsearch/api/actions/machine_learning/models/params_registry.rb

@@ -0,0 +1,45 @@
+module Elasticsearch
+  module API
+    module MachineLearning
+      module Models
+          module Actions
+            module ParamsRegistry
+              extend self
+
+              # A Mapping of all the actions to their list of valid params.
+              #
+              # @since 6.1.1
+              PARAMS = {}
+
+              # Register an action with its list of valid params.
+              #
+              # @example Register the action.
+              #   ParamsRegistry.register(:benchmark, [ :verbose ])
+              #
+              # @param [ Symbol ] action The action to register.
+              # @param [ Array[Symbol] ] valid_params The list of valid params.
+              #
+              # @since 6.1.1
+              def register(action, valid_params)
+                PARAMS[action.to_sym] = valid_params
+              end
+
+              # Get the list of valid params for a given action.
+              #
+              # @example Get the list of valid params.
+              #   ParamsRegistry.get(:benchmark)
+              #
+              # @param [ Symbol ] action The action.
+              #
+              # @return [ Array<Symbol> ] The list of valid params for the action.
+              #
+              # @since 6.1.1
+              def get(action)
+                PARAMS.fetch(action, [])
+              end
+            end
+          end
+        end
+    end
+  end
+end

lib/elasticsearch/api/actions/machine_learning/models/register.rb

@@ -0,0 +1,45 @@
+module Elasticsearch
+  module API
+    module MachineLearning
+	    module Models
+	      module Actions
+	        # Register a model.
+	        #
+	        # @option arguments [Hash] :body The model definition **Required**
+	        # @option arguments [Boolean] :deploy Whether to deploy the model after registering it. The deploy operation is performed by calling the [Deploy Model API](https://opensearch.org/docs/latest/ml-commons-plugin/api/model-apis/deploy-model/). Default is `false`
+	        #
+	        # Example
+          # client.machine_learing_model.register(body: {
+          #	  "name": "huggingface/sentence-transformers/msmarco-distilbert-base-tas-b",
+          #	  "version": "1.0.1",
+          #	  "model_group_id": "Z1eQf4oB5Vm0Tdw8EIP2",
+          #	  "model_format": "TORCH_SCRIPT"
+          #	},
+          #	deploy: true 
+          #
+          # POST /_plugins/_ml/models/_register
+	        def register(arguments = {})
+	          raise ArgumentError, "Required argument 'body' missing" unless arguments[:body]
+
+	          arguments = arguments.clone
+            headers = arguments.delete(:headers) || {}
+
+	          method = Elasticsearch::API::HTTP_POST
+	          path   = "_plugins/_ml/models/_register"
+	          params = Utils.__validate_and_extract_params arguments, ParamsRegistry.get(__method__)
+
+	          body = arguments[:body]
+	          perform_request(method, path, params, body, headers).body
+	        end
+
+	        # Register this action with its valid params when the module is loaded.
+	        #
+	        # @since 6.2.0
+	        ParamsRegistry.register(:put_pipeline, %i[
+				deploy
+	        ].freeze)
+	      end
+	    end
+    end
+  end
+end

lib/elasticsearch/api/actions/machine_learning/models/undeploy.rb

@@ -0,0 +1,32 @@
+module Elasticsearch
+  module API
+    module MachineLearning
+	    module Models
+	      module Actions
+	        # Register a model.
+	        #
+	        # @option arguments [String] :id The model id
+	        # @option arguments [Hash] :body The deploy options
+	        #
+          # 
+          # POST /_plugins/_ml/models/<model_id>/_undeploy
+	        def undeploy(arguments = {})
+		      raise ArgumentError, "Required argument 'id' missing" unless arguments[:id]
+	          raise ArgumentError, "Required argument 'body' missing" unless arguments[:body]
+	          _id = arguments.delete(:id)
+	          arguments = arguments.clone
+            headers = arguments.delete(:headers) || {}
+
+	          method = Elasticsearch::API::HTTP_POST
+	          path   = "_plugins/_ml/models/#{Utils.__listify(_id)}/_undeploy"
+	          params = Utils.__validate_and_extract_params arguments, ParamsRegistry.get(__method__)
+
+	          body = arguments[:body]
+	          perform_request(method, path, params, body, headers).body
+	        end
+
+	      end
+	    end
+    end
+  end
+end