[HWORKS-1224] Add llm signature and openai endpoint

Author: javierdlrm

python/hopsworks_common/constants.py

@@ -158,6 +158,7 @@ class MODEL:
     FRAMEWORK_TORCH = "TORCH"
     FRAMEWORK_PYTHON = "PYTHON"
     FRAMEWORK_SKLEARN = "SKLEARN"
+    FRAMEWORK_LLM = "LLM"
 
 
 class MODEL_REGISTRY:
@@ -210,6 +211,7 @@ class PREDICTOR:
     # model server
     MODEL_SERVER_PYTHON = "PYTHON"
     MODEL_SERVER_TF_SERVING = "TENSORFLOW_SERVING"
+    MODEL_SERVER_VLLM = "VLLM"
     # serving tool
     SERVING_TOOL_DEFAULT = "DEFAULT"
     SERVING_TOOL_KSERVE = "KSERVE"

python/hsml/core/serving_api.py

@@ -417,4 +417,7 @@ def _get_hopsworks_inference_path(self, project_id: int, deployment_instance):
         ]
 
     def _get_istio_inference_path(self, deployment_instance):
+        if deployment_instance.model_server == "VLLM":
+            return ["openai", "v1", "completions"]
+
         return ["v1", "models", deployment_instance.name + ":predict"]

python/hsml/engine/serving_engine.py

@@ -493,7 +493,10 @@ def predict(
         inputs: Union[Dict, List[Dict]],
     ):
         # validate user-provided payload
-        self._validate_inference_payload(deployment_instance.api_protocol, data, inputs)
+        if deployment_instance.model_server != "VLLM":
+            self._validate_inference_payload(
+                deployment_instance.api_protocol, data, inputs
+            )
 
         # build inference payload based on API protocol
         payload = self._build_inference_payload(

python/hsml/llm/__init__.py

@@ -0,0 +1,15 @@
+#
+#   Copyright 2024 Hopsworks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#

python/hsml/llm/model.py

@@ -0,0 +1,75 @@
+#
+#   Copyright 2024 Hopsworks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import humps
+from hsml.constants import MODEL
+from hsml.model import Model
+
+
+class Model(Model):
+    """Metadata object representing a LLM model in the Model Registry."""
+
+    def __init__(
+        self,
+        id,
+        name,
+        version=None,
+        created=None,
+        creator=None,
+        environment=None,
+        description=None,
+        project_name=None,
+        metrics=None,
+        program=None,
+        user_full_name=None,
+        model_schema=None,
+        training_dataset=None,
+        input_example=None,
+        model_registry_id=None,
+        tags=None,
+        href=None,
+        feature_view=None,
+        training_dataset_version=None,
+        **kwargs,
+    ):
+        super().__init__(
+            id,
+            name,
+            version=version,
+            created=created,
+            creator=creator,
+            environment=environment,
+            description=description,
+            project_name=project_name,
+            metrics=metrics,
+            program=program,
+            user_full_name=user_full_name,
+            model_schema=model_schema,
+            training_dataset=training_dataset,
+            input_example=input_example,
+            framework=MODEL.FRAMEWORK_LLM,
+            model_registry_id=model_registry_id,
+            feature_view=feature_view,
+            training_dataset_version=training_dataset_version,
+        )
+
+    def update_from_response_json(self, json_dict):
+        json_decamelized = humps.decamelize(json_dict)
+        json_decamelized.pop("framework")
+        if "type" in json_decamelized:  # backwards compatibility
+            _ = json_decamelized.pop("type")
+        self.__init__(**json_decamelized)
+        return self

python/hsml/llm/predictor.py

@@ -0,0 +1,28 @@
+#
+#   Copyright 2024 Hopsworks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+from hsml.constants import MODEL, PREDICTOR
+from hsml.predictor import Predictor
+
+
+class Predictor(Predictor):
+    """Configuration for a predictor running with the vLLM backend"""
+
+    def __init__(self, **kwargs):
+        kwargs["model_framework"] = MODEL.FRAMEWORK_LLM
+        kwargs["model_server"] = PREDICTOR.MODEL_SERVER_VLLM
+
+        super().__init__(**kwargs)

python/hsml/llm/signature.py

@@ -0,0 +1,77 @@
+#
+#   Copyright 2024 Hopsworks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+from typing import Optional, Union
+
+import numpy
+import pandas
+from hopsworks_common import usage
+from hsml.model_schema import ModelSchema
+from hsml.llm.model import Model
+
+
+_mr = None
+
+
+@usage.method_logger
+def create_model(
+    name: str,
+    version: Optional[int] = None,
+    metrics: Optional[dict] = None,
+    description: Optional[str] = None,
+    input_example: Optional[
+        Union[pandas.DataFrame, pandas.Series, numpy.ndarray, list]
+    ] = None,
+    model_schema: Optional[ModelSchema] = None,
+    feature_view=None,
+    training_dataset_version: Optional[int] = None,
+):
+    """Create an LLM model metadata object.
+
+    !!! note "Lazy"
+        This method is lazy and does not persist any metadata or uploads model artifacts in the
+        model registry on its own. To save the model object and the model artifacts, call the `save()` method with a
+        local file path to the directory containing the model artifacts.
+
+    # Arguments
+        name: Name of the model to create.
+        version: Optionally version of the model to create, defaults to `None` and
+            will create the model with incremented version from the last
+            version in the model registry.
+        metrics: Optionally a dictionary with model evaluation metrics (e.g., accuracy, MAE)
+        description: Optionally a string describing the model, defaults to empty string
+            `""`.
+        input_example: Optionally an input example that represents a single input for the model, defaults to `None`.
+        model_schema: Optionally a model schema for the model inputs and/or outputs.
+
+    # Returns
+        `Model`. The model metadata object.
+    """
+    model = Model(
+        id=None,
+        name=name,
+        version=version,
+        description=description,
+        metrics=metrics,
+        input_example=input_example,
+        model_schema=model_schema,
+        feature_view=feature_view,
+        training_dataset_version=training_dataset_version,
+    )
+    model._shared_registry_project_name = _mr.shared_registry_project_name
+    model._model_registry_id = _mr.model_registry_id
+
+    return model

python/hsml/model_registry.py

@@ -24,6 +24,7 @@
 from hsml.sklearn import signature as sklearn_signature  # noqa: F401
 from hsml.tensorflow import signature as tensorflow_signature  # noqa: F401
 from hsml.torch import signature as torch_signature  # noqa: F401
+from hsml.llm import signature as llm_signature  # noqa: F401
 
 
 class ModelRegistry:
@@ -49,11 +50,13 @@ def __init__(
         self._python = python_signature
         self._sklearn = sklearn_signature
         self._torch = torch_signature
+        self._llm = llm_signature
 
         tensorflow_signature._mr = self
         python_signature._mr = self
         sklearn_signature._mr = self
         torch_signature._mr = self
+        llm_signature._mr = self
 
     @classmethod
     def from_response_json(cls, json_dict):
@@ -191,6 +194,12 @@ def python(self):
 
         return python_signature
 
+    @property
+    def llm(self):
+        """Module for exporting a Large Language Model."""
+
+        return llm_signature
+
     def __repr__(self):
         project_name = (
             self._shared_registry_project_name

python/hsml/util.py

@@ -100,6 +100,7 @@ def set_model_class(model):
     from hsml.sklearn.model import Model as SkLearnModel
     from hsml.tensorflow.model import Model as TFModel
     from hsml.torch.model import Model as TorchModel
+    from hsml.llm.model import Model as LLMModel
 
     if "href" in model:
         _ = model.pop("href")
@@ -120,6 +121,8 @@ def set_model_class(model):
         return SkLearnModel(**model)
     elif framework == MODEL.FRAMEWORK_PYTHON:
         return PyModel(**model)
+    elif framework == MODEL.FRAMEWORK_LLM:
+        return LLMModel(**model)
     else:
         raise ValueError(
             "framework {} is not a supported framework".format(str(framework))
@@ -242,6 +245,8 @@ def get_predictor_for_model(model, **kwargs):
     from hsml.tensorflow.predictor import Predictor as TFPredictor
     from hsml.torch.model import Model as TorchModel
     from hsml.torch.predictor import Predictor as TorchPredictor
+    from hsml.llm.model import Model as LLMModel
+    from hsml.llm.predictor import Predictor as vLLMPredictor
 
     if not isinstance(model, BaseModel):
         raise ValueError(
@@ -258,6 +263,8 @@ def get_predictor_for_model(model, **kwargs):
         return SkLearnPredictor(**kwargs)
     if type(model) is PyModel:
         return PyPredictor(**kwargs)
+    if type(model) is LLMModel:
+        return vLLMPredictor(**kwargs)
     if type(model) is BaseModel:
         return BasePredictor(  # python as default framework and model server
             model_framework=MODEL.FRAMEWORK_PYTHON,