Llm Model

score.yaml (view on GitHub) :

apiVersion: score.dev/v1b1
metadata:
  name: my-workload
containers:
  my-container:
    image: busybox
    command: ["/bin/sh"]
    args: ["-c", "while true; do echo $LLM_BASE_URL; sleep 5; done"]
    variables:
      LLM_MODEL_NAME: "${resources.model.model}"
      LLM_BASE_URL: "${resources.model.url}"
resources:
  model:
    type: llm-model
    params:
      model: ai/smollm2:135M-Q4_0

score-compose
score-k8s

10-dmr-llm-model-via-curl-cmd.provisioners.yaml (view on GitHub) :

- uri: cmd://bash#dmr-llm-model-via-curl
  type: llm-model
  description: Runs curl to download the model with the Docker Model Runner (DMR).
  supported_params:
    - model
  expected_outputs:
    - model
    - url
    - api-key
  args:
    - -c
    - |
      STDIN=$(cat)
      MODEL=$(echo $STDIN | yq eval -p json '.resource_params.model')
      if [ "$MODEL" == "" ]; then
        MODEL="ai/smollm2:135M-Q4_0"
      fi
      set -eu -o pipefail
      curl -v -d '{"from":"'"$MODEL"'"}' "http://localhost:12434/models/create" >&2
      OUTPUTS='{"resource_outputs":{"model":"%s","url":"http://172.17.0.1:12434/engines/v1/","api-key":"not-needed"}}'
      printf "$OUTPUTS" "$MODEL"

10-dmr-llm-model-via-curl-service.provisioners.yaml (view on GitHub) :

- uri: template://community-provisioners/dmr-llm-model-via-curl
  type: llm-model
  description: Generates a curl service downloading the model with the Docker Model Runner (DMR).
  supported_params:
    - model
  outputs: |
    model: {{ .Init.model }}
    url: "http://172.17.0.1:12434/engines/v1/"
    api-key: "not-needed"    
  expected_outputs:
    - model
    - url
    - api-key
  init: |
    model: {{ .Params.model | default "ai/smollm2:135M-Q4_0" }}    
  services: |
    {{ .Id }}:
      image: curlimages/curl:latest
      command: ["curl", "-v", "-d", '{"from": "{{ .Init.model }}"}', "http://172.17.0.1:12434/models/create"]

10-dmr-llm-model-via-service-provider.provisioners.yaml (view on GitHub) :

- uri: template://community-provisioners/dmr-llm-model-via-service-provider
  type: llm-model
  description: Generates the LLM model service via the Docker Model Runner (DMR) provider.
  supported_params:
    - model
  outputs: |
    model: {{ .Init.model }}
    url: "http://172.17.0.1:12434/engines/v1/"
    api-key: "not-needed"    
  expected_outputs:
    - model
    - url
    - api-key
  init: |
    model: {{ .Params.model | default "ai/smollm2:135M-Q4_0" }}    
  services: |
    {{ .Id }}:
      provider:
        type: model
        options:
          model: {{ .Init.model }}

Initialize your local workspace, by importing a specific community provisioner:

score-commpose init --provisioners REPLACE-ME-WITH-ACTUAL-PROVISIONER-FILE-URL.yaml

Note: you need to replace REPLACE-ME-WITH-ACTUAL-PROVISIONER-FILE-URL.yaml by the actual provisioner file you want to use and import. More information here.

Get the provisioners definition:

score-compose provisioners list

Generate the platform specific manifests:

score-commpose generate score.yaml

See the resource outputs:

score-commpose resources list

You can run the following command on each resource listed with the previous command to get their outputs:

score-commpose resources get-outputs

Deploy the generated manifests:

docker compose up -d

See the running containers:

docker ps