Text & image generation

Stream an answer, reuse conversation context, and cancel generation.

Use your own .slab model and a runtime license. Model files carry their input sizes, preprocessing, labels, and backend preferences.

chat.py

Python setup →

Run Python inside a signed macOS app with a customer runtime build, or use a development build.

"""Ask a vision-language model about an image and stream the answer, then
ask a follow-up question in the same conversation.

usage: python chat.py MODEL.slab IMAGE LICENSE_FILE
"""

import sys

import slab

model, image, license_file = sys.argv[1:4]

cfg = slab.Config(license=open(license_file).read())
session = slab.Session(cfg, model, context_length=2048)

inp = slab.Input()
inp.max_tokens = 64
inp.temperature = 0.0
inp.set_image_tiled(True, 1536)   # large images are split into tiles, not shrunk

out = slab.Output()
inp.add_message("user", "What is this document?", image=slab.Frame.open(image))
for piece in session.generate(inp, out):
    print(piece, end="", flush=True)
print()

# Send the conversation again with the answer and a new question.
# The shared beginning is reused, not recomputed.
inp.add_message("assistant", out.full_text.strip())
inp.add_message("user", "What is the total amount?")
for piece in session.generate(inp, out):
    print(piece, end="", flush=True)
print()
print(f"prompt {out.prompt_tokens} tokens ({out.cached_prompt_tokens} reused), generated {out.generated_tokens}")

session.reset()   # forget the conversation before an unrelated one