Add parameter for image description; Refactor library core

examples/cli/main.go

@@ -38,7 +38,7 @@ func main() {
 			MaxTokens:   *maxTokens,
 		}).
 		SetPrompt(*prompt).
-		Execute(context.Background(), agency.NewMessage(agency.UserRole, agency.TextKind, []byte(content)))
+		Execute(context.Background(), agency.NewTextMessage(agency.UserRole, content))
 
 	if err != nil {
 		fmt.Println(err)

examples/custom_operation/main.go

@@ -17,11 +17,7 @@ func main() {
 		increment,
 	).Execute(
 		context.Background(),
-		agency.NewMessage(
-			agency.UserRole,
-			agency.TextKind,
-			[]byte("0"),
-		),
+		agency.NewTextMessage(agency.UserRole, "0"),
 	)
 	if err != nil {
 		panic(err)
@@ -36,5 +32,5 @@ func incrementFunc(ctx context.Context, msg agency.Message, _ *agency.OperationC
 		return nil, err
 	}
 	inc := strconv.Itoa(int(i) + 1)
-	return agency.NewMessage(agency.ToolRole, agency.TextKind, []byte(inc)), nil
+	return agency.NewTextMessage(agency.ToolRole, inc), nil
 }

examples/func_call/main.go

@@ -69,7 +69,7 @@ Examples:
 	// test for first function call
 	answer, err := t2tOp.Execute(
 		ctx,
-		agency.NewMessage(agency.UserRole, agency.TextKind, []byte("what is the meaning of life?")),
+		agency.NewTextMessage(agency.UserRole, "what is the meaning of life?"),
 	)
 	if err != nil {
 		panic(err)
@@ -79,7 +79,7 @@ Examples:
 	// test for second function call
 	answer, err = t2tOp.Execute(
 		ctx,
-		agency.NewMessage(agency.UserRole, agency.TextKind, []byte("1+1?")),
+		agency.NewTextMessage(agency.UserRole, "1+1?"),
 	)
 	if err != nil {
 		panic(err)
@@ -89,7 +89,7 @@ Examples:
 	// test for both function calls at the same time
 	answer, err = t2tOp.Execute(
 		ctx,
-		agency.NewMessage(agency.UserRole, agency.TextKind, []byte("1+1 and what is the meaning of life?")),
+		agency.NewTextMessage(agency.UserRole, "1+1 and what is the meaning of life?"),
 	)
 	if err != nil {
 		panic(err)

examples/image_to_stream/main.go

@@ -27,7 +27,7 @@ func main() {
 		SetPrompt("describe what you see").
 		Execute(
 			context.Background(),
-			agency.NewMessage(agency.UserRole, agency.ImageKind, imgBytes),
+			agency.NewImageMessage(agency.UserRole, imgBytes, ""),
 		)
 	if err != nil {
 		panic(err)

examples/image_to_text/main.go

@@ -11,7 +11,7 @@ import (
 	"github.com/sashabaranov/go-openai"
 )
 
-func main() { 
+func main() {
 	imgBytes, err := os.ReadFile("example.png")
 	if err != nil {
 		panic(err)
@@ -22,7 +22,7 @@ func main() {
 		SetPrompt("describe what you see").
 		Execute(
 			context.Background(),
-			agency.NewMessage(agency.UserRole, agency.ImageKind, imgBytes),
+			agency.NewImageMessage(agency.UserRole, imgBytes, ""),
 		)
 	if err != nil {
 		panic(err)

examples/logging/main.go

@@ -22,7 +22,7 @@ func main() {
 	).
 		Execute(
 			context.Background(),
-			agency.NewMessage(agency.UserRole, agency.TextKind, []byte("Kazakhstan alga!")),
+			agency.NewTextMessage(agency.UserRole, "Kazakhstan alga!"),
 			Logger,
 		)
 

examples/prompt_template/main.go

@@ -22,7 +22,7 @@ func main() {
 		).
 		Execute(
 			context.Background(),
-			agency.NewMessage(agency.UserRole, agency.TextKind, []byte("I love programming.")),
+			agency.NewTextMessage(agency.UserRole, "I love programming."),
 		)
 
 	if err != nil {

examples/rag_vector_database/main.go

@@ -36,7 +36,7 @@ func main() {
 		retrieve,
 		summarize,
 		voice,
-	).Execute(ctx, agency.NewMessage(agency.UserRole, agency.TextKind, []byte("programming")))
+	).Execute(ctx, agency.NewTextMessage(agency.UserRole, "programming"))
 	if err != nil {
 		panic(err)
 	}
@@ -76,10 +76,9 @@ func RAGoperation(client *weaviate.Client) *agency.Operation {
 			content += string(bb)
 		}
 
-		return agency.NewMessage(
+		return agency.NewTextMessage(
 			agency.AssistantRole,
-			agency.TextKind,
-			[]byte(content),
+			content,
 		), nil
 	})
 }
@@ -103,9 +102,9 @@ func prepareDB(openAPIKey string, ctx context.Context) (*weaviate.Client, error)
 	classObj := &models.Class{
 		Class:      "Records",
 		Vectorizer: "text2vec-openai",
-		ModuleConfig: map[string]interface{}{
-			"text2vec-openai":   map[string]interface{}{},
-			"generative-openai": map[string]interface{}{},
+		ModuleConfig: map[string]any{
+			"text2vec-openai":   map[string]any{},
+			"generative-openai": map[string]any{},
 		},
 	}
 	if err = client.Schema().ClassCreator().WithClass(classObj).Do(context.Background()); err != nil {

examples/speech_to_text/main.go

@@ -26,7 +26,7 @@ func main() {
 		Model: goopenai.Whisper1,
 	}).Execute(
 		context.Background(),
-		agency.NewMessage(agency.UserRole, agency.VoiceKind, data),
+		agency.NewVoiceMessage(agency.UserRole, data),
 	)
 
 	if err != nil {

examples/speech_to_text_multi_model/main.go

@@ -52,7 +52,7 @@ func main() {
 	}
 
 	ctx := context.Background()
-	speechMsg := agency.NewMessage(agency.UserRole, agency.VoiceKind, sound)
+	speechMsg := agency.NewVoiceMessage(agency.UserRole, sound)
 
 	_, err = agency.NewProcess(
 		hear,

examples/speech_to_text_to_image/main.go

@@ -28,7 +28,7 @@ func main() {
 			Model:     goopenai.CreateImageModelDallE2,
 			ImageSize: goopenai.CreateImageSize256x256,
 		}),
-	).Execute(context.Background(), agency.NewMessage(agency.UserRole, agency.VoiceKind, data))
+	).Execute(context.Background(), agency.NewVoiceMessage(agency.UserRole, data))
 	if err != nil {
 		panic(err)
 	}

examples/text_to_image_dalle2/main.go

@@ -26,7 +26,7 @@ func main() {
 		Style:     "vivid",
 	}).Execute(
 		context.Background(),
-		agency.NewMessage(agency.UserRole, agency.TextKind, []byte("Halloween night at a haunted museum")),
+		agency.NewTextMessage(agency.UserRole, "Halloween night at a haunted museum"),
 	)
 	if err != nil {
 		panic(err)

examples/text_to_speech/main.go

@@ -11,12 +11,12 @@ import (
 )
 
 func main() {
-	input := agency.NewMessage(
+	input := agency.NewTextMessage(
 		agency.UserRole,
-		agency.TextKind,
-		[]byte(`One does not simply walk into Mordor.
-Its black gates are guarded by more than just Orcs.
-There is evil there that does not sleep, and the Great Eye is ever watchful.`))
+		`One does not simply walk into Mordor.
+		Its black gates are guarded by more than just Orcs.
+		There is evil there that does not sleep, and the Great Eye is ever watchful.`,
+	)
 
 	msg, err := openai.New(openai.Params{Key: os.Getenv("OPENAI_API_KEY")}).
 		TextToSpeech(openai.TextToSpeechParams{

examples/text_to_stream/main.go

@@ -32,11 +32,7 @@ func main() {
 		SetPrompt("Write a few sentences about topic").
 		Execute(
 			context.Background(),
-			agency.NewMessage(
-				agency.UserRole,
-				agency.TextKind,
-				[]byte("I love programming."),
-			),
+			agency.NewTextMessage(agency.UserRole, "I love programming."),
 		)
 	if err != nil {
 		panic(err)

examples/translate_text/main.go

@@ -20,11 +20,7 @@ func main() {
 		SetPrompt("You are a helpful assistant that translates English to French").
 		Execute(
 			context.Background(),
-			agency.NewMessage(
-				agency.UserRole,
-				agency.TextKind,
-				[]byte("I love programming."),
-			),
+			agency.NewTextMessage(agency.UserRole, "I love programming."),
 		)
 
 	if err != nil {

messages.go

@@ -1,11 +1,9 @@
 package agency
 
-import "encoding/json"
-
 type Message interface {
 	Role() Role
 	Content() []byte
-	Kind() Kind
+	Kind() Kind // do we need this?
 }
 
 type Kind string
@@ -26,51 +24,77 @@ const (
 	ToolRole      Role = "tool"
 )
 
-type BaseMessage struct {
-	content []byte
+// --- Text Message ---
+
+type TextMessage struct {
 	role    Role
-	kind    Kind
+	content string
 }
 
-func (bm BaseMessage) Role() Role {
-	return bm.role
+func (m TextMessage) Role() Role      { return m.role }
+func (m TextMessage) Kind() Kind      { return TextKind }
+func (m TextMessage) Content() []byte { return []byte(m.content) }
+
+func NewTextMessage(role Role, content string) TextMessage {
+	return TextMessage{
+		content: content,
+		role:    role,
+	}
 }
 
-func (bm BaseMessage) Kind() Kind {
-	return bm.kind
+// --- Image Message ---
+
+type ImageMessage struct {
+	role        Role
+	content     []byte
+	description string
 }
-func (bm BaseMessage) Content() []byte {
-	return bm.content
+
+func (m ImageMessage) Role() Role          { return m.role }
+func (m ImageMessage) Kind() Kind          { return ImageKind }
+func (m ImageMessage) Content() []byte     { return m.content }
+func (m ImageMessage) Description() string { return m.description }
+
+func NewImageMessage(role Role, content []byte, description string) ImageMessage {
+	return ImageMessage{
+		content:     content,
+		description: description,
+		role:        role,
+	}
 }
 
-// NewMessage creates new `Message` with the specified `Role` and `Kind`
-func NewMessage(role Role, kind Kind, content []byte) BaseMessage {
-	return BaseMessage{
+// --- Voice Message ---
+
+type VoiceMessage struct {
+	role    Role
+	content []byte
+}
+
+func (m VoiceMessage) Role() Role      { return m.role }
+func (m VoiceMessage) Kind() Kind      { return VoiceKind }
+func (m VoiceMessage) Content() []byte { return m.content }
+
+func NewVoiceMessage(role Role, content []byte) VoiceMessage {
+	return VoiceMessage{
 		content: content,
 		role:    role,
-		kind:    kind,
 	}
 }
 
-// NewTextMessage creates new `Message` with Text kind and the specified `Role`
-func NewTextMessage(role Role, content string) BaseMessage {
-	return BaseMessage{
-		content: []byte(content),
-		role:    role,
-		kind:    TextKind,
-	}
+// --- Embedding Message ---
+
+type EmbeddingMessage struct {
+	role    Role
+	content []byte
 }
 
-// NewJsonMessage marshals content and creates new `Message` with text kind and the specified `Role`
-func NewJsonMessage(role Role, content any) (BaseMessage, error) {
-	data, err := json.Marshal(content)
-	if err != nil {
-		return BaseMessage{}, err
-	}
+func (m EmbeddingMessage) Role() Role      { return m.role }
+func (m EmbeddingMessage) Kind() Kind      { return EmbeddingKind }
+func (m EmbeddingMessage) Content() []byte { return m.content }
 
-	return BaseMessage{
-		content: data,
+func NewEmbeddingMessage(role Role, content []byte) EmbeddingMessage {
+	return EmbeddingMessage{
+		content: content,
 		role:    role,
-		kind:    TextKind,
-	}, nil
+	}
 }

providers/openai/helpers.go

@@ -4,6 +4,9 @@ import (
 	"encoding/binary"
 	"fmt"
 	"math"
+
+	"github.com/neurocult/agency"
+	"github.com/sashabaranov/go-openai"
 )
 
 type Embedding []float32
@@ -71,3 +74,34 @@ func nullableToFloat32(v NullableFloat32) float32 {
 	}
 	return *v
 }
+
+// textMessageToOpenAI works with any agency message but ignores everything except role and content.
+func textMessageToOpenAI(message agency.Message) openai.ChatCompletionMessage {
+	return openai.ChatCompletionMessage{
+		Role:    string(message.Role()),
+		Content: string(message.Content()),
+	}
+}
+
+// agencyToOpenAIMessages returns slice of openai chat completion messages created from given config and message.
+// Resulting slices starts with config prompt followed by config messages and ends with given message.
+func agencyToOpenAIMessages(cfg *agency.OperationConfig, msg agency.Message) ([]openai.ChatCompletionMessage, error) {
+	openAIMessages := append(
+		make([]openai.ChatCompletionMessage, 0, len(cfg.Messages)+2),
+		openai.ChatCompletionMessage{
+			Role:    openai.ChatMessageRoleSystem,
+			Content: cfg.Prompt,
+		},
+	)
+
+	for _, cfgMsg := range cfg.Messages {
+		openAIMessages = append(openAIMessages, textMessageToOpenAI(cfgMsg))
+	}
+
+	openAIMessages = append(openAIMessages, openai.ChatCompletionMessage{
+		Role:    string(msg.Role()),
+		Content: string(msg.Content()),
+	})
+
+	return openAIMessages, nil
+}