Skip to content

Commit 7f71b07

Browse files
astappievastappiev
committed
feat(chat): support image input
1 parent 7a4ef36 commit 7f71b07

8 files changed

Lines changed: 237 additions & 15 deletions

File tree

‎connectors/AnthropicConnector/src/main/java/de/l3s/interweb/connector/anthropic/entity/CompletionBody.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
import com.fasterxml.jackson.annotation.JsonProperty;
99

1010
import de.l3s.interweb.core.chat.CompletionsQuery;
11-
import de.l3s.interweb.core.chat.Message;
1211
import de.l3s.interweb.core.chat.Role;
1312

1413
@JsonInclude(JsonInclude.Include.NON_NULL)
@@ -39,7 +38,7 @@ public CompletionBody(CompletionsQuery query) {
3938
this.system = query.getMessages().stream()
4039
.filter(m -> m.getRole() == Role.system)
4140
.findFirst()
42-
.map(Message::getContent)
41+
.map(CompletionMessage::extractSystemMessage)
4342
.orElse(null);
4443

4544
this.temperature = query.getTemperature();

‎connectors/AnthropicConnector/src/main/java/de/l3s/interweb/connector/anthropic/entity/CompletionMessage.java‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
11
package de.l3s.interweb.connector.anthropic.entity;
22

3+
import java.util.List;
4+
import java.util.Map;
5+
36
import io.quarkus.runtime.annotations.RegisterForReflection;
47

58
import de.l3s.interweb.core.chat.Message;
9+
import de.l3s.interweb.core.chat.MessagePart;
610

711
@RegisterForReflection
812
public final class CompletionMessage {
@@ -11,7 +15,38 @@ public final class CompletionMessage {
1115

1216
public CompletionMessage(Message message) {
1317
this.role = message.getRole().name();
14-
this.content = message.getContent();
18+
this.content = extractContentString(message.getContent());
19+
}
20+
21+
private static String extractContentString(Object contentObj) {
22+
if (contentObj == null) {
23+
return null;
24+
}
25+
if (contentObj instanceof String s) {
26+
return s;
27+
}
28+
if (contentObj instanceof List<?> list) {
29+
StringBuilder sb = new StringBuilder();
30+
for (Object item : list) {
31+
if (item instanceof MessagePart part) {
32+
if (MessagePart.TYPE_TEXT.equals(part.getType()) && part.getText() != null) {
33+
sb.append(part.getText());
34+
}
35+
} else if (item instanceof Map<?, ?> map) {
36+
Object typeObj = map.get("type");
37+
Object textObj = map.get("text");
38+
if (MessagePart.TYPE_TEXT.equals(typeObj) && textObj instanceof String text) {
39+
sb.append(text);
40+
}
41+
}
42+
}
43+
return !sb.isEmpty() ? sb.toString() : null;
44+
}
45+
return contentObj.toString();
46+
}
47+
48+
public static String extractSystemMessage(Message message) {
49+
return extractContentString(message.getContent());
1550
}
1651

1752
public String getRole() {

‎connectors/OllamaConnector/src/main/java/de/l3s/interweb/connector/ollama/entity/Message.java‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
11
package de.l3s.interweb.connector.ollama.entity;
22

3+
import java.util.ArrayList;
34
import java.util.List;
5+
import java.util.Map;
46

57
import io.quarkus.runtime.annotations.RegisterForReflection;
68

79
import com.fasterxml.jackson.annotation.JsonInclude;
810
import com.fasterxml.jackson.annotation.JsonProperty;
911

12+
import de.l3s.interweb.core.chat.MessagePart;
13+
1014
@RegisterForReflection
1115
@JsonInclude(JsonInclude.Include.NON_NULL)
1216
public final class Message {
@@ -63,8 +67,53 @@ public void setToolCalls(Object toolCalls) {
6367
public static Message of(de.l3s.interweb.core.chat.Message message) {
6468
Message result = new Message();
6569
result.setRole(Role.of(message.getRole()));
66-
result.setContent(message.getContent());
6770
result.setToolCalls(message.getToolCalls());
71+
72+
if (message.getContent() instanceof String s) {
73+
result.setContent(s);
74+
} else if (message.getContent() instanceof List<?> list) {
75+
StringBuilder textContent = new StringBuilder();
76+
List<String> images = new ArrayList<>();
77+
for (Object obj : list) {
78+
String type = null;
79+
String text = null;
80+
String url = null;
81+
82+
if (obj instanceof MessagePart part) {
83+
type = part.getType();
84+
text = part.getText();
85+
if (part.getImageUrl() != null) {
86+
url = part.getImageUrl().getUrl();
87+
}
88+
} else if (obj instanceof Map<?, ?> map) {
89+
Object typeObj = map.get("type");
90+
Object textObj = map.get("text");
91+
type = typeObj instanceof String ? (String) typeObj : null;
92+
text = textObj instanceof String ? (String) textObj : null;
93+
Object urlObj = map.get("image_url");
94+
if (urlObj instanceof Map<?, ?> urlMap) {
95+
Object urlValue = urlMap.get("url");
96+
url = urlValue instanceof String ? (String) urlValue : null;
97+
}
98+
}
99+
100+
if (MessagePart.TYPE_TEXT.equals(type) && text != null) {
101+
textContent.append(text);
102+
} else if (MessagePart.TYPE_IMAGE_URL.equals(type) && url != null && url.startsWith(MessagePart.MIME_DATA_IMAGE)) {
103+
int commaIndex = url.indexOf(',');
104+
if (commaIndex != -1) {
105+
images.add(url.substring(commaIndex + 1));
106+
}
107+
}
108+
}
109+
if (!textContent.isEmpty()) {
110+
result.setContent(textContent.toString());
111+
}
112+
if (!images.isEmpty()) {
113+
result.setImages(images);
114+
}
115+
}
116+
68117
return result;
69118
}
70119

‎interweb-core/src/main/java/de/l3s/interweb/core/chat/Message.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,9 @@ public class Message implements Serializable {
3131
private String name;
3232
/**
3333
* The contents of the message.
34+
* Can be a simple String or a List of MessagePart objects for multimodal messages (e.g., text and images).
3435
*/
35-
private String content;
36+
private Object content;
3637
/**
3738
* The refusal message generated by the model.
3839
*/
@@ -65,7 +66,7 @@ public Message(final Role role) {
6566
this.role = role;
6667
}
6768

68-
public Message(Role role, String content) {
69+
public Message(Role role, Object content) {
6970
this.role = role;
7071
this.content = content;
7172
this.created = Instant.now();
@@ -95,11 +96,11 @@ public void setName(final String name) {
9596
this.name = name;
9697
}
9798

98-
public String getContent() {
99+
public Object getContent() {
99100
return content;
100101
}
101102

102-
public void setContent(final String content) {
103+
public void setContent(final Object content) {
103104
this.content = content;
104105
}
105106

@@ -151,15 +152,15 @@ public void setCreated(final Instant created) {
151152
this.created = created;
152153
}
153154

154-
public static Message system(String content) {
155+
public static Message system(Object content) {
155156
return new Message(Role.system, content);
156157
}
157158

158-
public static Message user(String content) {
159+
public static Message user(Object content) {
159160
return new Message(Role.user, content);
160161
}
161162

162-
public static Message assistant(String content) {
163+
public static Message assistant(Object content) {
163164
return new Message(Role.assistant, content);
164165
}
165166
}

‎interweb-core/src/main/java/de/l3s/interweb/core/chat/MessagePart.java‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
package de.l3s.interweb.core.chat;
2+
3+
import java.io.Serial;
4+
import java.io.Serializable;
5+
6+
import io.quarkus.runtime.annotations.RegisterForReflection;
7+
8+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
9+
import com.fasterxml.jackson.annotation.JsonInclude;
10+
import com.fasterxml.jackson.annotation.JsonProperty;
11+
12+
@RegisterForReflection
13+
@JsonIgnoreProperties(ignoreUnknown = true)
14+
@JsonInclude(JsonInclude.Include.NON_NULL)
15+
public class MessagePart implements Serializable {
16+
@Serial
17+
private static final long serialVersionUID = 7321150327615238431L;
18+
19+
public static final String TYPE_TEXT = "text";
20+
public static final String TYPE_IMAGE_URL = "image_url";
21+
public static final String MIME_DATA_IMAGE = "data:image/";
22+
23+
/**
24+
* The type of the content part. Can be "text" or "image_url".
25+
*/
26+
private String type;
27+
28+
/**
29+
* The text content.
30+
*/
31+
private String text;
32+
33+
/**
34+
* The image URL.
35+
*/
36+
@JsonProperty("image_url")
37+
private ImageUrl imageUrl;
38+
39+
public MessagePart() {
40+
}
41+
42+
public MessagePart(String type, String text) {
43+
this.type = type;
44+
this.text = text;
45+
}
46+
47+
public MessagePart(String type, ImageUrl imageUrl) {
48+
this.type = type;
49+
this.imageUrl = imageUrl;
50+
}
51+
52+
public String getType() {
53+
return type;
54+
}
55+
56+
public void setType(String type) {
57+
this.type = type;
58+
}
59+
60+
public String getText() {
61+
return text;
62+
}
63+
64+
public void setText(String text) {
65+
this.text = text;
66+
}
67+
68+
public ImageUrl getImageUrl() {
69+
return imageUrl;
70+
}
71+
72+
public void setImageUrl(ImageUrl imageUrl) {
73+
this.imageUrl = imageUrl;
74+
}
75+
76+
public static MessagePart text(String text) {
77+
return new MessagePart("text", text);
78+
}
79+
80+
public static MessagePart imageUrl(String url) {
81+
ImageUrl imageUrl = new ImageUrl();
82+
imageUrl.setUrl(url);
83+
return new MessagePart("image_url", imageUrl);
84+
}
85+
86+
@RegisterForReflection
87+
@JsonIgnoreProperties(ignoreUnknown = true)
88+
@JsonInclude(JsonInclude.Include.NON_NULL)
89+
public static class ImageUrl implements Serializable {
90+
@Serial
91+
private static final long serialVersionUID = -3746742450532357308L;
92+
93+
private String url;
94+
/**
95+
* Specifies the detail level of the image. Can be "low", "high", or "auto".
96+
*/
97+
private String detail;
98+
99+
public String getUrl() {
100+
return url;
101+
}
102+
103+
public void setUrl(String url) {
104+
this.url = url;
105+
}
106+
107+
public String getDetail() {
108+
return detail;
109+
}
110+
111+
public void setDetail(String detail) {
112+
this.detail = detail;
113+
}
114+
}
115+
}
116+

‎interweb-server/src/main/java/de/l3s/interweb/server/features/chat/ChatMessage.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,13 +49,28 @@ public ChatMessage() {
4949
public ChatMessage(final Message message) {
5050
this.id = message.getId();
5151
this.role = message.getRole();
52-
this.content = message.getContent();
52+
if (message.getContent() instanceof String s) {
53+
this.content = s;
54+
} else if (message.getContent() != null) {
55+
this.content = JsonUtils.toJson(message.getContent());
56+
} else {
57+
this.content = null;
58+
}
5359
this.toolCalls = JsonUtils.toJson(message.getToolCalls());
5460
this.created = message.getCreated();
5561
}
5662

5763
public Message toMessage() {
58-
Message message = new Message(role, content);
64+
Object parsedContent = content;
65+
if (content != null && !content.isEmpty() && (content.charAt(0) == '[' || content.charAt(0) == '{')) {
66+
try {
67+
parsedContent = JsonUtils.fromJson(content, Object.class);
68+
} catch (RuntimeException e) {
69+
// Keep as string if JSON parsing fails (e.g., malformed JSON)
70+
parsedContent = content;
71+
}
72+
}
73+
Message message = new Message(role, parsedContent);
5974
message.setCreated(created);
6075
message.setId(id);
6176
message.setToolCalls(JsonUtils.fromJson(toolCalls, new TypeReference<>() {}));

‎interweb-server/src/main/java/de/l3s/interweb/server/features/chat/ChatResource.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,11 @@ public Uni<CompletionsResults> completions(@Valid CompletionsQuery query) {
5656
Uni<Void> chatUni = Uni.createFrom().voidItem();
5757
if (chat.title == null) {
5858
chatUni = chatUni.call(() -> chatService.generateTitle(chat)
59-
.onFailure().recoverWithItem(() -> StringUtils.shorten(query.getMessages().getLast().getContent(), 120))
59+
.onFailure().recoverWithItem(() -> {
60+
Object lastContentObj = query.getMessages().getLast().getContent();
61+
String lastContent = lastContentObj instanceof String ? (String) lastContentObj : "Chat";
62+
return StringUtils.shorten(lastContent, 120);
63+
})
6064
.invoke(title -> {
6165
chat.title = title;
6266
results.setChatTitle(title);

‎interweb-server/src/main/java/de/l3s/interweb/server/features/chat/ChatService.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@ public Uni<String> generateTitle(final Chat chat) {
8080
Summarize the conversation in 5 words or less, in a way that sounds like a book title.
8181
Don't use any formatting. You can use emojis. Only print the title, nothing else.
8282
""".formatted(sb), Role.user);
83-
return completions(query).map(results -> results.getLastMessage().getContent().trim());
83+
return completions(query).map(results -> {
84+
Object content = results.getLastMessage().getContent();
85+
return content instanceof String ? ((String) content).trim() : String.valueOf(content).trim();
86+
});
8487
}
8588
}

0 commit comments

Comments
 (0)