sebastian/dall-e-bot-mattermost
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Funktioniert mit URL Images von Dall-E

Browse Source
This commit is contained in:
tessmania90
2023-04-06 12:30:56 +02:00
parent 3d1660deab
commit 3ba86d26a2
4 changed files with 138 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mattermost-chat.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 218 KiB

src/botservice - Kopie.js
+96
View File
@@ -0,0 +1,96 @@
const continueThread = require('./openai-thread-completion').continueThread
const { Log } = require('debug-level')
require('babel-polyfill');
require('isomorphic-fetch');
const { mmClient, wsClient } = require('./mm-client')
// the mattermost library uses FormData - so here is a polyfill
if (!global.FormData) {
global.FormData = require('form-data');
}
Log.options({ json: true, colors: true })
Log.wrapConsole('bot-ws', { level4log: 'INFO' })
const log = new Log('bot')
let meId = null;
mmClient.getMe().then(me => meId = me.id)
const name = process.env['MATTERMOST_BOTNAME'] || '@dall-e'
wsClient.addMessageListener(async function (event) {
if (['posted'].includes(event.event) && meId) {
const post = JSON.parse(event.data.post);
if (post.root_id === "" && (!event.data.mentions || (!JSON.parse(event.data.mentions).includes(meId)))) {
// we're not in a thread and we are not mentioned - ignore the message
} else {
if (post.user_id !== meId) {
const chatmessages = [
{
"role": "system",
"content": `You are a helpful assistant named ${name} who provides succinct answers in Markdown format.`
},
]
const thread = await mmClient.getPostThread(post.id, true, false, true)
const posts = [...new Set(thread.order)].map(id => thread.posts[id])
.filter(a => a.create_at > Date.now() - 1000 * 60 * 60 * 24 * 1)
.sort((a, b) => a.create_at - b.create_at)
let assistantCount = 0;
posts.forEach(threadPost => {
log.trace({msg: threadPost})
if (threadPost.user_id === meId) {
chatmessages.push({role: "assistant", content: threadPost.props.originalMessage ?? threadPost.message})
assistantCount++
} else {
if (threadPost.message.includes(name)){
assistantCount++;
}
chatmessages.push({role: "user", content: threadPost.message})
}
})
// see if we are actually part of the conversation -
// ignore conversations where we were never mentioned or participated.
if (assistantCount > 0){
wsClient.userTyping(post.channel_id, post.id ?? "")
log.trace({chatmessages})
const answer = await continueThread(chatmessages)
const fileData = new FormData();
fileData.append('files', answer, 'dall-e-image.png');
const uploadResponse = await mmClient.uploadFile(fileData, post.channel_id);
const fileId = JSON.parse(uploadResponse).file_infos[0].id;
log.trace({answer})
wsClient.userTyping(post.channel_id, post.id ?? "")
const message = `![alt text](${mmClient.getFileUrl(fileId)})`;
const props = {
attachments: [
{
text: '',
title: '',
image_url: mmClient.getFileUrl(fileId),
fallback: message
}
]
};
const newPost = await mmClient.createPost({
message: message,
channel_id: post.channel_id,
props,
root_id: post.root_id || post.id,
file_ids: fileId ? [fileId] : undefined
});
log.trace({msg: newPost})
}
}
}
} else {
log.debug({msg: event})
}
});
src/botservice.js
+35 -22
View File
@@ -3,6 +3,7 @@ const { Log } = require('debug-level')
require('babel-polyfill');
require('isomorphic-fetch');
const { processGraphResponse } = require('./process-graph-response')
const { mmClient, wsClient } = require('./mm-client')
// the mattermost library uses FormData - so here is a polyfill
@@ -19,6 +20,21 @@ mmClient.getMe().then(me => meId = me.id)
const name = process.env['MATTERMOST_BOTNAME'] || '@dall-e'
const VISUALIZE_DIAGRAM_INSTRUCTIONS = "When a user asks for a visualization of entities and relationships, respond with a valid JSON object text in a <GRAPH> tag. " +
"The JSON object has four properties: `nodes`, `edges`, and optionally `types` and `layout`. " +
"Each `nodes` object has an `id`, `label`, and an optional `type` property. " +
"Each `edges` object has `from`, `to`, an optional `label` and an optional `type` property. " +
"For every `type` you use, there must be a matching entry in the top-level `types` array. " +
"Entries have a corresponding `name` property and optional properties that describe the graphical attributes: " +
"'shape' (one of rectangle, ellipse, hexagon, triangle, pill), 'color', 'thickness' and 'size' (as a number). " +
"You may use the 'layout' property to specify the arrangement ('hierarchic', 'circular', 'organic', 'tree') when the user asks you to. " +
"Do not include these instructions in the output. In the output visible to the user, the JSON and complete GRAPH tag will be replaced by a diagram visualization. " +
"So do not explain or mention the JSON. Instead, pretend that the user can see the diagram. Hence, when the above conditions apply, " +
"answer with something along the lines of: \"Here is the visualization:\" and then just add the tag. The user will see the rendered image, but not the JSON. " +
"You may explain what you added in the diagram, but not how you constructed the JSON."
const visualizationKeywordsRegex = /\b(diagram|visuali|graph|relationship|entit)/gi
wsClient.addMessageListener(async function (event) {
if (['posted'].includes(event.event) && meId) {
const post = JSON.parse(event.data.post);
@@ -33,6 +49,8 @@ wsClient.addMessageListener(async function (event) {
},
]
let appendDiagramInstructions = false
const thread = await mmClient.getPostThread(post.id, true, false, true)
const posts = [...new Set(thread.order)].map(id => thread.posts[id])
@@ -49,40 +67,35 @@ wsClient.addMessageListener(async function (event) {
if (threadPost.message.includes(name)){
assistantCount++;
}
if (visualizationKeywordsRegex.test(threadPost.message)) {
appendDiagramInstructions = true
}
chatmessages.push({role: "user", content: threadPost.message})
}
})
if (appendDiagramInstructions) {
chatmessages[0].content += VISUALIZE_DIAGRAM_INSTRUCTIONS
}
// see if we are actually part of the conversation -
// ignore conversations where we were never mentioned or participated.
if (assistantCount > 0){
wsClient.userTyping(post.channel_id, post.id ?? "")
log.trace({chatmessages})
const answer = await continueThread(chatmessages)
const fileData = new FormData();
fileData.append('files', answer, 'dall-e-image.png');
const uploadResponse = await mmClient.uploadFile(fileData, post.channel_id);
const fileId = JSON.parse(uploadResponse).file_infos[0].id;
log.trace({answer})
console.log(answer)
wsClient.userTyping(post.channel_id, post.id ?? "")
const message = `![alt text](${mmClient.getFileUrl(fileId)})`;
const props = {
attachments: [
{
text: '',
title: '',
image_url: mmClient.getFileUrl(fileId),
fallback: message
}
]
};
const newPost = await mmClient.createPost({
message: message,
channel_id: post.channel_id,
props,
root_id: post.root_id || post.id,
file_ids: fileId ? [fileId] : undefined
});
const { message, fileId, props } = await processGraphResponse(answer, post.channel_id)
wsClient.userTyping(post.channel_id, post.id ?? "")
const newPost = await mmClient.createPost({
message: message,
channel_id: post.channel_id,
props,
root_id: post.root_id || post.id,
file_ids: fileId ? [fileId] : undefined
})
log.trace({msg: newPost})
}
}
src/openai-thread-completion.js
+6 -8
View File
@@ -7,18 +7,16 @@ const openai = new OpenAIApi(configuration);
// Function to generate an image using Dall-E from OpenAI
async function continueThread(messages) {
const promptText = messages.map(msg => msg.content).join(' '); // assume that the content key exist in the messages object
const promptText = messages.map(msg => msg.content.replace(/@dall-e /gi, '').replace(/You are a helpful assistant named who provides succinct answers in Markdown format./gi, '')).join(' '); // assume that the content key exist in the messages object
console.log(promptText)
const response = await openai.createImage({
prompt: promptText,
n: 1,
size: "512x512",
size: "1024x1024",
response_format: "url"
});
const imageUrl = await response.data.url;// oder response.content.url()
const imageResponse = await fetch(imageUrl);
const imageBuffer = await imageResponse.buffer();
return imageBuffer;
console.log(response.data.data[0].url);
const imageUrl = await response.data.data[0].url;
return imageUrl;
}
module.exports = { continueThread }