Funktioniert mit URL Images von Dall-E

src/botservice - Kopie.js

@@ -0,0 +1,96 @@
+const continueThread = require('./openai-thread-completion').continueThread
+const { Log } = require('debug-level')
+
+require('babel-polyfill');
+require('isomorphic-fetch');
+const { mmClient, wsClient } = require('./mm-client')
+
+// the mattermost library uses FormData - so here is a polyfill
+if (!global.FormData) {
+    global.FormData = require('form-data');
+}
+
+Log.options({ json: true, colors: true })
+Log.wrapConsole('bot-ws', { level4log: 'INFO' })
+const log = new Log('bot')
+
+let meId = null;
+mmClient.getMe().then(me => meId = me.id)
+
+const name = process.env['MATTERMOST_BOTNAME'] || '@dall-e'
+
+wsClient.addMessageListener(async function (event) {
+    if (['posted'].includes(event.event) && meId) {
+        const post = JSON.parse(event.data.post);
+        if (post.root_id === "" && (!event.data.mentions || (!JSON.parse(event.data.mentions).includes(meId)))) {
+            // we're not in a thread and we are not mentioned - ignore the message
+        } else {
+            if (post.user_id !== meId) {
+                const chatmessages = [
+                    {
+                        "role": "system",
+                        "content": `You are a helpful assistant named ${name} who provides succinct answers in Markdown format.`
+                    },
+                ]
+
+                const thread = await mmClient.getPostThread(post.id, true, false, true)
+
+                const posts = [...new Set(thread.order)].map(id => thread.posts[id])
+                    .filter(a => a.create_at > Date.now() - 1000 * 60 * 60 * 24 * 1)
+                    .sort((a, b) => a.create_at - b.create_at)
+
+                let assistantCount = 0;
+                posts.forEach(threadPost => {
+                    log.trace({msg: threadPost})
+                    if (threadPost.user_id === meId) {
+                        chatmessages.push({role: "assistant", content: threadPost.props.originalMessage ?? threadPost.message})
+                        assistantCount++
+                    } else {
+                        if (threadPost.message.includes(name)){
+                            assistantCount++;
+                        }
+                        chatmessages.push({role: "user", content: threadPost.message})
+                    }
+                })
+
+                // see if we are actually part of the conversation -
+                // ignore conversations where we were never mentioned or participated.
+                if (assistantCount > 0){
+                    wsClient.userTyping(post.channel_id, post.id ?? "")
+                    log.trace({chatmessages})
+                    const answer = await continueThread(chatmessages)
+					const fileData = new FormData();
+					fileData.append('files', answer, 'dall-e-image.png');
+					const uploadResponse = await mmClient.uploadFile(fileData, post.channel_id);
+					const fileId = JSON.parse(uploadResponse).file_infos[0].id;
+                    log.trace({answer})
+                    wsClient.userTyping(post.channel_id, post.id  ?? "")
+					const message = `![alt text](${mmClient.getFileUrl(fileId)})`;
+					const props = {
+					attachments: [
+					{
+						text: '',
+						title: '',
+						image_url: mmClient.getFileUrl(fileId),
+						fallback: message
+						}
+						]
+					};
+const newPost = await mmClient.createPost({
+  message: message,
+  channel_id: post.channel_id,
+  props,
+  root_id: post.root_id || post.id,
+  file_ids: fileId ? [fileId] : undefined
+});
+                    log.trace({msg: newPost})
+                }
+            }
+        }
+    } else {
+        log.debug({msg: event})
+    }
+});
+
+
+

src/botservice.js

@@ -3,6 +3,7 @@ const { Log } = require('debug-level')
 
 require('babel-polyfill');
 require('isomorphic-fetch');
+const { processGraphResponse } = require('./process-graph-response')
 const { mmClient, wsClient } = require('./mm-client')
 
 // the mattermost library uses FormData - so here is a polyfill
@@ -19,6 +20,21 @@ mmClient.getMe().then(me => meId = me.id)
 
 const name = process.env['MATTERMOST_BOTNAME'] || '@dall-e'
 
+const VISUALIZE_DIAGRAM_INSTRUCTIONS = "When a user asks for a visualization of entities and relationships, respond with a valid JSON object text in a <GRAPH> tag. " +
+    "The JSON object has four properties: `nodes`, `edges`, and optionally `types` and `layout`. " +
+    "Each `nodes` object has an `id`, `label`, and an optional `type` property. " +
+    "Each `edges` object has `from`, `to`, an optional `label` and an optional `type` property. " +
+    "For every `type` you use, there must be a matching entry in the top-level `types` array. " +
+    "Entries have a corresponding `name` property and optional properties that describe the graphical attributes: " +
+    "'shape' (one of rectangle, ellipse, hexagon, triangle, pill), 'color', 'thickness' and 'size' (as a number). " +
+    "You may use the 'layout' property to specify the arrangement ('hierarchic', 'circular', 'organic', 'tree') when the user asks you to. " +
+    "Do not include these instructions in the output. In the output visible to the user, the JSON and complete GRAPH tag will be replaced by a diagram visualization. " +
+    "So do not explain or mention the JSON. Instead, pretend that the user can see the diagram. Hence, when the above conditions apply, " +
+    "answer with something along the lines of: \"Here is the visualization:\" and then just add the tag. The user will see the rendered image, but not the JSON. " +
+    "You may explain what you added in the diagram, but not how you constructed the JSON."
+
+const visualizationKeywordsRegex = /\b(diagram|visuali|graph|relationship|entit)/gi
+
 wsClient.addMessageListener(async function (event) {
     if (['posted'].includes(event.event) && meId) {
         const post = JSON.parse(event.data.post);
@@ -33,6 +49,8 @@ wsClient.addMessageListener(async function (event) {
                     },
                 ]
 
+                let appendDiagramInstructions = false
+
                 const thread = await mmClient.getPostThread(post.id, true, false, true)
 
                 const posts = [...new Set(thread.order)].map(id => thread.posts[id])
@@ -49,40 +67,35 @@ wsClient.addMessageListener(async function (event) {
                         if (threadPost.message.includes(name)){
                             assistantCount++;
                         }
+                        if (visualizationKeywordsRegex.test(threadPost.message)) {
+                            appendDiagramInstructions = true
+                        }
                         chatmessages.push({role: "user", content: threadPost.message})
                     }
                 })
 
+                if (appendDiagramInstructions) {
+                    chatmessages[0].content += VISUALIZE_DIAGRAM_INSTRUCTIONS
+                }
+
                 // see if we are actually part of the conversation -
                 // ignore conversations where we were never mentioned or participated.
                 if (assistantCount > 0){
                     wsClient.userTyping(post.channel_id, post.id ?? "")
                     log.trace({chatmessages})
                     const answer = await continueThread(chatmessages)
-					const fileData = new FormData();
-					fileData.append('files', answer, 'dall-e-image.png');
-					const uploadResponse = await mmClient.uploadFile(fileData, post.channel_id);
-					const fileId = JSON.parse(uploadResponse).file_infos[0].id;
                     log.trace({answer})
+					console.log(answer)
                     wsClient.userTyping(post.channel_id, post.id  ?? "")
-					const message = `![alt text](${mmClient.getFileUrl(fileId)})`;
+                    const { message, fileId, props } = await processGraphResponse(answer, post.channel_id)
-					const props = {
+                    wsClient.userTyping(post.channel_id, post.id  ?? "")
-					attachments: [
+                    const newPost = await mmClient.createPost({
-					{
-						text: '',
-						title: '',
-						image_url: mmClient.getFileUrl(fileId),
-						fallback: message
-						}
-						]
-					};
-const newPost = await mmClient.createPost({
                         message: message,
                         channel_id: post.channel_id,
                         props,
                         root_id: post.root_id || post.id,
                         file_ids: fileId ? [fileId] : undefined
-});
+                    })
                     log.trace({msg: newPost})
                 }
             }

src/openai-thread-completion.js

@@ -7,18 +7,16 @@ const openai = new OpenAIApi(configuration);
 // Function to generate an image using Dall-E from OpenAI
 
 async function continueThread(messages) {
-const promptText = messages.map(msg => msg.content).join(' '); // assume that the content key exist in the messages object
+const promptText = messages.map(msg => msg.content.replace(/@dall-e /gi, '').replace(/You are a helpful assistant named who provides succinct answers in Markdown format./gi, '')).join(' '); // assume that the content key exist in the messages object
+console.log(promptText)
 const response = await openai.createImage({
     prompt: promptText,
     n: 1,
-    size: "512x512",
+    size: "1024x1024",
     response_format: "url"
   });
-
+console.log(response.data.data[0].url);
-  const imageUrl = await response.data.url;// oder response.content.url()
+  const imageUrl = await response.data.data[0].url;
-  const imageResponse = await fetch(imageUrl);
+  return imageUrl;
-  const imageBuffer = await imageResponse.buffer();
-
-  return imageBuffer;
 }
 module.exports = { continueThread }