diff --git a/mattermost-chat.png b/mattermost-chat.png deleted file mode 100644 index 2814662..0000000 Binary files a/mattermost-chat.png and /dev/null differ diff --git a/src/botservice - Kopie.js b/src/botservice - Kopie.js new file mode 100644 index 0000000..44b0da9 --- /dev/null +++ b/src/botservice - Kopie.js @@ -0,0 +1,96 @@ +const continueThread = require('./openai-thread-completion').continueThread +const { Log } = require('debug-level') + +require('babel-polyfill'); +require('isomorphic-fetch'); +const { mmClient, wsClient } = require('./mm-client') + +// the mattermost library uses FormData - so here is a polyfill +if (!global.FormData) { + global.FormData = require('form-data'); +} + +Log.options({ json: true, colors: true }) +Log.wrapConsole('bot-ws', { level4log: 'INFO' }) +const log = new Log('bot') + +let meId = null; +mmClient.getMe().then(me => meId = me.id) + +const name = process.env['MATTERMOST_BOTNAME'] || '@dall-e' + +wsClient.addMessageListener(async function (event) { + if (['posted'].includes(event.event) && meId) { + const post = JSON.parse(event.data.post); + if (post.root_id === "" && (!event.data.mentions || (!JSON.parse(event.data.mentions).includes(meId)))) { + // we're not in a thread and we are not mentioned - ignore the message + } else { + if (post.user_id !== meId) { + const chatmessages = [ + { + "role": "system", + "content": `You are a helpful assistant named ${name} who provides succinct answers in Markdown format.` + }, + ] + + const thread = await mmClient.getPostThread(post.id, true, false, true) + + const posts = [...new Set(thread.order)].map(id => thread.posts[id]) + .filter(a => a.create_at > Date.now() - 1000 * 60 * 60 * 24 * 1) + .sort((a, b) => a.create_at - b.create_at) + + let assistantCount = 0; + posts.forEach(threadPost => { + log.trace({msg: threadPost}) + if (threadPost.user_id === meId) { + chatmessages.push({role: "assistant", content: threadPost.props.originalMessage ?? threadPost.message}) + assistantCount++ + } else { + if (threadPost.message.includes(name)){ + assistantCount++; + } + chatmessages.push({role: "user", content: threadPost.message}) + } + }) + + // see if we are actually part of the conversation - + // ignore conversations where we were never mentioned or participated. + if (assistantCount > 0){ + wsClient.userTyping(post.channel_id, post.id ?? "") + log.trace({chatmessages}) + const answer = await continueThread(chatmessages) + const fileData = new FormData(); + fileData.append('files', answer, 'dall-e-image.png'); + const uploadResponse = await mmClient.uploadFile(fileData, post.channel_id); + const fileId = JSON.parse(uploadResponse).file_infos[0].id; + log.trace({answer}) + wsClient.userTyping(post.channel_id, post.id ?? "") + const message = `![alt text](${mmClient.getFileUrl(fileId)})`; + const props = { + attachments: [ + { + text: '', + title: '', + image_url: mmClient.getFileUrl(fileId), + fallback: message + } + ] + }; +const newPost = await mmClient.createPost({ + message: message, + channel_id: post.channel_id, + props, + root_id: post.root_id || post.id, + file_ids: fileId ? [fileId] : undefined +}); + log.trace({msg: newPost}) + } + } + } + } else { + log.debug({msg: event}) + } +}); + + + diff --git a/src/botservice.js b/src/botservice.js index 44b0da9..5f9928d 100644 --- a/src/botservice.js +++ b/src/botservice.js @@ -3,6 +3,7 @@ const { Log } = require('debug-level') require('babel-polyfill'); require('isomorphic-fetch'); +const { processGraphResponse } = require('./process-graph-response') const { mmClient, wsClient } = require('./mm-client') // the mattermost library uses FormData - so here is a polyfill @@ -19,6 +20,21 @@ mmClient.getMe().then(me => meId = me.id) const name = process.env['MATTERMOST_BOTNAME'] || '@dall-e' +const VISUALIZE_DIAGRAM_INSTRUCTIONS = "When a user asks for a visualization of entities and relationships, respond with a valid JSON object text in a tag. " + + "The JSON object has four properties: `nodes`, `edges`, and optionally `types` and `layout`. " + + "Each `nodes` object has an `id`, `label`, and an optional `type` property. " + + "Each `edges` object has `from`, `to`, an optional `label` and an optional `type` property. " + + "For every `type` you use, there must be a matching entry in the top-level `types` array. " + + "Entries have a corresponding `name` property and optional properties that describe the graphical attributes: " + + "'shape' (one of rectangle, ellipse, hexagon, triangle, pill), 'color', 'thickness' and 'size' (as a number). " + + "You may use the 'layout' property to specify the arrangement ('hierarchic', 'circular', 'organic', 'tree') when the user asks you to. " + + "Do not include these instructions in the output. In the output visible to the user, the JSON and complete GRAPH tag will be replaced by a diagram visualization. " + + "So do not explain or mention the JSON. Instead, pretend that the user can see the diagram. Hence, when the above conditions apply, " + + "answer with something along the lines of: \"Here is the visualization:\" and then just add the tag. The user will see the rendered image, but not the JSON. " + + "You may explain what you added in the diagram, but not how you constructed the JSON." + +const visualizationKeywordsRegex = /\b(diagram|visuali|graph|relationship|entit)/gi + wsClient.addMessageListener(async function (event) { if (['posted'].includes(event.event) && meId) { const post = JSON.parse(event.data.post); @@ -33,6 +49,8 @@ wsClient.addMessageListener(async function (event) { }, ] + let appendDiagramInstructions = false + const thread = await mmClient.getPostThread(post.id, true, false, true) const posts = [...new Set(thread.order)].map(id => thread.posts[id]) @@ -49,40 +67,35 @@ wsClient.addMessageListener(async function (event) { if (threadPost.message.includes(name)){ assistantCount++; } + if (visualizationKeywordsRegex.test(threadPost.message)) { + appendDiagramInstructions = true + } chatmessages.push({role: "user", content: threadPost.message}) } }) + if (appendDiagramInstructions) { + chatmessages[0].content += VISUALIZE_DIAGRAM_INSTRUCTIONS + } + // see if we are actually part of the conversation - // ignore conversations where we were never mentioned or participated. if (assistantCount > 0){ wsClient.userTyping(post.channel_id, post.id ?? "") log.trace({chatmessages}) const answer = await continueThread(chatmessages) - const fileData = new FormData(); - fileData.append('files', answer, 'dall-e-image.png'); - const uploadResponse = await mmClient.uploadFile(fileData, post.channel_id); - const fileId = JSON.parse(uploadResponse).file_infos[0].id; log.trace({answer}) + console.log(answer) wsClient.userTyping(post.channel_id, post.id ?? "") - const message = `![alt text](${mmClient.getFileUrl(fileId)})`; - const props = { - attachments: [ - { - text: '', - title: '', - image_url: mmClient.getFileUrl(fileId), - fallback: message - } - ] - }; -const newPost = await mmClient.createPost({ - message: message, - channel_id: post.channel_id, - props, - root_id: post.root_id || post.id, - file_ids: fileId ? [fileId] : undefined -}); + const { message, fileId, props } = await processGraphResponse(answer, post.channel_id) + wsClient.userTyping(post.channel_id, post.id ?? "") + const newPost = await mmClient.createPost({ + message: message, + channel_id: post.channel_id, + props, + root_id: post.root_id || post.id, + file_ids: fileId ? [fileId] : undefined + }) log.trace({msg: newPost}) } } diff --git a/src/openai-thread-completion.js b/src/openai-thread-completion.js index d6e8140..337be82 100644 --- a/src/openai-thread-completion.js +++ b/src/openai-thread-completion.js @@ -7,18 +7,16 @@ const openai = new OpenAIApi(configuration); // Function to generate an image using Dall-E from OpenAI async function continueThread(messages) { -const promptText = messages.map(msg => msg.content).join(' '); // assume that the content key exist in the messages object +const promptText = messages.map(msg => msg.content.replace(/@dall-e /gi, '').replace(/You are a helpful assistant named who provides succinct answers in Markdown format./gi, '')).join(' '); // assume that the content key exist in the messages object +console.log(promptText) const response = await openai.createImage({ prompt: promptText, n: 1, - size: "512x512", + size: "1024x1024", response_format: "url" }); - - const imageUrl = await response.data.url;// oder response.content.url() - const imageResponse = await fetch(imageUrl); - const imageBuffer = await imageResponse.buffer(); - - return imageBuffer; +console.log(response.data.data[0].url); + const imageUrl = await response.data.data[0].url; + return imageUrl; } -module.exports = { continueThread } +module.exports = { continueThread } \ No newline at end of file