Intercepting and Customizing OpenAI Chat Completion Streams

ExpressJS example

app.post('/completions', async (req, res) => {
  try {
    // an array of messages (system, user, assistant)
    const { messages } = req.body

    // stream request
    const stream = await openai.chat.completions.create({
      model: 'gpt-4o',
      messages,
      stream: true,
      // any other parameters
    })

    // set the content type to text/plain
    res.header('Content-Type', 'text/plain')

    // send analyzing status
    res.write('{"event":"analyzing"}\n')

    // any required processing

    // store the assistant answer
    let assistantAnswer = ''

    // read the stream
    const readableStream = stream.toReadableStream()
    const reader = readableStream.getReader()
    const decoder = new TextDecoder()

    // read the stream
    while (true) {
      // read the value from the stream
      const { done, value } = await reader.read()
      if (done) break

      // decode the value
      const payloads = decoder.decode(value, { stream: true })

      let response = ''

      // split the payloads by newline
      for (const payload of payloads.split('\n')) {
        // check if the payload is empty
        if (payload === '') {
          break
        }

        // check if the payload is not empty
        if (payload) {
          const data = JSON.parse(payload)
          const text = data.choices[0].delta?.content || ''
          if (text) {
            assistantAnswer += text

            // stringify the response and add a newline
            response =
              JSON.stringify({
                event: 'answering',
                content: text
              }) + '\n'
          }
        }
      }

      // send the response to the client
      res.write(response)
    }

    // any required post-processing

    // send event completed
    res.write('{"event":"completed"}\n')

    // end the stream
    res.end()
  } catch (error) {
    console.error(error)
    return res.status(500).json({ 
      error: 'Internal server error',
      message: error.toString()
    })
  }
})