How to render Streamable image on React coming from the FastAPI

Question

I want to render a StreamingResponse image on React from FastAPI. The image is in the form of numpy array which is of cv2 type of object

@app.post("/predict")
async def root(file: UploadFile = File(...)):
    global model
    global store_coordinates
    global store_faces
    global store_mesh

    content = await file.read()
    nparr = np.fromstring(content, np.uint8)
    bg_img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
    ......................
    for i in range(len(store_coordinates)):
            x, y, w, h = store_coordinates[i]
            bg_img [b:b + d, a:a + c] = store_mesh[i]
    
    res,im_png = cv2.imencode(".png", bg_img)
    return StreamingResponse(io.BytesIO(im_png.tobytes()), media_type="image/png")

Here I have created an API endpoint in which the uploaded image is received using post request and an StreamableResponse(Image) is returned. How can I render this returned response on React frontend

React Code:

import React, { Component } from "react";
import axios from "axios";

class Detect extends Component {
  state = {
    title: "",
    content: "",
    image: null,
  };

  handleChange = (e) => {
    this.setState({
      [e.target.id]: e.target.value,
    });
  };

  handleImageChange = (e) => {
    this.setState({
      image: e.target.files[0],
    });
  };

  
  handleSubmit = (e) => {
    e.preventDefault();
    console.log(this.state);
    let form_data = new FormData();
    form_data.append("image", this.state.image, this.state.image.name);
    let url = "http://127.0.0.1:8000/predict";
    axios
      .post(url, form_data, {
        headers: {
          "content-type": "multipart/form-data",
        },
      })
      .then((res) => {
        console.log(res.data);
      })
      .catch((err) => console.log(err));
  };

  render() {
    return (
      <div className="App">
        <form onSubmit={this.handleSubmit}>
          <p>
            <input
              type="file"
              id="image"
              accept="image/png, image/jpeg"
              onChange={this.handleImageChange}
              required
            />
          </p>
          <input type="submit" />
        </form>
        <div id="image-render">
          <img></img>
        </div>
      </div>
    );
  }
}

export default Detect;

I would like to render the returned image in the div tag which has the id rendered-image

Edit -

Answer 1

You could either encode the image data to base64 format and return it, which can then be used to display the image in HTML as shown here (e.g., <img src="data:image/png;base64, ...), or send the raw bytes, as you already do, and convert them into base64 format (using btoa(), String.fromCharCode() and Uint8Array) or Blob object (and then call URL.createObjectURL() to convert the blob into a URL.) on client side. The below examples show how to achieve the last two methods, using Axios library, which you seem to be using in your project, as well as Fetch API.

Using Axios

Option 1 - Convert raw image bytes into Blob

axios({
        method: 'POST',
        url: '/upload',
        data: formData,
        headers: {
            'Content-Type': 'multipart/form-data'
        },
        responseType: "blob"
    })
    .then(response => {
        var blobURL = URL.createObjectURL(response.data);
        var image = document.getElementById("myImage");
        image.onload = function(){
            URL.revokeObjectURL(this.src); // release the blob URL once the image is loaded
        }
        image.src = blobURL;
    })
    .catch(error => {
        console.error(error);
    });

Option 2 - Convert raw image bytes into Base64 encoded string

axios({
        method: 'POST',
        url: '/predict',
        data: formData,
        headers: {
            'Content-Type': 'multipart/form-data'
        },
        responseType: "arraybuffer"
    })
    .then(response => {
        base64string = btoa(String.fromCharCode(...new Uint8Array(response.data)))
        contentType = response.headers['content-type']
        return base64string;
    })
    .then(base64string => {
        var image = document.getElementById("myImage");
        image.src = "data:" + contentType + ";base64," + base64string;
    })
    .catch(error => {
        console.error(error);
    });

Using Fetch API

Option 1 - Convert raw image bytes into Blob

fetch('/predict', {
        method: 'POST',
        body: formData,
    })
    .then(response => response.blob())
    .then(blob => {
        var blobURL = URL.createObjectURL(blob);
        var image = document.getElementById("myImage");
        image.onload = function(){
            URL.revokeObjectURL(this.src); // release the blob URL once the image is loaded
        }
        image.src = blobURL;
    })
    .catch(error => {
        console.error(error);
    });

Option 2 - Convert raw image bytes into Base64 encoded string

fetch('/predict', {
        method: 'POST',
        body: formData,
    })
    .then(response => {
        contentType = response.headers.get('content-type')
        return response.arrayBuffer();
    })
    .then(arrayBuffer => {
        base64string = btoa(String.fromCharCode(...new Uint8Array(arrayBuffer)))
        var image = document.getElementById("myImage");
        image.src = "data:" + contentType + ";base64," + base64string;
    })
    .catch(error => {
        console.error(error);
    });

Remember to define an <img> tag in your HTML file, where you wish to display the image:

 <img id="myImage" src="">