Drawig disappears on canvas on mouseup : Fabric JS : ctx

I have made a canvas with Fabric JS. I loaded an image on canvas with Fabric Js. Now, I want to do some drawing with using ctx object. I can successfully draw on canvas, but when ever I leaves the mouse, all drawing disappears.

Here is the code:

let el = document.getElementById('canvas');
let ctx = el.getContext('2d');
// canvas variable is Fabric JS canvas.

function simpleDraw() {
    let isDrawing;
    canvas.on('mouse:down', function (o) {
        let p1 = canvas.getPointer(o.e);
        isDrawing = true;
        ctx.lineWidth = 10;
        ctx.lineJoin = ctx.lineCap = 'round';
        ctx.moveTo(p1.x, p1.y);
   });

   canvas.on('mouse:move', function (o) {
      let p2 = canvas.getPointer(o.e);
      if (isDrawing) {
          ctx.lineTo(p2.x, p2.y);
          ctx.stroke();
      }
   });

  canvas.on('mouse:up', function (o) {
      isDrawing = false;
      canvas.requestRenderAll();
  });

}

I have to do drawing with ctx only, beacause I have to do some animation stuff. I cannot use Fabric JS freeDrawingBrush.

Here is problem video link