Solution 1 :

This is caused by anti-aliasing. Over transparent pixels, antialiasing will produce dark pixels, because transparent pixels are actually transparent black pixels.

To workaround that you can force both your v_brush and tmp_canvas (which is kind of useless b.t.w. if it’s the only thing you draw on your visible one), to have their imageSmoothingEnabled set to false, which will prevent it from generating antialiasing when drawing bitmaps:

var drawon_ctx = canvas.getContext("2d"); //is our drawon
var tmp_canvas = createCanvas(canvas.width, canvas.height); //is our tmp
var tmp_ctx = tmp_canvas.ctx;
// var brushSize = 64;
var bs = 64;
var bsh = bs / 2;
var smudgeAmount = 0.25; // values from 0 none to 1 full

// helpers
var doFor = function doFor(count, cb) {
    var i = 0;
    while (i < count && cb(i++) !== true) {
        ;
    }
}; // the ; after while loop is important don't remove
var randI = function randI(min) {
    var max = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : min + (min = 0);
    return Math.random() * (max - min) + min | 0;
};

// simple mouse
var mouse = {
    x: 0,
    y: 0,
    button: false
};

function mouseEvents(e) {
    mouse.x = e.pageX;
    mouse.y = e.pageY;
    mouse.button = e.type === "mousedown" ? true : e.type === "mouseup" ? false : mouse.button;
}
["down", "up", "move"].forEach(function (name) {
    return document.addEventListener("mouse" + name, mouseEvents);
});

// brush gradient for feather
var grad = drawon_ctx.createRadialGradient(bsh, bsh, 0, bsh, bsh, bsh); //center coords/ bsh is half of bs 
grad.addColorStop(0, "black");
grad.addColorStop(1, "rgba(0,0,0,0)");
var v_brush = createCanvas(bs); // our v_brush
// prevent antialising which would produce black pixels over transparent area
v_brush.ctx.imageSmoothingEnabled = false;
// creates an offscreen canvas
function createCanvas(w) {
    var h = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : w;
    var c = document.createElement("canvas");
    c.width = w;
    c.height = h;
    c.ctx = c.getContext("2d");
    return c;
}

// get the brush from source ctx at x,y
function brushFrom(tmp_ctx, x, y) {
    v_brush.ctx.globalCompositeOperation = "source-over";
    v_brush.ctx.globalAlpha = 1;
    v_brush.ctx.drawImage(tmp_canvas, -(x - bsh), -(y - bsh));
    // v_brush.ctx.drawImage(tmp_ctx.canvas, -(x - bsh), -(y - bsh));
    v_brush.ctx.globalCompositeOperation = "destination-in";
    v_brush.ctx.globalAlpha = 1;
    v_brush.ctx.fillStyle = grad;
    v_brush.ctx.fillRect(0, 0, bs, bs);
}

// short cut vars 
var w = canvas.width;
var h = canvas.height;
var cw = w / 2; // center 
var ch = h / 2;
var globalTime;
var lastX;
var lastY;

// update tmp_canvas is size changed
function createBackground() {
    tmp_canvas.width = w;
    tmp_canvas.height = h;
    // tmp_ctx.fillStyle = "white";
    // tmp_ctx.fillRect(0, 0, w, h);
    doFor(64, function () {
        tmp_ctx.fillStyle = "rgb(".concat(randI(255), ",").concat(randI(255), ",").concat(randI(
            255));
        tmp_ctx.fillRect(randI(w), randI(h), randI(10, 100), randI(10, 100));
    });
    // prevent antialising which would produce black pixels over transparent area
    tmp_ctx.imageSmoothingEnabled = false;
}

// main update function
function update(timer) {
    globalTime = timer;
    drawon_ctx.setTransform(1, 0, 0, 1, 0, 0); // reset transform
    drawon_ctx.globalAlpha = 1; // reset alpha
    if (w !== innerWidth || h !== innerHeight) {
        cw = (w = canvas.width = innerWidth) / 2;
        ch = (h = canvas.height = innerHeight) / 2;
        createBackground();
    } else {
        drawon_ctx.clearRect(0, 0, w, h);
    }
    drawon_ctx.drawImage(tmp_canvas, 0, 0);

    // if mouse down then do the smudge for all pixels between last mouse and mouse now
    if (mouse.button) {
        v_brush.ctx.globalAlpha = smudgeAmount;
        var dx = mouse.x - lastX;
        var dy = mouse.y - lastY;
        var dist = Math.sqrt(dx * dx + dy * dy);
        for (var i = 0; i < dist; i += 1) {
            var ni = i / dist;
            brushFrom(tmp_ctx, lastX + dx * ni, lastY + dy * ni);
            ni = (i + 1) / dist;
            tmp_ctx.drawImage(v_brush, lastX + dx * ni - bsh, lastY + dy * ni - bsh);
        }
    } else {
        v_brush.ctx.clearRect(0, 0, bs, bs); /// clear brush if not used
    }

    lastX = mouse.x;
    lastY = mouse.y;
    requestAnimationFrame(update);
}
requestAnimationFrame(update);
canvas {
  position: absolute;
  top: 0px;
  left: 0px;
  /* CSS checkerboard stolen from https://drafts.csswg.org/css-images-4/#example-2de97f53 */
  background: repeating-conic-gradient(rgba(0,0,0,0.1) 0deg 25%, white 0deg 50%);
  background-size: 2em 2em;
}
<canvas id="canvas"></canvas>

Solution 2 :

I think I got it, it is not perfect but I don’t think I can do any better.

What I did is I added another brush which is inverted image of the original brush i.e. it has transparent pixels as solid pixels and vice versa, then after drawing with the original brush on the canvas I draw with the inverted brush using destination-out which is deleting pixels that should be getting transparency as we smudge them into the visible pixels.

I still get some artifacts though.
Also in this case “imageSmoothingEnabled = false” doesn’t seem to have any effect.

If I somehow get those artifacts removed I will update this answer, till then I think this satisfies me.

The example has transparent canvas and the black color is the HTML body color underneath.
I deleted the temp canvas to leave only one canvas as it is simpler to read and probably a bit faster.

I also want to thank @Kaiido for inspiring me and of course @Blindman67 who created the original code, I learned a lot from various answers by @Blindman67, he is the canvas guru to me 😉

 "use strict";

    var ctx = canvas.getContext("2d");
    var bs = 64; //brushSize 
    var bsh = bs / 2;
    var smudgeAmount = 0.25; // values from 0 none to 1 full

    // helpers
    var doFor = function doFor(count, cb) {
        var i = 0;
        while (i < count && cb(i++) !== true) {
            ;
        }
    }; // the ; after while loop is important don't remove
    var randI = function randI(min) {
        var max = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : min + (min = 0);
        return Math.random() * (max - min) + min | 0;
    };

    // simple mouse
    var mouse = {
        x: 0,
        y: 0,
        button: false
    };

    function mouseEvents(e) {
        mouse.x = e.pageX;
        mouse.y = e.pageY;
        mouse.button = e.type === "mousedown" ? true : e.type === "mouseup" ? false : mouse.button;
    }
    ["down", "up", "move"].forEach(function (name) {
        return document.addEventListener("mouse" + name, mouseEvents);
    });

    // brush gradient for feather
    var grad = ctx.createRadialGradient(bsh, bsh, 0, bsh, bsh, bsh); //center coords, bsh is half of bs 
    grad.addColorStop(0, "black");
    grad.addColorStop(1, "rgba(0,0,0,0)");
    var v_brush = createCanvas(bs); // our virtual brush
    var v_brush_inv = createCanvas(bs); // our inverted virtual brush

    v_brush.ctx.imageSmoothingEnabled = false; // seems it doesnt have any effect in this example
    v_brush.ctx.imageSmoothingEnabled = false; // seems it doesnt have any effect in this example

    // creates an offscreen canvas
    function createCanvas(w) {
        var h = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : w;
        var c = document.createElement("canvas");
        c.width = w;
        c.height = h;
        c.ctx = c.getContext("2d");
        return c;
    }

    // get the brush from source ctx at x,y
    function brushFrom(x, y) {

        v_brush_inv.ctx.globalCompositeOperation = "source-over";
        v_brush_inv.ctx.globalAlpha = 1;
        v_brush_inv.ctx.clearRect(0, 0, bs, bs); //clear the inverted brush
        v_brush_inv.ctx.drawImage(canvas, -(x - bsh), -(y - bsh));

        v_brush.ctx.globalCompositeOperation = "source-over";
        v_brush.ctx.globalAlpha = 1;
        v_brush.ctx.drawImage(v_brush_inv, 0, 0); //using the v_brush_inv as it is already drawn

        v_brush_inv.ctx.globalCompositeOperation = "source-out";
        v_brush_inv.ctx.fillStyle = "black";
        v_brush_inv.ctx.fillRect(0, 0, bs, bs);

        v_brush.ctx.globalCompositeOperation = "destination-in";
        v_brush.ctx.fillStyle = grad;
        v_brush.ctx.fillRect(0, 0, bs, bs);

        v_brush_inv.ctx.globalCompositeOperation = "destination-in";
        v_brush_inv.ctx.fillStyle = grad;
        v_brush_inv.ctx.fillRect(0, 0, bs, bs);

    }

    // short cut vars 
    var w = canvas.width;
    var h = canvas.height;
    var cw = w / 2; // center 
    var ch = h / 2;
    var globalTime;
    var lastX;
    var lastY;

    // update canvas if size changed
    function createBackground() {

        canvas.width = w;
        canvas.height = h;
        // commented out the below to test smudging on transparent canvas
        // ctx.fillStyle = "white";
        // ctx.fillRect(0, 0, w, h);
        ctx.imageSmoothingEnabled = false; // seems it doesnt have any effect in this example
        doFor(64, function () {
            ctx.fillStyle = "rgb(".concat(randI(255), ",").concat(randI(255), ",").concat(randI(
                255));
            ctx.fillRect(randI(w), randI(h), randI(10, 100), randI(10, 100));
        });

    }

    // main update function
    function update(timer) {
        globalTime = timer;

        // ctx.setTransform(1, 0, 0, 1, 0, 0); // reset transform
        // ctx.globalAlpha = 1; // reset alpha
        if (w !== innerWidth || h !== innerHeight) {
            cw = (w = canvas.width = innerWidth) / 2;
            ch = (h = canvas.height = innerHeight) / 2;
            createBackground();
        }

        // if mouse down then do the smudge for all pixels between last mouse and mouse now
        if (mouse.button) {
            v_brush.ctx.globalAlpha = smudgeAmount;
            v_brush_inv.ctx.globalAlpha = smudgeAmount;
            var dx = mouse.x - lastX;
            var dy = mouse.y - lastY;
            var dist = Math.sqrt(dx * dx + dy * dy);
            for (var i = 0; i < dist; i += 1) {
                var ni = i / dist;
                brushFrom(lastX + dx * ni, lastY + dy * ni);
                ni = (i + 1) / dist;

                //we draw the v_brush containing shifted layered pixels from the 
                ctx.globalCompositeOperation = "source-over";
                ctx.drawImage(v_brush, lastX + dx * ni - bsh, lastY + dy * ni - bsh);

                // then we draw inverted brush using destination-out to remove pixels that should be getting transparency as we smudge them into the ctx
                ctx.globalCompositeOperation = "destination-out";
                ctx.drawImage(v_brush_inv, lastX + dx * ni - bsh, lastY + dy * ni - bsh);

            }
        } else {
            v_brush.ctx.clearRect(0, 0, bs, bs); /// clear brush if not used
        }

        lastX = mouse.x;
        lastY = mouse.y;
        requestAnimationFrame(update);
    }
    requestAnimationFrame(update);
    canvas {
        position: absolute;
        top: 0px;
        left: 0px;
    }

    body {
        background-color: rgb(0, 0, 0);
    }
<body>
    <canvas id="canvas"></canvas>
</body>

Problem :

I want the same effect on canvas as in this answer by Blindman67

https://stackoverflow.com/a/45755177/5651569

but with transparent background i.e. have the two lines commented out:

  //background.ctx.fillStyle = "white";
  //background.ctx.fillRect(0,0,w,h);

when you do that the smeared pixels get weird colors.
How to achieve the original effect as if the white background is there but with transparent background?

I would like a clean effect where transparent pixels get smeared into visible pixels

enter image description here

but I am getting strange dark colored pixels appearing on edges that get smeared further

enter image description here

<canvas id="canvas"></canvas>


<style>
    canvas {
        position: absolute;
        top: 0px;
        left: 0px;
    }
</style>


<script>
    "use strict";

    var drawon_ctx = canvas.getContext("2d"); //is our drawon
    var tmp_canvas = createCanvas(canvas.width, canvas.height); //is our tmp
    var tmp_ctx = tmp_canvas.ctx;
    // var brushSize = 64;
    var bs = 64;
    var bsh = bs / 2;
    var smudgeAmount = 0.25; // values from 0 none to 1 full

    // helpers
    var doFor = function doFor(count, cb) {
        var i = 0;
        while (i < count && cb(i++) !== true) {
            ;
        }
    }; // the ; after while loop is important don't remove
    var randI = function randI(min) {
        var max = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : min + (min = 0);
        return Math.random() * (max - min) + min | 0;
    };

    // simple mouse
    var mouse = {
        x: 0,
        y: 0,
        button: false
    };

    function mouseEvents(e) {
        mouse.x = e.pageX;
        mouse.y = e.pageY;
        mouse.button = e.type === "mousedown" ? true : e.type === "mouseup" ? false : mouse.button;
    }
    ["down", "up", "move"].forEach(function (name) {
        return document.addEventListener("mouse" + name, mouseEvents);
    });

    // brush gradient for feather
    var grad = drawon_ctx.createRadialGradient(bsh, bsh, 0, bsh, bsh, bsh); //center coords/ bsh is half of bs 
    grad.addColorStop(0, "black");
    grad.addColorStop(1, "rgba(0,0,0,0)");
    var v_brush = createCanvas(bs); // our v_brush

    // creates an offscreen canvas
    function createCanvas(w) {
        var h = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : w;
        var c = document.createElement("canvas");
        c.width = w;
        c.height = h;
        c.ctx = c.getContext("2d");
        return c;
    }

    // get the brush from source ctx at x,y
    function brushFrom(tmp_ctx, x, y) {
        v_brush.ctx.globalCompositeOperation = "source-over";
        v_brush.ctx.globalAlpha = 1;
        v_brush.ctx.drawImage(tmp_canvas, -(x - bsh), -(y - bsh));
        // v_brush.ctx.drawImage(tmp_ctx.canvas, -(x - bsh), -(y - bsh));
        v_brush.ctx.globalCompositeOperation = "destination-in";
        v_brush.ctx.globalAlpha = 1;
        v_brush.ctx.fillStyle = grad;
        v_brush.ctx.fillRect(0, 0, bs, bs);
    }

    // short cut vars 
    var w = canvas.width;
    var h = canvas.height;
    var cw = w / 2; // center 
    var ch = h / 2;
    var globalTime;
    var lastX;
    var lastY;

    // update tmp_canvas is size changed
    function createBackground() {
        tmp_canvas.width = w;
        tmp_canvas.height = h;
        // tmp_ctx.fillStyle = "white";
        // tmp_ctx.fillRect(0, 0, w, h);
        doFor(64, function () {
            tmp_ctx.fillStyle = "rgb(".concat(randI(255), ",").concat(randI(255), ",").concat(randI(
                255));
            tmp_ctx.fillRect(randI(w), randI(h), randI(10, 100), randI(10, 100));
        });
    }

    // main update function
    function update(timer) {
        globalTime = timer;
        drawon_ctx.setTransform(1, 0, 0, 1, 0, 0); // reset transform
        drawon_ctx.globalAlpha = 1; // reset alpha
        if (w !== innerWidth || h !== innerHeight) {
            cw = (w = canvas.width = innerWidth) / 2;
            ch = (h = canvas.height = innerHeight) / 2;
            createBackground();
        } else {
            drawon_ctx.clearRect(0, 0, w, h);
        }
        drawon_ctx.drawImage(tmp_canvas, 0, 0);

        // if mouse down then do the smudge for all pixels between last mouse and mouse now
        if (mouse.button) {
            v_brush.ctx.globalAlpha = smudgeAmount;
            var dx = mouse.x - lastX;
            var dy = mouse.y - lastY;
            var dist = Math.sqrt(dx * dx + dy * dy);
            for (var i = 0; i < dist; i += 1) {
                var ni = i / dist;
                brushFrom(tmp_ctx, lastX + dx * ni, lastY + dy * ni);
                ni = (i + 1) / dist;
                tmp_ctx.drawImage(v_brush, lastX + dx * ni - bsh, lastY + dy * ni - bsh);
            }
        } else {
            v_brush.ctx.clearRect(0, 0, bs, bs); /// clear brush if not used
        }

        lastX = mouse.x;
        lastY = mouse.y;
        requestAnimationFrame(update);
    }
    requestAnimationFrame(update);

Comments

Comment posted by Alsat

Hi Kaiido, that is a very nice tip, very useful, but is there a way for those transparent pixels to smudge into the visible pixels when you move the mouse from the transparent area into the visible pixels, so that the transparent pixels push those visible ones like in the first image I pasted from the original effect?

Comment posted by Kaiido

@Alsat not with this code no. It works by mixing the colors together with a slight offset. Mixing transparent pixel with any other color will result in the other color unchanged (unless antialising kicks in to produce darker pixels). You’d need another round of compositing just for transparent pixels, but you might be better starting again from scratch.

Comment posted by Alsat

I was afraid this has to be a different code, much more complex, probably something with pixel level manipulation and getImageData(). My question still stands “How to shift pixels on HTML canvas for smudge effect with transparent background?” I now need some guidance/direction about how to start writing it from scratch, some sample code. I asked this question because I couldn’t find any on the Internet.

Comment posted by Kaiido

Bit that makes it too broad a question for stavkoverflow. Asking to fix your code is fine, asking to do it from scratch is not, even just for guidance, since there could be many ways to do so.

Comment posted by Kaiido

But your alpha channel doesn’t have the same weight as the other colors. Mixing forever the “white based” version, or if there is no transparent pixel at all at the beginning, you’d end up with a shade of gray, which is expected when mixing colors together. Here you’ll end with a(n almost) transparent layer.

By