JavaScript (ES6), 383 377 354 bytes
f=s=>{d=document,i=new Image,i.src=s,i.onload=$=>{c=d.createElement`canvas`,x=c.getContext`2d`,c.width=w=i.width,c.height=h=i.height,x.drawImage(i,0,0),D=x.getImageData(0,0,w,h),t=D.data,t.set([].concat(...[...t].map((v,i,T)=>i%4?[,,,0]:T.slice(i,i+4)).sort((a,b)=>a.some((v,i)=>k=v-b[i])&&k)).slice(12*w*h)),x.putImageData(D,0,0),d.body.appendChild(c)}}
Runnable demo:
f=function(s) {l=[i=new Image].slice,i.crossOrigin="anonymous",i.src=s,i.onload=function($){d=document,c=d.createElement`canvas`,c.width=w=i.width,c.height=h=i.height,x=c.getContext`2d`,x.drawImage(i,0,0),D=x.getImageData(0,0,w,h),t=D.data,t.set([].concat.apply([],l.call(t).map((v,i)=>i%4?[0,0,0,0]:l.call(t,i,i+4)).sort((a,b)=>a.reduce((A,v,i)=>A||v-b[i],0))).slice(3*t.length)),x.putImageData(D,0,0),d.body.appendChild(c)}}
$("img").click(function() { f(this.src); })
canvas { padding-right: 4px; } img { cursor: pointer; }
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script> <img src="https://upload.wikimedia.org/wikipedia/commons/thumb/f/f4/The_Scream.jpg/114px-The_Scream.jpg"> <img src="https://upload.wikimedia.org/wikipedia/commons/thumb/6/67/Claude_Monet_La_Grenouill%C3%A9re.jpg/193px-Claude_Monet_La_Grenouill%C3%A9re.jpg"> <img src="https://upload.wikimedia.org/wikipedia/commons/thumb/c/cc/Grant_Wood_-_American_Gothic_-_Google_Art_Project.jpg/120px-Grant_Wood_-_American_Gothic_-_Google_Art_Project.jpg"><br>
How this code works is to use getImageData to get an array of the form
[R,G,B,A,
R,G,B,A,
R,G,B,A,
...]
And map it to an array of the form
[[R,G,B,A],[0,0,0,0],[0,0,0,0],[0,0,0,0],
[R,G,B,A],[0,0,0,0],[0,0,0,0],[0,0,0,0],
[R,G,B,A],[0,0,0,0],[0,0,0,0],[0,0,0,0],
...]
So that the R values are mapped to arrays of the RGBA set, and the B, G, and A values turn into minimum-value zero arrays. When we sort this array, all the [0,0,0,0] arrays sort to the bottom and the real-value arrays sort normally at the top:
[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,0,0,0],
[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,0,0,0],
[0,0,0,0],..., [R,G,B,A],[R,G,B,A],[R,G,B,A],...]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
extract & flatten these sorted pixels
We skim the top fourth of the array (to lose the empty values we created), flatten it with [].concat.apply, and end up with an array of the first form again, but this time, it's sorted.
Slightly de-golfed with whitespace and comments:
f=s=>{
// first, load image, then do everything else onload
i=new Image,
i.src = s,
i.onload=$=>{
// set up the canvas
d=document,
c=d.createElement`canvas`,
w=c.width=i.width,
h=c.height=i.height,
x=c.getContext`2d`,
// draw image to canvas and capture pixel data
x.drawImage(i,0,0),
D=x.getImageData(0,0,w,h),
t=D.data,
// set pixel data to...
t.set(
// the flattened array...
[].concat(...
// that is a mapping of the pixel data...
[...t].map(
// that clumps RGBA families into subarrays
// by replacing every fourth value with [R,G,B,A]
// and all other values to [0,0,0,0]...
(v,i,T)=>i%4?[,,,0]:T.slice(i,i+4)
)
// and is then sorted...
.sort(
// by reducing each array to a positive, negative, or zero
// by comparing R,G,B,& A until the first nonzero difference
(a,b)=>a.some((v,i)=>k=v-b[i])&&k
)
)
// then eliminate the low-sorted empty [0,0,0,0] values we created,
// leaving only the top fourth, with real values
// (note that 3*4*w*h is the same as 3*t.length)
.slice(3*4*w*h)
),
// now that `t` is set, store `D` in canvas
x.putImageData(D,0,0),
// show canvas
d.body.appendChild(c)
}
}
Note that most browsers may fail to run this code for large images, because it passes a huge number of arguments into [].concat. When the browser environment does not allow sufficient memory for all the arguments, an alternative approach is to re-map the RGBA values out of the top fourth arrays back onto the array, for a total score of 361 bytes:
f=s=>{d=document,i=new Image,i.src=s,i.onload=$=>{c=d.createElement`canvas`,x=c.getContext`2d`,c.width=w=i.width,c.height=h=i.height,x.drawImage(i,0,0),D=x.getImageData(0,0,w,h),t=D.data,t.set([...t].map((v,i,T)=>i%4?[,,,0]:T.slice(i,i+4)).sort((a,b)=>a.some((v,i)=>k=v-b[i])&&k).map((v,i,A)=>A[3*w*h+(i>>2)][i%4])),x.putImageData(D,0,0),d.body.appendChild(c)}}
We simply replace the [].concat(...{stuff}).slice(12*w*h) with {stuff}.map((v,i,A)=>A[3*w*h+(i>>2)][i%4]).)
2Related. – Martin Ender – 2015-11-02T23:26:12.893