Very slow imagemagick convert call when using AWS Lambda

Question

I've got a relatively simple composite action that I'm carrying out with imagemagick on AWS Lambda. Why does it often take more than a minute to complete?

This composites four smaller images "behind" a larger background that has transparent holes:

convert -size 1280x720 'xc:#747784'  \
        \( /tmp/$uuid/face-1.png -background none -distort SRT "0.132 16.094" \) -geometry +324.929+110.781 -composite \
        \( /tmp/$uuid/face-2.png -background none -distort SRT "0.132 24.486" \) -geometry +401.5+106.375 -composite 
        \( /tmp/$uuid/face-3.png -background none -distort SRT "1 0" \) -geometry +-166+-250  -composite 
        \( /tmp/$uuid/face-4.png -background none -distort SRT "1 0" \) -geometry +-166+-250  -composite 
        /tmp/$uuid/____MASTER_720P_00230.png -composite \
        -depth 8 png:/tmp/$uuid/230.png

On my (6 year old) laptop - these operations typically take between 2 & 3 seconds.

On Lambda (even allowing for the overhead of copying the five source files from s3 to the /tmp filesystem) this was taking in the region of 30 seconds, and often hitting the max execution time that I'd set (1 minute). I upped the cpu and timeout to 2 minutes 30 seconds, but I was still seeing timeouts.

The Lambda built-in ImageMagick (v6) allegedly has a bug with OpenMP support, so I tried to mitigate by using:

OMP_NUM_THREADS=1 \
MAGICK_THREAD_LIMIT=1 \
convert -size 1280x720 'xc:#747784'  \
        \( /tmp/$uuid/face-1.png -background none -distort SRT "0.132 16.094" \) -geometry +324.929+110.781 -composite \
        \( /tmp/$uuid/face-2.png -background none -distort SRT "0.132 24.486" \) -geometry +401.5+106.375 -composite 
        \( /tmp/$uuid/face-3.png -background none -distort SRT "1 0" \) -geometry +-166+-250  -composite 
        \( /tmp/$uuid/face-4.png -background none -distort SRT "1 0" \) -geometry +-166+-250  -composite 
        /tmp/$uuid/____MASTER_720P_00230.png -composite \
        -depth 8 png:/tmp/$uuid/230.png

But this still was giving me run-times of two-minutes and above (often hitting timeout).

I then compiled a fully-static ImageMagick v7 (7.0.7-13) (adding --with-quantum-depth=8 to try and reduce the processing overhead) and added this to my Lambda package, and changed the convert call to this:

/var/task/bin/magick -size 1280x720 'xc:#747784'  
                     \( /tmp/$uuid/face-1.png -background none -distort SRT "0.132 16.094" \) -geometry +324.929+110.781  -composite \
                     \( /tmp/$uuid/face-2.png -background none -distort SRT "0.132 24.486" \) -geometry +401.5+106.375  -composite \
                     \( /tmp/$uuid/face-3.png -background none -distort SRT "1 0" \) -geometry +-166+-250  -composite \
                     \( /tmp/$uuid/face-4.png -background none -distort SRT "1 0" \) -geometry +-166+-250  -composite \
                     /tmp/$uuid/____MASTER_720P_00230.png -composite \
                     -depth 8 png:/tmp/$uuid/230.png

But there's still no real noticeable improvement in the run time.

Here's the code for the entire Lambda call (this is slightly hand-edited for posting here, so any mistakes are purely in this snippet)

var Q = require('q');
var path = require('path');

// A lovely set of composable modules
// https://github.com/lambduh/lambduh
var execute = require('lambduh-execute');
var s3Download = require('lambduh-get-s3-object');
var upload = require('lambduh-put-s3-object');
var s3Upload = require('lambduh-put-s3-object');

// This contains the relevant imagemagick convert call for each frame, indexed
// by frame
var animationFrames = require('./composite.json');

var bucket = "super.secret.bucket";

var frame, paddedFrame, uuid;

process.env['PATH'] = process.env['PATH'] + ':/tmp/:' + process.env['LAMBDA_TASK_ROOT']

exports.handler = function(event, context) {

    frame = event.frame;
    paddedFrame = ("00000" + frame).slice(-5);
    uuid = event.uuid;

        console.log("bucket: ", bucket);
        console.log("frame: ", frame);
        console.log("folder: ", uuid);

      // make the /tmp/uuid directory where we'll download the face pngs and
      // the relevant frame png to. Tidy up from old runs first.
      execute(event, {
          shell: "rm -rf /tmp/*; mkdir -p /tmp/" + uuid,
          logOutput: true
      })

      //now get the face pngs and frame png and put them there
      .then(function(event){

        var def = Q.defer();
        var s3Files = []
        var message = {};

        s3Files.push("backgrounds/____MASTER_720P_" + paddedFrame + ".png");
        s3Files.push("videos/" + uuid + "/face-1.png")
        s3Files.push("videos/" + uuid + "/face-2.png")
        s3Files.push("videos/" + uuid + "/face-3.png")
        s3Files.push("videos/" + uuid + "/face-4.png")


        var promises = [];

        s3Files.forEach(function(s3File) {
          console.log("Going to download %s/%s to /tmp/%s/%s", bucket, s3File, uuid, path.basename(s3File));
          promises.push(s3Download(event, {
            srcBucket: bucket,
            srcKey: s3File,
            downloadFilepath: "/tmp/" + uuid + "/" + path.basename(s3File)
          })
          .fail(function(){
            console.log("Couldn't download ", s3File);
          })
        )});

        Q.all(promises)
          .then(function(event) {
            def.resolve(event[0]);
          })
          .fail(function(err) {
            def.reject(err);
          });
        return def.promise;
      })

      .then(function(event){
        console.log("Compositing with imagemagick");
        return execute(event, {
          shell: "export uuid=" + uuid + " && " + animationFrames[frame],
          logOutput: true
        })
      })

      .then(function(event){
        console.log("Going to upload /tmp/%s/%s.png to %s/videos/%s/%s.png", uuid, frame, bucket, uuid, frame);
        return s3Upload(null, {
          dstBucket: bucket,
          dstKey: "videos/" + uuid + "/" + frame + ".png",
          uploadFilepath: "/tmp/" + uuid + "/" + frame + ".png"
        })
      })

      .then(function(event){
        console.log("finished");
        console.log(event);
        context.succeed(frame)
      })

      .fail(function(err) {
        console.log(err);
        //fail soft so lambda doesn't try to run this function again
        context.done(null, err);
      });
}