Save model fails APEX CPU time limit error / saving too many rows at once (javascript)

  • 1
  • Idea
  • Updated 3 days ago
  • (Edited)
When saving a model, the model intelligently knows which rows need inserts/updates and saves only those rows that need inserts/updates.

But there still exists the possibility that if you have so many rows saving that it would cause an error (generally CPU time limit error) because too many rows are saving at once.

We can get around this error by saving the items in the model incrementally in smaller batches.

To do this I made a custom function "modelSaver"

Example usage:

$.blockUI();
let pc = function (fparams){
console.log(`Saving Rows: ${fparams.nextStart}-${fparams.nextEnd}`);
}
$.when(skuid.custom.modelSaver(modelToSave,{limit:50,progressCallback: pc})).done( f => {
$.unblockUI();
});

Function definition:

// skuid.custom.modelSaver(model,fparams)
// Saves a model incrementally as to not overload the save process with too many saves at once
//  fparams: object
//  {
//      limit: Number of rows to limit by. If unspecified will choose
//          the model's recordsLimit property or if that is unspecified defaults to 100
//      progressCallback: function to call before running each individual query
//          in the format progressCallback(fparams), fparams is an object
//          progressCallback fparams = {
//              count: count of rows saved so far
//              limit: our limit for how many rows to save per run
//              nextStart: the row # of the next row to be saved,
//              nextEnd: the last row # to be saved (based on limit)
//          }
//  }
skuid.custom.modelSaver = function (model, fparams) {
    const deferred = $.Deferred();
    if (model === undefined) {
        deferred.reject('Model undefined');
        return deferred.promise();
    }
    fparams = fparams || {};
    const limit = fparams.limit || model.recordsLimit || 100;
    const progressCallback = fparams.progressCallback || undefined;
    // Set Initial Progress
    if (progressCallback !== undefined) {
        progressCallback.call(this, {
            count: 0,
            limit,
            nextStart: 1,
            nextEnd: limit
        });
    }
    // Back up the changes object from the model and clear it
    // We will only save the changes we want to save
    model.changesTemp = model.changes;
    model.changes = {};
    // Run through our recursive function to save all rows
    modelSave(model, {
        limit: limit,
        progressCallback: progressCallback,
        promiseResolve: deferred.resolve,
        promiseReject: deferred.reject
    });
    function modelSave (model, fparams) {
        fparams = fparams || {};
        const limit = fparams.limit || model.recordsLimit || 100;
        const count = fparams.count || 0;
        const progressCallback = fparams.progressCallback || undefined;
        const promiseResolve = fparams.promiseResolve || undefined;
        const promiseReject = fparams.promiseReject || undefined;
        
        let localCount = 0;
        for (const [key, value] of Object.entries(model.changesTemp)) {
            // Only process up to limit
            if (localCount >= limit) {
                break;
            }
            // Move from our temp changes back to changes
            model.changes[key] = value;
            // Make sure the model is set to hasChanged
            model.hasChanged = true;
            // Delete our temp key
            delete model.changesTemp[key];
            localCount++;
        }
        // Save the model
        if (Object.keys(model.changes).length > 0) {
            $.when(model.save()).done(f => {
                if (Object.keys(model.changesTemp).length > 0) {
                    // If we still have changesTemp
                    if (progressCallback !== undefined) {
                        // Call our progressCallback if defined
                        let localEnd = limit;
                        if (Object.keys(model.changesTemp).length < localEnd) {
                            localEnd = Object.keys(model.changesTemp).length;
                        }
                        progressCallback.call(this, {
                            count: count + localCount,
                            limit: limit,
                            nextStart: count + localCount + 1,
                            nextEnd: count + localCount + localEnd
                        });
                    }
                    // Recurse our modelSave
                    modelSave(model, {
                        limit: limit,
                        count: (count + localCount),
                        progressCallback: progressCallback,
                        promiseResolve: deferred.resolve,
                        promiseReject: deferred.reject
                    });
                } else {
                    // We have no remaining changesTemp, resolve
                    delete model.changesTemp;
                    model.hasChanged = false;
                    promiseResolve(f);
                }
            }).fail(f => {
                if (Object.keys(model.changesTemp).length === 0) {
                    model.hasChanged = false;
                }
                // Save failed, reject
                for (const [key, value] of Object.entries(model.changesTemp)) {
                    // Move from our temp changes back to changes
                    model.changes[key] = value;
                    // Make sure the model is set to hasChanged
                    model.hasChanged = true;
                    // Delete our temp key
                    delete model.changesTemp[key];
                }
                delete model.changesTemp;
                
                promiseReject(f);
            });
        } else {
            // No changes were needed, resolve
            delete model.changesTemp;
            model.hasChanged = false;
            promiseResolve(f);
        }
    }
    return deferred.promise();
};


Photo of Mark L

Mark L

  • 2,616 Points 2k badge 2x thumb

Posted 1 week ago

  • 1
Photo of Zach McElrath

Zach McElrath, Employee

  • 56,134 Points 50k badge 2x thumb
Mark, have you actually encountered an issue saving this many rows at once? In all of my time working at Skuid I have never encountered a customer who has saved so many rows at one time that Skuid's Model saving code caused the Apex CPU Time limit to be exceeded, or even the DML row limits to be exceeded --- unless the customer was trying to use Skuid to upload something like 1000's of new rows all at once, in which case I would really be curious as to what the customer is trying to do with that page, I don't think this snippet is realistically ever necessary.
Photo of Mark L

Mark L

  • 2,616 Points 2k badge 2x thumb
Hi Zach,

In our particular use case that was using too much APEX CPU time for the save, we have a workflow rule triggerd on a field update that sends an email alert. I believe it's this email alert sending that slows down the transaction, allowing us to only save something like around < 200 at a time.
Photo of Zach McElrath

Zach McElrath, Employee

  • 56,134 Points 50k badge 2x thumb
Okay, thanks Mark, good to know.