Map/ Reduce Structure for high data

For example, if there are 5 files and each file has over 50000 lines. We need to process each line from the files for all 5 files.

We need to create a script parameter for the map-reduce script with no default value or set the default value as the first file referring key value.

In the script get the input entry point and check the parameter value empty or not.

This logic can be implemented for large data execution in the map/ reduce script.

Referring to global variable for fetching related file folder internal ID.

const VENDOR_FOLDER_IDS = {
‘morrisFolder’: { ‘main’: 5174924, ‘backup’: 5174914 },
‘gedoreToolsFolder’: { ‘main’: 5174926, ‘backup’: 5174920 },
‘dasProductsFolder’: { ‘main’: 5174929, ‘backup’: 5174923 },
‘vestilFolder’: { ‘main’: 5174925, ‘backup’: 5174921 },
‘eagleToolsFolder’: { ‘main’: 5174928, ‘backup’: 5174913 },
‘ORSFolder’: { ‘main’: 5174927, ‘backup’: 5174922 },
‘ORSFolder2’: { ‘main’: 5335404, ‘backup’: 5335405 }
}

const getInputData = (inputContext) => {
         var scriptParamValues = runtime.getCurrentScript().getParameter({ name:               'custscript_jj_multiple_file_execution' });
         let flatArrayOutput;
         if (scriptParamValues=="ORSFolder") {
              // perform the functionality for file ORS
             flatArrayOutput="";
         }
         if (scriptParamValues=="vestilFolder") {
              // perform the functionality for file vestil
             flatArrayOutput="";
         } 

         return flatArrayOutput; // return the result

    }

    const map = (mapContext) => {
        try {
            var searchResultFromGetInput = JSON.parse(mapContext.value);
            var searchResult = searchResultFromGetInput.value
            mapContext.write({
                key: itemKey + timeStamp,
                value: finalDataObj1
            })
        } catch (er) {
            log.debug('error @ map function', er)
        }
    }

    const reduce = (reduceContext) => {
        try {
            reduceContext.write({
                key: reduceContext.key,
                value: reduceContext.values.length
            });
        } catch (errors) {
            log.debug('error @ reduce', errors)
        }
    }

    const summarize = (summaryContext) => {
        try {
            var scriptParamValues = runtime.getCurrentScript().getParameter({ name: 'script parameter id' });
            if (!scriptParamValues=="end") { // pass end value if all the files are executed. That will stop the task submission.
                var mapReduceTask = task.create({
                    taskType: task.TaskType.MAP_REDUCE,
                    scriptId: "script id",
                    deploymentId: 'deployment id'
                });
                mapReduceTask.params = {
                    'script parameter internal ID': 'vestilFolder' // pass the next file name
                };
                mapReduceTask.submit();
            }
        } catch (err) {
            log.error('err @ summarize function', err)
        }
    }

    return { getInputData, map, reduce, summarize }