WF2-1 Bulk get Person cases

This job was provided by an OpenFn.org user via the job library API.

Metadata

• Name: WF2-1 Bulk get Person cases
• Adaptor: @openfn/language-http
• Adaptor Version: v4.2.8
• Created over 1 year ago
• Updated 7 months ago
• Score: 92 (an indicator of how useful this job may be)

Key Functions

each, get, map, JSON

Expression

fn(state => {
  const { baseUrl } = state.configuration;

  const caseTypes = ['Person'];

  //NOTE: You can use 'limit' to modify this batch size as desired
  const limit = 1000; //E.g., change to 10 to process records in smaller batch sizes

  //NOTE: indexedOnStart is the default sync start data the FIRST time the job runs
  const indexedOnStart = '2024-05-30T03:00:00';
  

  //NOTE: After first job run, OpenFn will check the job sync data ("lastRunAt") to set as the indexedOnStart
  const lastRunAt =
    typeof state.lastRunAt !== 'undefined' ? state.lastRunAt : indexedOnStart;
  console.log('Filtering cases with indexed_on_start > than ::', lastRunAt);

  const queries = caseTypes.map(
    t => `?type=${t}&indexed_on_start=${lastRunAt}&limit=${limit}`
    //NOTE: If for testing, you want to fetch data for a specific historical range (e.g., between April 23 and 24)...
    //...then use the query string below instead of the one above on L16, and custom adjust the index_on start/end dates
    // t => `?type=${t}&indexed_on_start=2024-01-16T00:00:00&limit=${limit}&indexed_on_end=2024-01-16T23:59:59`
  );

  return { ...state, queries, baseUrl, payloads: [] };
});

// create a "recursiveGet" which will call itself if CommCare tells us there's
// more data to fetch for the same form
fn(state => {
  const recursiveGet = url =>
    get(
      url,
      {
        headers: { 'content-type': 'application/json' },
      },
      nextState => {
        const now = new Date();
        const { baseUrl, data, payloads } = nextState;

        const { meta, objects } = data;
        console.log('Metadata in CommCare response:', meta);

        const finalState = {
          ...nextState,
          payloads: [...payloads, ...objects],
        };

        if (meta.next) {
          console.log('Next query detected, recursing...');
          return recursiveGet(`${baseUrl}${meta.next}`)(finalState);
        }
        finalState.lastRunAt = now.toISOString().slice(0, 19);
        return finalState;
      }
    );

  return { ...state, recursiveGet };
});

// for each initial query, fetch data recursively
each(
  '$.queries[*]',
  fn(state => {
    return state.recursiveGet(`${state.baseUrl}${state.data}`)(state);
  })
);
// log the total number of payloads returned
fn(state => {
  console.log('Count of payloads', state.payloads.length);
  //HMN 11072023
 // console.log(JSON.stringify(state.payloads, null, 2));

  return { ...state, references: [], data: {} };
});