How we crafted a domain-specific language for JSON transformation at RudderStack

JAVASCRIPT
Copy
/**
 * Scans the provided code characters for punctuator tokens, specifically focusing on variations of dots ('.', '..', '(...)').
 * 
 * @param {string[]} codeChars - An array of characters representing the code being scanned.
 * @param {number} idx - The current index within the code to start scanning from.
 * @returns {Token | undefined} - Returns a Token object representing the identified punctuator or undefined if no match is found.
 */
function scanPunctuatorForDots(codeChars: string[], idx: number): Token | undefined {
  const start = idx; // Store the starting index for the token

  // Extract the characters at specific positions relative to the current index
  const ch1 = codeChars[idx];
  const ch2 = codeChars[idx + 1];
  const ch3 = codeChars[idx + 2];

  // Check if the first character is a dot ('.')
  if (ch1 !== '.') {
    return undefined; // No match, return undefined
  }

  // Handle different punctuator variations involving dots:
  if (ch2 === '(' && ch3 === ')') {
    return {
      type: TokenType.PUNCT,
      value: '.()',
      range: [start, idx + 3], // Update range to include all three characters
    };
  }
  if (ch2 === '.' && ch3 === '.') {
    return {
      type: TokenType.PUNCT,
      value: '...',
      range: [start, idx + 3], // Update range to include all three characters
    };
  }
  if (ch2 === '.') {
    return {
      type: TokenType.PUNCT,
      value: '..',
      range: [start, idx + 2], // Update range to include both dots
    };
  }

  // Default case: single dot
  return {
    type: TokenType.PUNCT,
    value: '.',
    range: [start, idx + 1], // Update range to include single dot
  };
}

JAVASCRIPT
Copy
/**
 * Checks if the current token from the lexer is a dot (.) or double-dot (..) punctuator.
 * 
 * @returns {boolean} True if the current token is a dot or double-dot punctuator, false otherwise.
 */
function matchPathPartSelector(): boolean {
  const token = this.lexer.lookahead(); // Peek at the next token without consuming it
  if (token.type === TokenType.PUNCT) {
    return token.value === '.' || token.value === '..'; // Check if the value is '.' or '..'
  }
  return false;
}

/**
 * Parses a selector expression, which can be a simple identifier, wildcard (*), or a string literal.
 * 
 * @returns {SelectorExpression | IndexFilterExpression | Expression} The parsed selector expression object.
 */
function parseSelector(): SelectorExpression | IndexFilterExpression | Expression {
  const selector = this.lexer.value(); // Consume the current token as the selector
  // Other code

  let prop: Token | undefined;
  if (this.lexer.match('*') || this.lexer.matchID() || this.lexer.matchTokenType(TokenType.STR)) { // Check for specific token types
    prop = this.lexer.lex(); // Consume the next token as the property
  }
  return {
    type: SyntaxType.SELECTOR,
    selector,
    prop,
  };
}

/**
 * Parses a path part, which can be an expression, an array of expressions, or a selector expression based on the current token.
 * 
 * @returns {Expression | Expression[] | undefined} The parsed path part or undefined if not applicable.
 */
function parsePathPart(): Expression | Expression[] | undefined {
  // Other code
  } else if (matchPathPartSelector()) { // Check if the current token is a dot or double-dot
    return parseSelector(); // Parse a selector if it is
  } 
  // Other code
}

JAVASCRIPT
Copy
{
  "type": "statements_expr",
  "statements": [
    {
      "type": "path",
      "parts": [
        {
          "type": "selector",
          "selector": ".",
          "prop": {
            "type": "id",
            "value": "employees",
            "range": [
              1,
              10
            ]
          }
        },
        {
          "type": "selector",
          "selector": "..",
          "prop": {
            "type": "id",
            "value": "name",
            "range": [
              12,
              16
            ]
          }
        }
      ],
      "pathType": "rich"
    }
  ]
}

JAVASCRIPT
Copy
/**
 * Translates a selector expression with descendant operator (..) into executable code.
 * 
 * @param {SelectorExpression} expr - The selector expression containing the descendant operator.
 * @param {string} dest - The variable name to store the final result.
 * @param {string} baseCtx - The starting context for traversing descendant properties.
 * @returns {string} The generated JavaScript code representing the translation.
 */
function translateDescendantSelector(
  expr: SelectorExpression,
  dest: string,
  baseCtx: string,
): string {
  const code: string[] = []; // Array to store generated code lines

  // Acquire temporary variables for the translation process
  const ctxs = this.acquireVar();
  const currCtx = this.acquireVar();
  const result = this.acquireVar();

  // Initialize the result variable to an empty array
  code.push(JsonTemplateTranslator.generateAssignmentCode(result, '[]')); // Call a helper function to generate assignment code

  // Extract the property from the selector expression (if any)
  const { prop } = expr;
  const propStr = CommonUtils.escapeStr(prop?.value); // Escape the property value for safe string inclusion

  // Push initial code to set up the context list
  code.push(`${ctxs}=[${baseCtx}];`); // Assign the base context to the contexts list

  // Loop through contexts while there are more to process
  code.push(`while(${ctxs}.length > 0) {`);
  // Shift the current context from the list
  code.push(`${currCtx} = ${ctxs}.shift();`);

  // Handle empty contexts (skip if empty)
  code.push(`if(${JsonTemplateTranslator.returnIsEmpty(currCtx)}){continue;}`); // Call a helper function to check for emptiness

  // Handle context being an array (recursively process elements)
  code.push(`if(Array.isArray(${currCtx})){`);
  code.push(`${ctxs} = ${ctxs}.concat(${currCtx});`); // Concatenate the array elements to the contexts list
  code.push('continue;'); // Skip to the next iteration
  code.push('}');

  // Handle context being an object (process its properties)
  code.push(`if(typeof ${currCtx} === "object") {`);
  const valuesCode = JsonTemplateTranslator.returnObjectValues(currCtx); // Call a helper function to get object values
  code.push(`${ctxs} = ${ctxs}.concat(${valuesCode});`); // Concatenate object values to the contexts list
  if (prop) { // If there's a property in the selector
    if (prop?.value === '*') { // If the property is a wildcard (*)
      code.push(`${result} = ${result}.concat(${valuesCode});`); // Concatenate all object values to the result
    } else { // If the property is a specific key
      code.push(`if(Object.prototype.hasOwnProperty.call(${currCtx}, ${propStr})){`); // Check if the property exists on the object
      code.push(`${result} = ${result}.concat(${currCtx}[${propStr}]);`); // Append the property value to the result
      code.push('}');
    }
  }
  code.push('}');

  // If no property was specified, add the entire current context to the result
  if (!prop) {
    code.push(`${result}.push(${currCtx});`);
  }

  // Close the loop
  code.push('}');

  // Flatten the final result array (remove nested arrays)
  code.push(`${dest} = ${result}.flat();`);

  // Join all code lines and return the generated code
  return code.join('');
}

JAVASCRIPT
Copy
// This function takes an input object and processes its 'employees' property to extract names into an array.
function extractEmployeeNames(inputObject) {
  let result; // Initialize variable to store final result
  let currentObject; // Temporary variable for iterating over input object
  let employeesArray; // Temporary variable for storing 'employees' array
  let i; // Counter variable for looping over input object
  let j; // Counter variable for looping over 'employees' array
  let currentEmployee; // Temporary variable for each 'employee' object
  let extractedNames; // Temporary variable for storing names extracted from 'employee' object
  let collectedNames; // Array to collect extracted names
  let queue; // Queue for BFS traversal of objects
  let currentQueueItem; // Temporary variable for BFS traversal
  let tempNames; // Temporary array to collect names during traversal

  // Initialize result with the input object
  result = inputObject;

  // Initialize collectedNames as an empty array to collect extracted names
  collectedNames = [];

  // Assign currentObject to input object
  currentObject = result;

  // Check if currentObject is not null or undefined
  if (currentObject !== null && currentObject !== undefined) {
    // Convert currentObject to an array if it's not already one
    currentObject = Array.isArray(currentObject) ? currentObject : [currentObject];
  }

  // Loop through each item in currentObject
  for (i = 0; i < currentObject.length; i++) {
    // Assign employeesArray to 'employees' property of current item
    employeesArray = currentObject[i]?.employees;

    // Continue if employeesArray is null or undefined
    if (employeesArray === null || employeesArray === undefined) {
      continue;
    }

    // Loop through each item in employeesArray
    for (j = 0; j < employeesArray.length; j++) {
      // Assign currentEmployee to current item in employeesArray
      currentEmployee = employeesArray[j];

      // Initialize tempNames as an empty array to collect names
      tempNames = [];

      // Initialize queue with currentEmployee in an array
      queue = [currentEmployee];

      // Perform BFS traversal on queue until it's empty
      while (queue.length > 0) {
        // Pop the first item from queue
        currentQueueItem = queue.shift();

        // Continue if currentQueueItem is null or undefined
        if (currentQueueItem === null || currentQueueItem === undefined) {
          continue;
        }

        // If currentQueueItem is an array, concatenate it with queue
        if (Array.isArray(currentQueueItem)) {
          queue = queue.concat(currentQueueItem);
          continue;
        }

        // If currentQueueItem is an object, extract values and filter out null or undefined ones
        if (typeof currentQueueItem === "object") {
          queue = queue.concat(Object.values(currentQueueItem).filter(v => v !== null && v !== undefined));
          // If 'name' property exists in currentQueueItem, add it to tempNames
          if (currentQueueItem.hasOwnProperty('name')) {
            tempNames = tempNames.concat(currentQueueItem.name);
          }
        }
      }

      // Flatten tempNames and assign it to extractedNames
      extractedNames = tempNames.flat();

      // Continue if extractedNames is null or undefined
      if (extractedNames === null || extractedNames === undefined) {
        continue;
      }

      // Push extractedNames into collectedNames
      collectedNames.push(extractedNames);
    }
  }

  // If collectedNames has only one element, assign that element to collectedNames
  collectedNames = collectedNames.length < 2 ? collectedNames[0] : collectedNames;

  // Assign result to collectedNames
  result = collectedNames;

  // Return the final result
  return result;
}