YAML Environment Variable Interpolation in SnakeYAML scala

Question

Leveraging the best from SnakeYAML & Jackson in scala, I am using the following method to parse YAML files. This method supports the usage of anchors in YAML

import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import java.io.{File, FileInputStream}
import org.yaml.snakeyaml.{DumperOptions, LoaderOptions, Yaml}
/**
 * YAML Parser using SnakeYAML & Jackson Implementation
 *
 * @param yamlFilePath : Path to the YAML file that has to be parsed
 * @return: JsonNode of YAML file
 */
def parseYaml(yamlFilePath: String): JsonNode = {
  // Parsing the YAML file with SnakeYAML - since Jackson Parser does not have Anchors and reference support
  val ios = new FileInputStream(new File(yamlFilePath))
  val loaderOptions = new LoaderOptions
  loaderOptions.setAllowDuplicateKeys(false)
  val yaml = new Yaml(
    loaderOptions
  )

  val mapper = new ObjectMapper().registerModules(DefaultScalaModule)
  val yamlObj = yaml.loadAs(ios, classOf[Any])

  // Converting the YAML to Jackson YAML - since it has more flexibility for traversing through nodes
  val jsonString = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(yamlObj)
  val jsonObj = mapper.readTree(jsonString)
  println(jsonString)
  jsonObj
}

However, this currently does not support the interpolation of environment variables within the YAML file. Eg: If we get the following environment variables when we do

>>> println(System.getenv())
{PATH=/usr/bin:/bin:/usr/sbin:/sbin, XPC_FLAGS=0x0, SHELL=/bin/bash}

The question is how do we achieve environment variable interpolation in yaml file, lets say we have the following YAML file:

path_value: ${PATH}
xpc: ${XPC_FLAGS}
shell_path: ${SHELL}

Then after parsing the YAML should be:

{
   "path_value": "/usr/bin:/bin:/usr/sbin:/sbin",
   "xpc": "0x0",
   "shell_path": "/bin/bash"
}

Thanks for your time & efforts to answer in advance!

Answer 1

Thanks to the comments & guidance from the community. Here is my solution for the parser with custom constructors and represented:

import java.io.{File, FileInputStream}
import scala.util.matching.Regex
import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.yaml.snakeyaml.{DumperOptions, LoaderOptions, Yaml}
import org.yaml.snakeyaml.constructor.{AbstractConstruct, Constructor}
import org.yaml.snakeyaml.error.MissingEnvironmentVariableException
import org.yaml.snakeyaml.nodes.Node
import org.yaml.snakeyaml.nodes.ScalarNode
import org.yaml.snakeyaml.nodes.Tag
import org.yaml.snakeyaml.representer.Representer

/**
 * Copyright (c) 2008, http://www.snakeyaml.org
 * Class dedicated for SnakeYAML Support for Environment Variables
 */


/**
 * Construct scalar for format ${VARIABLE} replacing the template with the value from the environment.
 *
 * @see <a href="https://bitbucket.org/asomov/snakeyaml/wiki/Variable%20substitution">Variable substitution</a>
 * @see <a href="https://docs.docker.com/compose/compose-file/#variable-substitution">Variable substitution</a>
 */

class EnvScalarConstructor() extends Constructor {
  val ENV_TAG = new Tag("!ENV")
  this.yamlConstructors.put(ENV_TAG, new ConstructEnv)

  val ENV_regex: Regex = "\\$\\{\\s*((?<name>\\w+)((?<separator>:?(-|\\?))(?<value>\\w+)?)?)\\s*\\}".r

  private class ConstructEnv extends AbstractConstruct {
    override def construct(node: Node) = {
      val matchValue = constructScalar(node.asInstanceOf[ScalarNode])
      val patternMatch = ENV_regex.findAllIn(matchValue)
      val eval = patternMatch.toString()
      val name = patternMatch.group(1)
      val value = patternMatch.group(2)
      val separator = null
      apply(name, separator, if (value != null) value else "", ENV_regex.replaceAllIn(matchValue, getEnv(name)))

    }
  }

  /**
   * Implement the logic for missing and unset variables
   *
   * @param name        - variable name in the template
   * @param separator   - separator in the template, can be :-, -, :?, ?
   * @param value       - default value or the error in the template
   * @param environment - the value from the environment for the provided variable
   * @return the value to apply in the template
   */
  def apply(name: String, separator: String, value: String, environment: String): String = {
    if (environment != null && !environment.isEmpty) return environment
    // variable is either unset or empty
    if (separator != null) { //there is a default value or error
      if (separator == "?") if (environment == null) throw new MissingEnvironmentVariableException("Missing mandatory variable " + name + ": " + value)
      if (separator == ":?") {
        if (environment == null) throw new MissingEnvironmentVariableException("Missing mandatory variable " + name + ": " + value)
        if (environment.isEmpty) throw new MissingEnvironmentVariableException("Empty mandatory variable " + name + ": " + value)
      }
      if (separator.startsWith(":")) if (environment == null || environment.isEmpty) return value
      else if (environment == null) return value
    }
    ""
  }

  /**
   * Get the value of the environment variable
   *
   * @param key - the name of the variable
   * @return value or null if not set
   */
  def getEnv(key: String) = sys.env.getOrElse(key, System.getProperty(key, s"UNKNOWN_ENV_VAR-$key"))
}

The above constructor can be used in YAML Parser as follows:

/**
 * Function that will be used to load the YAML file
 * @param yamlFilePath - String with YAML path to read
 * @return - FasterXML JsonNode
 */
def parseYaml(yamlFilePath: String): JsonNode = {
  val ios = new FileInputStream(new File(yamlFilePath))
  // Parsing the YAML file with SnakeYAML - since Jackson Parser does not have Anchors and reference support
  val loaderOptions = new LoaderOptions
  loaderOptions.setAllowDuplicateKeys(false)
  val yaml = new Yaml(
    new EnvScalarConstructor,
    new Representer,
    new DumperOptions,
    loaderOptions
  )
  val mapper = new ObjectMapper().registerModules(DefaultScalaModule)
  val yamlObj = yaml.loadAs(ios, classOf[Any])

  // Converting the YAML to Jackson YAML - since it has more flexibility
  val jsonString = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(yamlObj)
  val jsonObj = mapper.readTree(jsonString)
  println(jsonString)

  jsonObj
}