'Java.io.FileNotFoundException : YAML file does not exists
When I am submitting the spark job from terminal I am getting below error that file does not exists. Although I have already placed config file to my local.
spark-submit command:
spark-submit --class com.yotpo.metorikku.Metorikku /home/hadoopuser/IdeaProjects/metorikku/target/scala-2.11/metorikku.jar -c /home/hadoopuser/IdeaProjects/metorikku/examples/movies.yaml
code
import com.yotpo.metorikku.configuration.job.{ConfigurationParser, Periodic}
import com.yotpo.metorikku.metric.MetricSet
import org.apache.log4j.LogManager
object Metorikku extends App {
val log = LogManager.getLogger(this.getClass)
log.info("Starting Metorikku - Parsing configuration")
val session = Job(ConfigurationParser.parse(args))
session.config.periodic match {
case Some(periodic) => {
executePeriodicTask(periodic)
}
case _ => runMetrics(session)
}
private def executePeriodicTask(periodic: Periodic) = {
val task = new Runnable {
def run() = {
session.sparkSession.catalog.clearCache()
runMetrics(session)
}
}
val ex = new ScheduledThreadPoolExecutor(1)
val initialDelay = 0
ex.scheduleAtFixedRate(task, initialDelay, periodic.getTriggerDurationInSeconds(), TimeUnit.SECONDS)
}
def runMetrics(job: Job): Unit = {
job.config.metrics match {
case Some(metrics) => metrics.foreach(metricSetPath => {
val metricSet = new MetricSet(metricSetPath)
metricSet.run(job)
})
case None => log.warn("No mertics were defined, exiting")
}
}
}
error
org.apache.hadoop.ipc.RemoteException(java.io.FileNotFoundException): File does not exist: /home/hadoopuser/IdeaProjects/metorikku/examples/movies.yaml
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
Solution | Source |
---|