KafkaInput.scala

package com.powerpanel.kso;

import kafka.serializer.StringDecoder
import kafka.serializer.DefaultDecoder
import com.powerpanel.kso.model._
import org.apache.spark.streaming.kafka010.KafkaUtils
import org.apache.spark.streaming.kafka010.ConsumerStrategies
import org.apache.spark.streaming.kafka010.LocationStrategies
import org.apache.spark.streaming.StreamingContext

class KafkaInput extends Serializable {
    def readFromKafka (ssc: StreamingContext) = {
        val props = AppConfig.loadProperties();
        val topicsSet = props.getProperty("kafka.topic").split(",").toSet;
        val kafkaParams = collection.mutable.Map[String, String]("metadata.broker.list" -> props.getProperty("kafka.brokers"))
        if (props.getProperty("kafka.consume_from_beginning").toBoolean)
        {
            kafkaParams.put("auto.offset.reset", "smallest");
        }
        val messages = KafkaUtils.createDirectStream[String, Array[Byte]](
          ssc,
          LocationStrategies.PreferConsistent,
          ConsumerStrategies.Subscribe[String, Array[Byte]](topicsSet, kafkaParams)
        );
        //.repartition(Integer.parseInt(props.getProperty("app.processing_parallelism")));

        // Decode avro container format
        // val avroSchemaString = StaticHelpers.loadResourceFile("dataplatform-raw.avsc");
        val rawMessages = messages.map(x => {
            // val eventDecoder = new DataPlatformEventDecoder(avroSchemaString);
            val dataPackageDecoder = DataPackage.getDecoder();
            val payload = x.value();
            val dataPackage = dataPackageDecoder.decode(payload);
            dataPackage;
        });
        rawMessages;
    };
}