conda create -y -n yz_spark_kafka_demo python=3.6
source activate spark_kafka_demo
pip install kafka-python
pip install pyspark
pip install pandas

docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=localhost --env ADVERTISED_PORT=9092 spotify/kafka

source activate yz_spark_kafka_demo
gunzip apache-access-log.txt.gz
python step1-kafka-producer.py -i ./apache-access-log.txt --host 127.0.0.1:9092 --topic logs

source activate yz_spark_kafka_demo
python step2-streaming-processor.py --host 127.0.0.1:9092 --topic logs

source activate yz_spark_kafka_demo
python step3-QA.py --path ./data