Use Apache Kafka and Azure Databricks for streaming
Cunwei zhao
0
Reputation points
Use Apache Kafka and Azure Databricks for streaming,The code is as follows
# Read data from Apache Kafka
from pyspark.sql.functions import *
from pyspark.sql.types import StructType, StructField
from pyspark.sql.types import *
#import pdb; pdb.set_trace()
kafka_df_zcw = spark.readStream.format("kafka").option("kafka.bootstrap.servers", "10.10.173.17:9092").option("kafka.security.protocol", "PLAINTEXT").option("subscribe", "zcw").load()
schema = StructType([
StructField("speed", IntegerType()),
StructField("volkswagen", StringType()),
StructField("version", StringType()),
StructField("ts", LongType())
])
kafka_df_zcw_temp = kafka_df_zcw.selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)")
.outputMode("append")\
.format("console")\
.start() query_zcw.awaitTermination()
Always prompt as follows:
Sign in to answer