| Basic Operations |
Variable Declaration |
val x: Int = 10 // Immutable
var y: Int = 20 // Mutable |
|
Collections |
val list = List(1, 2, 3, 4, 5)
val array = Array(1, 2, 3, 4, 5)
val map = Map("a" -> 1, "b" -> 2, "c" -> 3) |
| Spark Setup |
Initialize Spark Session |
import org.apache.spark.sql.SparkSession
val spark = SparkSession.builder.appName("Spark App").config("spark.master", "local").getOrCreate() |
| RDD Operations |
Create RDD |
val rdd = spark.sparkContext.parallelize(Seq(1, 2, 3, 4, 5)) |
|
Transformations |
val mappedRDD = rdd.map(_ * 2)
val filteredRDD = rdd.filter(_ > 2) |
|
Actions |
val collected = rdd.collect()
val count = rdd.count()
val firstElement = rdd.first() |
| DataFrame Operations |
Create DataFrame |
import spark.implicits._
val df = Seq((1, "a"), (2, "b"), (3, "c")).toDF("id", "value") |
|
Show DataFrame |
df.show() |
|
DataFrame Transformations |
val filteredDF = df.filter($"id" > 1)
val selectedDF = df.select("value")
val withColumnDF = df.withColumn("new_column", $"id" * 2) |
|
SQL Queries |
df.createOrReplaceTempView("table")
val sqlDF = spark.sql("SELECT * FROM table WHERE id > 1") |
| Dataset Operations |
Create Dataset |
case class Record(id: Int, value: String)
val ds = Seq(Record(1, "a"), Record(2, "b"), Record(3, "c")).toDS() |
|
Dataset Transformations |
val filteredDS = ds.filter(_.id > 1)
val mappedDS = ds.map(record => record.copy(value = record.value.toUpperCase)) |
| Conversions |
RDD to DataFrame |
val rddToDF = rdd.toDF("numbers") |
|
DataFrame to RDD |
val dfToRDD = df.rdd |
|
DataFrame to Dataset |
val dfToDS = df.as[Record] |
|
Dataset to DataFrame |
val dsToDF = ds.toDF() |
| Reading and Writing Data |
Read CSV |
val csvDF = spark.read.option("header", "true").csv("path/to/file.csv") |
|
Write CSV |
df.write.option("header", "true").csv("path/to/save") |
|
Read Parquet |
val parquetDF = spark.read.parquet("path/to/file.parquet") |
|
Write Parquet |
df.write.parquet("path/to/save") |
| Common Data Engineering Functions |
GroupBy and Aggregations |
val groupedDF = df.groupBy("id").count()
val aggregatedDF = df.groupBy("id").agg(sum("value")) |
|
Join Operations |
val df1 = Seq((1, "a"), (2, "b")).toDF("id", "value1")
val df2 = Seq((1, "x"), (2, "y")).toDF("id", "value2")
val joinedDF = df1.join(df2, "id") |
|
Window Functions |
import org.apache.spark.sql.expressions.Window
import org.apache.spark.sql.functions._
val windowSpec = Window.partitionBy("id").orderBy("value")
val windowedDF = df.withColumn("rank", rank().over(windowSpec)) |
|
UDFs (User-Defined Functions) |
import org.apache.spark.sql.functions.udf
val addOne = udf((x: Int) => x + 1)
val dfWithUDF = df.withColumn("new_value", addOne($"id")) |