andygrove / how-query-engines-work Goto Github PK

Contributors

Stargazers

Forkers

zznq jfz lvheyang mattyw harsha2010 codingcat fabianmurariu liliu88 mhatrep fedomn chenliu0831 rustin170506 xiaohan2013 abhishek-das-gupta waitingkuo wususu sel-fish fengys1996 chaojun-zhang at15 huaxiangsun adedayoominiyi yuan-yuan-jia duanmeng brucekellan huymq1710 risyomei yutiansut asad-awadia zen0fpy llama90 cyyeh tanvn jtt9340 lor-ela dborchard jalpan-randeri xuxiaotuan macohen zhangjiashen khalefa-ow manick02 ntileio4 rpmartz luyou-2023 dharanad

override fun toField(input: LogicalPlan): Field {
      return Field("mult", l.toField(input).dataType)
}

  val ctx = new ExecutionContext(Map.empty[String,String].asJava)
  val pqtSource = new ParquetDataSource("data/alltypes_plain.parquet")

  println(pqtSource.schema().toString)

  ctx.registerDataSource("pdata",pqtSource)
  val df2 = ctx.sql("select id,bool_col from pdata")
  val c2 =  ctx.execute(df2).iterator().asScala.toList.map(r => println(r))

Schema(fields=[Field(name=id, dataType=Int(32, true)), Field(name=bool_col, dataType=Bool), Field(name=tinyint_col, dataType=Int(32, true)), Field(name=smallint_col, dataType=Int(32, true)), Field(name=int_col, dataType=Int(32, true)), Field(name=bigint_col, dataType=Int(64, true)), Field(name=float_col, dataType=FloatingPoint(SINGLE)), Field(name=double_col, dataType=FloatingPoint(DOUBLE)), Field(name=date_string_col, dataType=Binary), Field(name=string_col, dataType=Binary), Field(name=timestamp_col, dataType=Binary)])

Reading 8 rows
null,null
null,null
null,null
null,null
null,null
null,null
null,null
null,null

data class Field(val name: String, val type: JDBCType)
data class Schema(val fields: List<Field>)

interface DataSource {
    fun schema(): Schema
    fun scan(projection: List<String>): Sequence<Map<String, Any?>>
}

// Physical plans return iterators over rows.
interface PhysicalPlan {
    fun schema(): Schema
    fun children(): List<PhysicalPlan>
    fun execute(): Sequence<Map<String, Any?>>
}

// Physical expression interface
interface Expression {
    fun evaluate(rows: Map<String, Any?>): Any?
}