DType Format

FlatBuffer Definition

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

enum PType: uint8 {
    U8,
    U16,
    U32,
    U64,
    I8,
    I16,
    I32,
    I64,
    F16,
    F32,
    F64,
}

table Null {}

table Bool {
    nullable: bool;
}

table Primitive {
    ptype: PType;
    nullable: bool;
}

table Decimal {
    precision: uint8;
    scale: int8;
    nullable: bool;
}

table Utf8 {
    nullable: bool;
}

table Binary {
    nullable: bool;
}

table Struct_ {
    names: [string];
    dtypes: [DType];
    nullable: bool;
}

table List {
    element_type: DType;
    nullable: bool;
}

table FixedSizeList {
    element_type: DType;
    size: uint32;
    nullable: bool;
}

table Extension {
    id: string;
    storage_dtype: DType;
    metadata: [ubyte];
}

table Variant {
    nullable: bool;
}

table Union {
    names: [string];
    dtypes: [DType];
    type_ids: [byte]; // length must equal dtypes.len(); interpreted as unsigned
    nullable: bool;
}

table Map {
    key_type: DType;
    value_type: DType;
    keys_sorted: bool;
    nullable: bool;
}

union Type {
    Null = 1,
    Bool = 2,
    Primitive = 3,
    Decimal = 4,
    Utf8 = 5,
    Binary = 6,
    Struct_ = 7,
    List = 8,
    Extension = 9,
    FixedSizeList = 10, // This is after `Extension` for backwards compatibility.
    Variant = 11,
    Union = 12,
    Map = 13,
}

table DType {
  type: Type;
}

root_type DType;

Protobuf Definition

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

syntax = "proto3";

package vortex.dtype;

option java_package = "dev.vortex.proto";
option java_outer_classname = "DTypeProtos";

enum PType {
  U8 = 0;
  U16 = 1;
  U32 = 2;
  U64 = 3;
  I8 = 4;
  I16 = 5;
  I32 = 6;
  I64 = 7;
  F16 = 8;
  F32 = 9;
  F64 = 10;
}

message Null {}

message Bool {
  bool nullable = 1;
}

message Primitive {
  PType type = 1;
  bool nullable = 2;
}

message Decimal {
  uint32 precision = 1;
  int32 scale = 2;
  bool nullable = 3;
}

message Utf8 {
  bool nullable = 1;
}

message Binary {
  bool nullable = 1;
}

message Struct {
  repeated string names = 1;
  repeated DType dtypes = 2;
  bool nullable = 3;
}

message List {
  DType element_type = 1;
  bool nullable = 2;
}

message FixedSizeList {
  DType element_type = 1;
  uint32 size = 2;
  bool nullable = 3;
}

message Extension {
  string id = 1;
  DType storage_dtype = 2;
  optional bytes metadata = 3;
}

message Variant {
  bool nullable = 1;
}

message Union {
  repeated string names = 1;
  repeated DType dtypes = 2;
  repeated int32 type_ids = 3; // length must equal dtypes.len(); each value must fit in uint8
  bool nullable = 4;
}

message Map {
  DType key_type = 1;
  DType value_type = 2;
  bool keys_sorted = 3;
  bool nullable = 4;
}

message DType {
  oneof dtype_type {
    Null null = 1;
    Bool bool = 2;
    Primitive primitive = 3;
    Decimal decimal = 4;
    Utf8 utf8 = 5;
    Binary binary = 6;
    Struct struct = 7;
    List list = 8;
    Extension extension = 9;
    FixedSizeList fixed_size_list = 10; // This is after `Extension` for backwards compatibility.
    Variant variant = 11;
    Union union = 12;
    Map map = 13;
  }
}

message Field {
  oneof field_type {
    string name = 1;
  }
}

message FieldPath {
  repeated Field path = 1;
}