Rust

Install

Install vortex and all the first-party array encodings:

cargo add vortex

Convert

You can either use your own Parquet file or download the example used here.

Use Arrow to read a Parquet file and then construct an uncompressed Vortex array:

use std::fs::File;

use arrow_array::RecordBatchReader;
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use vortex::array::arrays::ChunkedArray;
use vortex::arrow::ArrowSessionExt;
use vortex::session::VortexSession;

let session = VortexSession::default();

let reader = ParquetRecordBatchReaderBuilder::try_new(File::open(
    "../docs/_static/example.parquet",
)?)?
.build()?;

let dtype = session
    .arrow()
    .from_arrow_schema(reader.schema().as_ref())?;
let chunks: Vec<_> = reader
    .map(|record_batch| {
        let batch = record_batch?;
        let schema = batch.schema();
        session.arrow().from_arrow_record_batch(batch, &schema)
    })
    .collect::<VortexResult<_>>()?;
let vortex_array = ChunkedArray::try_new(chunks, dtype)?.into_array();

Compress

Use the sampling compressor to compress the Vortex array and check the relative size:

use vortex::compressor::BtrBlocksCompressor;

let array = PrimitiveArray::new(buffer![42u64; 100_000], Validity::NonNullable);

// You can compress an array in-memory with the BtrBlocks compressor
let session = VortexSession::default();
let compressed = BtrBlocksCompressor::default().compress(
    &array.clone().into_array(),
    &mut session.create_execution_ctx(),
)?;
println!(
    "BtrBlocks size: {} / {}",
    compressed.nbytes(),
    array.into_array().nbytes()
);

Write

Reading and writing both require an async runtime; in this example we use Tokio. The VortexFileWriter knows how to write Vortex arrays to disk:

let array = PrimitiveArray::new(buffer![0u64, 1, 2, 3, 4], Validity::NonNullable);

// Write a Vortex file with the default compression and layout strategy.
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("example.vortex");

session
    .write_options()
    .write(
        &mut tokio::fs::File::create(&path).await?,
        array.into_array().to_array_stream(),
    )
    .await?;

Read

let file = session.open_options().open_path(path.clone()).await?;
let filter = gt(root(), lit(2u64))
    .optimize_recursive(file.dtype())?
    .bind(file.dtype())?;
let array = file
    .scan()?
    .with_filter(filter)
    .into_array_stream()?
    .read_all()
    .await?;

assert_eq!(array.len(), 2);