Datasets

import { FlexOrchClient } from 'flexorch-sdk';

const client = new FlexOrchClient();

const dataset = await client.datasets.get('ds_abc123');
console.log(dataset.name);             // "q1-invoices"
console.log(dataset.rowCount);         // 42
console.log(dataset.status);           // "ready"
console.log(dataset.availableFormats); // ["jsonl", "csv", "parquet"]

const datasets = await client.datasets.list({ page: 1, pageSize: 20 });

for (const ds of datasets) {
  console.log(ds.id, ds.name, ds.rowCount);
}

import { writeFile } from 'node:fs/promises';

const dataset = await client.datasets.get('ds_abc123');

// JSONL for LLM fine-tuning
const jsonl = await dataset.export('jsonl');
await writeFile('output.jsonl', jsonl);

// Parquet for analytics
const parquet = await dataset.export('parquet');
await writeFile('output.parquet', parquet);

// Markdown for RAG
const md = await dataset.export('md');
await writeFile('output.md', md);

const result = await dataset.exportToS3(
  'connector_id_here',
  'jsonl',
  'exports/datasets/', // optional prefix
);

console.log(result.s3Key);     // "exports/datasets/q1-invoices.jsonl"
console.log(result.sizeBytes); // 148302

// Start indexing (async, returns immediately)
await dataset.index();

// Poll status
let status = await dataset.indexStatus();
while (status.status === 'indexing') {
  await new Promise(r => setTimeout(r, 3000));
  status = await dataset.indexStatus();
}

console.log(status.status);        // "ready"
console.log(status.totalChunks);   // 84

const results = await client.search('invoices over 10000 EUR from Germany', {
  topK: 5,
  filters: { qualityGrade: 'A', language: 'de' },
});

for (const r of results) {
  console.log(r.score, r.chunkText.slice(0, 80));
}