Datasets

from flexorch_sdk import FlexOrch

client = FlexOrch()

dataset = client.datasets.get("ds_abc123")
print(dataset.name)              # "q1-invoices"
print(dataset.row_count)         # 42
print(dataset.status)            # "ready"
print(dataset.available_formats) # ["jsonl", "csv", "parquet"]

datasets = client.datasets.list(page=1, page_size=20)

for ds in datasets.items:
    print(ds.id, ds.name, ds.row_count)

dataset = client.datasets.build(
    name="q1-invoices",
    job_ids=["job_001", "job_002", "job_003"],
)

# JSONL for LLM fine-tuning
client.datasets.export(dataset.id, format="jsonl", path="output.jsonl")

# Parquet for analytics
client.datasets.export(dataset.id, format="parquet", path="output.parquet")

# Markdown for RAG
client.datasets.export(dataset.id, format="md", path="output.md")

result = client.datasets.export_to_s3(
    dataset_id=dataset.id,
    connector_id=1,
    format="jsonl",
    prefix="exports/datasets/",  # optional
)

print(result["s3_key"])      # "exports/datasets/q1-invoices.jsonl"
print(result["size_bytes"])  # 148302

import time

# Start indexing (non-blocking)
client.datasets.index(dataset.id)

# Poll status
while True:
    status = client.datasets.index_status(dataset.id)
    if status["status"] != "indexing":
        break
    time.sleep(3)

print(status["status"])        # "ready"
print(status["total_chunks"])  # 84

results = client.search(
    "invoices over 10000 EUR from Germany",
    top_k=5,
    filters={"quality_grade": "A", "language": "de"},
)

for r in results:
    print(r["score"], r["chunk_text"][:80])

profile = client.datasets.profile(dataset.id)
print(profile["quality_distribution"])  # {"A": 12, "B": 5, "C": 1}
print(profile["pii_type_summary"])