Create experiments

Eval("My Project", {
  data: myDataset,
  task: myTask,
  scores: [Factuality],

  // Experiment name
  experiment: "gpt-5-mini-experiment",

  // Metadata for filtering/analysis
  metadata: {
    model: "gpt-5-mini",
    prompt_version: "v2",
  },

  // Maximum concurrency
  maxConcurrency: 10,

  // Trial count for averaging
  trialCount: 3,
});

Eval("My Project", {
  data: myDataset,
  task: myTask,
  scores: [Factuality],
  trialCount: 10, // Run each input 10 times
});

import { Battle } from "autoevals";
import { Eval, BaseExperiment } from "braintrust";

Eval<string, string, string>(
  "Say Hi Bot", // Replace with your project name
  {
    data: BaseExperiment(),
    task: (input) => {
      return "Hi " + input; // Replace with your task function
    },
    scores: [Battle.partial({ instructions: "Which response said 'Hi'?" })],
  },
);

import { Battle } from "autoevals";
import { Eval, BaseExperiment } from "braintrust";

Eval<string, string, string>(
  "Say Hi Bot", // Replace with your project name
  {
    data: BaseExperiment({ name: "main-123" }),
    task: (input) => {
      return "Hi " + input; // Replace with your task function
    },
    scores: [Battle.partial({ instructions: "Which response said 'Hi'?" })],
  },
);

import { Reporter } from "braintrust";

Reporter(
  "My reporter", // Replace with your reporter name
  {
    reportEval(evaluator, result, opts) {
      // Summarizes the results of a single reporter, and return whatever you
      // want (the full results, a piece of text, or both!)
    },

    reportRun(results) {
      // Takes all the results and summarizes them. Return a true or false
      // which tells the process to exit.
      return true;
    },
  },
);

import { Eval, Attachment } from "braintrust";
import { NumericDiff } from "autoevals";
import path from "path";

function loadPdfs() {
  return ["example.pdf"].map((pdf) => ({
    input: {
      file: new Attachment({
        filename: pdf,
        contentType: "application/pdf",
        data: path.join("files", pdf),
      }),
    },
    // This is a toy example where we check that the file size is what we expect.
    expected: 469513,
  }));
}

async function getFileSize(input: { file: Attachment }) {
  return (await input.file.data()).size;
}

Eval("Project with PDFs", {
  data: loadPdfs,
  task: getFileSize,
  scores: [NumericDiff],
});

import { NumericDiff } from "autoevals";
import { initDataset, Eval, ReadonlyAttachment } from "braintrust";

async function getFileSize(input: {
  file: ReadonlyAttachment;
}): Promise<number> {
  return (await input.file.data()).size;
}

Eval("Project with PDFs", {
  data: initDataset({
    project: "Project with PDFs",
    dataset: "My PDF Dataset",
  }),
  task: getFileSize,
  scores: [NumericDiff],
});

import { initDataset, wrapOpenAI, ReadonlyAttachment } from "braintrust";
import { OpenAI } from "openai";

const client = wrapOpenAI(
  new OpenAI({
    apiKey: process.env.OPENAI_API_KEY,
  }),
);

async function main() {
  const dataset = initDataset({
    project: "Project with images",
    dataset: "My Image Dataset",
  });
  for await (const row of dataset) {
    const attachment: ReadonlyAttachment = row.input.file;
    const attachmentUrl = (await attachment.metadata()).downloadUrl;
    const response = await client.chat.completions.create({
      model: "gpt-5-mini",
      messages: [
        {
          role: "system",
          content: "You are a helpful assistant",
        },
        {
          role: "user",
          content: [
            { type: "text", text: "Please summarize the attached image" },
            { type: "image_url", image_url: { url: attachmentUrl } },
          ],
        },
      ],
    });
    const summary = response.choices[0].message.content || "Unknown";
    console.log(
      `Summary for file ${attachment.reference.filename}: ${summary}`,
    );
  }
}

main();

import { Eval, traced } from "braintrust";

async function callModel(input: string) {
  return traced(
    async (span) => {
      const messages = { messages: [{ role: "system", text: input }] };
      span.log({ input: messages });

      // Replace this with a model call
      const result = {
        content: "China",
        latency: 1,
        prompt_tokens: 10,
        completion_tokens: 2,
      };

      span.log({
        output: result.content,
        metrics: {
          latency: result.latency,
          prompt_tokens: result.prompt_tokens,
          completion_tokens: result.completion_tokens,
        },
      });
      return result.content;
    },
    {
      name: "My AI model",
    },
  );
}

const exactMatch = (args: {
  input: string;
  output: string;
  expected?: string;
}) => {
  return {
    name: "Exact match",
    score: args.output === args.expected ? 1 : 0,
  };
};

Eval("My Evaluation", {
  data: () => [
    { input: "Which country has the highest population?", expected: "China" },
  ],
  task: async (input, { span }) => {
    return await callModel(input);
  },
  scores: [exactMatch],
});