from typing import Iterable, Dict, Union, List import gzip import json import itertools import requests import numpy as np def post_log(payload): try: url = "https://humaneval-logger.onrender.com/payload" response = requests.post(url, json=payload) return response except Exception as e: print(f"failed to post payload error: {e}") def read_problems(evalset_file: str) -> Dict[str, Dict]: return {task["task_id"]: task for task in stream_jsonl(evalset_file)} def stream_jsonl(filename: str) -> Iterable[Dict]: """ Parses each jsonl line and yields it as a dictionary """ if filename.endswith(".gz"): with open(filename, "rb") as gzfp: with gzip.open(gzfp, 'rt') as fp: for line in fp: if any(not x.isspace() for x in line): yield json.loads(line) else: with open(filename, "r") as fp: for line in fp: if any(not x.isspace() for x in line): yield json.loads(line) def estimate_pass_at_k( num_samples: Union[int, List[int], np.ndarray], num_correct: Union[List[int], np.ndarray], k: int ) -> np.ndarray: """ Estimates pass@k of each problem and returns them in an array. """ def estimator(n: int, c: int, k: int) -> float: """ Calculates 1 - comb(n - c, k) / comb(n, k). """ if n - c < k: return 1.0 return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1)) if isinstance(num_samples, int): num_samples_it = itertools.repeat(num_samples, len(num_correct)) else: assert len(num_samples) == len(num_correct) num_samples_it = iter(num_samples) return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)])