Coverage for src\cmvk\hf_utils.py: 0%

1"""

2 Hugging Face Hub Utilities for CMVK

3

4 Lightweight utilities for pushing/pulling datasets and experiment results

5 to the Hugging Face Hub. Designed for research reproducibility.

6

7 Usage:

8 from cmvk.hf_utils import upload_experiment_logs, download_dataset

9

10 # Upload experiment results

11 upload_experiment_logs(

12 results_path="experiments/results.json",

13 repo_id="imran-siddique/cmvk-benchmark-data"

14 )

15

16 # Download shared datasets

17 download_dataset(

18 repo_id="imran-siddique/cmvk-benchmark-data",

19 filename="datasets/humaneval_50.json",

20 local_path="experiments/datasets/"

21 )

22

23 Requirements:

24 pip install huggingface_hub

25"""

26

27 from __future__ import annotations

28

29 import json

30 import logging

31 from pathlib import Path

32 from typing import TYPE_CHECKING, Any

33

34 if TYPE_CHECKING:

35 from huggingface_hub import HfApi

36

37 logger = logging.getLogger(__name__)

38

39# Default repository configuration

40 DEFAULT_REPO_ID = "imran-siddique/cmvk-benchmark-data"

41 DEFAULT_REPO_TYPE = "dataset"

42

43

44 class HuggingFaceHubError(Exception):

45 """Raised when Hugging Face Hub operations fail."""

46

47 pass

48

49

50 def _get_api() -> HfApi:

51 """Get HfApi instance, raising helpful error if not installed."""

52 try:

53 from huggingface_hub import HfApi

54

55 return HfApi()

56 except ImportError as e:

57 raise HuggingFaceHubError(

58 "huggingface_hub is required for HF integration. "

59 "Install with: pip install huggingface_hub"

60 ) from e

61

62

63 def check_auth() -> bool:

64 """

65 Check if Hugging Face authentication is configured.

66

67 Returns:

68 True if authenticated, False otherwise.

69

70 Example:

71 >>> if check_auth():

72 ... upload_experiment_logs("results.json")

73 """

74 try:

75 api = _get_api()

76 api.whoami()

77 return True

78 except Exception:

79 return False

80

81

82 def upload_experiment_logs(

83 results_path: str | Path,

84 repo_id: str = DEFAULT_REPO_ID,

85 path_in_repo: str | None = None,

86 commit_message: str | None = None,

87 private: bool = False,

88) -> str:

89 """

90 Upload experiment results/logs to Hugging Face Hub.

91

92 Args:

93 results_path: Path to local results file (JSON, CSV, etc.)

94 repo_id: Hugging Face repository ID (e.g., "username/repo-name")

95 path_in_repo: Path within the repository. If None, uses filename.

96 commit_message: Custom commit message. Auto-generated if None.

97 private: Whether to make the repository private.

98

99 Returns:

100 URL of the uploaded file.

101

102 Raises:

103 HuggingFaceHubError: If upload fails or file not found.

104 FileNotFoundError: If results_path doesn't exist.

105

106 Example:

107 >>> url = upload_experiment_logs(

108 ... "experiments/results.json",

109 ... repo_id="imran-siddique/cmvk-benchmark-data"

110 ... )

111 >>> print(f"Uploaded to: {url}")

112 """

113 from huggingface_hub import create_repo

114

115 results_path = Path(results_path)

116 if not results_path.exists():

117 raise FileNotFoundError(f"Results file not found: {results_path}")

118

119 api = _get_api()

120

121 # Create repo if needed

122 try:

123 create_repo(

124 repo_id,

125 repo_type=DEFAULT_REPO_TYPE,

126 private=private,

127 exist_ok=True,

128 )

129 except Exception as e:

130 logger.debug(f"Repository creation skipped: {e}")

131

132 # Determine path in repo

133 if path_in_repo is None:

134 path_in_repo = f"results/{results_path.name}"

135

136 # Auto-generate commit message

137 if commit_message is None:

138 commit_message = f"Upload experiment results: {results_path.name}"

139

140 try:

141 url: str = api.upload_file(

142 path_or_fileobj=str(results_path),

143 path_in_repo=path_in_repo,

144 repo_id=repo_id,

145 repo_type=DEFAULT_REPO_TYPE,

146 commit_message=commit_message,

147 )

148 logger.info(f"Uploaded {results_path.name} to {url}")

149 return url

150 except Exception as e:

151 raise HuggingFaceHubError(f"Failed to upload {results_path}: {e}") from e

152

153

154 def upload_traces(

155 trace_dir: str | Path,

156 repo_id: str = DEFAULT_REPO_ID,

157 max_files: int = 100,

158) -> list[str]:

159 """

160 Upload execution traces directory to Hugging Face Hub.

161

162 Args:

163 trace_dir: Directory containing trace JSON files.

164 repo_id: Hugging Face repository ID.

165 max_files: Maximum number of files to upload.

166

167 Returns:

168 List of uploaded file URLs.

169

170 Example:

171 >>> urls = upload_traces("logs/traces/")

172 >>> print(f"Uploaded {len(urls)} trace files")

173 """

174 trace_path = Path(trace_dir)

175 if not trace_path.exists():

176 raise FileNotFoundError(f"Trace directory not found: {trace_path}")

177

178 api = _get_api()

179 trace_files = sorted(trace_path.glob("*.json"))[:max_files]

180

181 urls = []

182 for trace_file in trace_files:

183 try:

184 url = api.upload_file(

185 path_or_fileobj=str(trace_file),

186 path_in_repo=f"traces/{trace_file.name}",

187 repo_id=repo_id,

188 repo_type=DEFAULT_REPO_TYPE,

189 commit_message=f"Upload trace: {trace_file.name}",

190 )

191 urls.append(url)

192 except Exception as e:

193 logger.warning(f"Failed to upload {trace_file.name}: {e}")

194

195 logger.info(f"Uploaded {len(urls)}/{len(trace_files)} trace files")

196 return urls

197

198

199 def download_dataset(

200 repo_id: str = DEFAULT_REPO_ID,

201 filename: str = "datasets/humaneval_50.json",

202 local_path: str | Path = "experiments/datasets/",

203 revision: str = "main",

204) -> Path:

205 """

206 Download a dataset file from Hugging Face Hub.

207

208 Args:

209 repo_id: Hugging Face repository ID.

210 filename: Path to file within the repository.

211 local_path: Local directory to save the file.

212 revision: Git revision (branch, tag, or commit).

213

214 Returns:

215 Path to the downloaded file.

216

217 Example:

218 >>> path = download_dataset(

219 ... filename="datasets/humaneval_50.json",

220 ... local_path="experiments/datasets/"

221 ... )

222 >>> with open(path) as f:

223 ... data = json.load(f)

224 """

225 from huggingface_hub import hf_hub_download

226

227 local_path = Path(local_path)

228 local_path.mkdir(parents=True, exist_ok=True)

229

230 try:

231 downloaded = hf_hub_download(

232 repo_id=repo_id,

233 filename=filename,

234 repo_type=DEFAULT_REPO_TYPE,

235 revision=revision,

236 local_dir=str(local_path),

237 )

238 result_path = Path(downloaded)

239 logger.info(f"Downloaded {filename} to {result_path}")

240 return result_path

241 except Exception as e:

242 raise HuggingFaceHubError(f"Failed to download {filename}: {e}") from e

243

244

245 def upload_verification_dataset(

246 data: list[dict[str, Any]],

247 dataset_name: str,

248 repo_id: str = DEFAULT_REPO_ID,

249 description: str | None = None,

250) -> str:

251 """

252 Upload a verification dataset (list of problems/results) to HF Hub.

253

254 Args:

255 data: List of problem dictionaries with verification results.

256 dataset_name: Name for the dataset file.

257 repo_id: Hugging Face repository ID.

258 description: Optional description to include in metadata.

259

260 Returns:

261 URL of the uploaded dataset.

262

263 Example:

264 >>> results = [

265 ... {"task_id": "HE/1", "success": True, "drift_score": 0.15},

266 ... {"task_id": "HE/2", "success": False, "drift_score": 0.89},

267 ... ]

268 >>> upload_verification_dataset(results, "humaneval_run_001")

269 """

270 import tempfile

271 from datetime import datetime

272

273 # Create metadata wrapper

274 dataset = {

275 "name": dataset_name,

276 "created_at": datetime.now().isoformat(),

277 "description": description or f"CMVK verification dataset: {dataset_name}",

278 "count": len(data),

279 "data": data,

280 }

281

282 # Write to temp file and upload

283 with tempfile.NamedTemporaryFile(

284 mode="w",

285 suffix=".json",

286 delete=False,

287 ) as f:

288 json.dump(dataset, f, indent=2)

289 temp_path = Path(f.name)

290

291 try:

292 return upload_experiment_logs(

293 temp_path,

294 repo_id=repo_id,

295 path_in_repo=f"datasets/{dataset_name}.json",

296 commit_message=f"Upload verification dataset: {dataset_name}",

297 )

298 finally:

299 temp_path.unlink(missing_ok=True)