Coverage for src/debputy/packager_provided_files.py: 87%

234 statements  

« prev     ^ index     » next       coverage.py v7.16.1, created at 2026-10-28 20:14 +0000

1import collections.abc 

2import dataclasses 

3from typing import ( 

4 TYPE_CHECKING, 

5) 

6from collections.abc import Mapping, Iterable, Sequence, Container 

7 

8from debputy.packages import BinaryPackage 

9from debputy.plugin.api import VirtualPath 

10from debputy.plugin.api.impl_types import ( 

11 PackagerProvidedFileClassSpec, 

12) 

13from debputy.util import _error, CAN_DETECT_TYPOS, detect_possible_typo 

14 

15if TYPE_CHECKING: 

16 from debputy.plugin.api.feature_set import PluginProvidedFeatureSet 

17 

18 

19_KNOWN_NON_TYPO_EXTENSIONS = frozenset( 

20 { 

21 "conf", 

22 "config", 

23 "sh", 

24 "yml", 

25 "yaml", 

26 "json", 

27 "bash", 

28 "pl", 

29 "py", 

30 "md", 

31 "rst", 

32 # Fairly common image format in older packages 

33 "xpm", 

34 # Cruft of various kind (for unclean work directories). Just to avoid stupid false-positives. 

35 "bak", 

36 "tmp", 

37 "temp", 

38 "orig", 

39 "rej", 

40 } 

41) 

42 

43 

44@dataclasses.dataclass(frozen=True, slots=True) 

45class PackagerProvidedFile: 

46 path: VirtualPath 

47 package_name: str 

48 installed_as_basename: str 

49 provided_key: str 

50 definition: PackagerProvidedFileClassSpec 

51 match_priority: int = 0 

52 fuzzy_match: bool = False 

53 uses_explicit_package_name: bool = False 

54 name_segment: str | None = None 

55 architecture_restriction: str | None = None 

56 expected_path: str | None = None 

57 

58 def compute_dest(self) -> tuple[str, str]: 

59 return self.definition.compute_dest( 

60 self.installed_as_basename, 

61 owning_package=self.package_name, 

62 path=self.path, 

63 ) 

64 

65 

66@dataclasses.dataclass(slots=True) 

67class PerPackagePackagerProvidedResult: 

68 auto_installable: list[PackagerProvidedFile] 

69 reserved_only: dict[str, list[PackagerProvidedFile]] 

70 

71 

72def _find_package_name_prefix( 

73 binary_packages: Mapping[str, BinaryPackage], 

74 main_binary_package: str, 

75 max_periods_in_package_name: int, 

76 path: VirtualPath, 

77 *, 

78 allow_fuzzy_matches: bool = False, 

79) -> Iterable[tuple[str, str, bool, bool]]: 

80 if max_periods_in_package_name < 1: 

81 prefix, remaining = path.name.split(".", 1) 

82 package_name = prefix 

83 bug_950723 = False 

84 if allow_fuzzy_matches and package_name.endswith("@"): 

85 package_name = package_name[:-1] 

86 bug_950723 = True 

87 if package_name in binary_packages: 

88 yield package_name, remaining, True, bug_950723 

89 else: 

90 yield main_binary_package, path.name, False, False 

91 return 

92 

93 parts = path.name.split(".", max_periods_in_package_name + 1) 

94 for p in range(len(parts) - 1, 0, -1): 

95 name = ".".join(parts[0:p]) 

96 bug_950723 = False 

97 if allow_fuzzy_matches and name.endswith("@"): 97 ↛ 98line 97 didn't jump to line 98 because the condition on line 97 was never true

98 name = name[:-1] 

99 bug_950723 = True 

100 

101 if name in binary_packages: 

102 remaining = ".".join(parts[p:]) 

103 yield name, remaining, True, bug_950723 

104 # main package case 

105 yield main_binary_package, path.name, False, False 

106 

107 

108def _iterate_stem_splits( 

109 basename: str, 

110) -> collections.abc.Iterator[tuple[str, str | None, int]]: 

111 stem = basename 

112 period_count = stem.count(".") 

113 yield stem, None, period_count 

114 install_as_name = "" 

115 while period_count > 0: 

116 period_count -= 1 

117 install_as_name_part, stem = stem.split(".", 1) 

118 install_as_name = ( 

119 install_as_name + "." + install_as_name_part 

120 if install_as_name != "" 

121 else install_as_name_part 

122 ) 

123 yield stem, install_as_name, period_count 

124 

125 

126def _find_definition( 

127 packager_provided_files: Mapping[str, PackagerProvidedFileClassSpec], 

128 basename: str, 

129 *, 

130 period2stems: Mapping[int, Sequence[str]] | None = None, 

131 had_arch: bool = False, 

132) -> tuple[str | None, PackagerProvidedFileClassSpec | None, str | None]: 

133 for stem, install_as_name, period_count in _iterate_stem_splits(basename): 

134 definition = packager_provided_files.get(stem) 

135 if definition is not None: 

136 return install_as_name, definition, None 

137 if not period2stems: 

138 continue 

139 stems = period2stems.get(period_count) 

140 

141 if not stems: 

142 continue 

143 # If the stem is also the extension and a known one at that, then 

144 # we do not consider it a typo match (to avoid false positives). 

145 # 

146 # We also ignore "foo.1" since manpages are kind of common. 

147 if not had_arch and (stem in _KNOWN_NON_TYPO_EXTENSIONS or stem.isdigit()): 

148 continue 

149 max_edit_distance = 2 if len(stem) > 3 else 1 

150 matches = detect_possible_typo(stem, stems, max_edit_distance=max_edit_distance) 

151 if matches is not None and len(matches) == 1: 

152 definition = packager_provided_files[matches[0]] 

153 return install_as_name, definition, stem 

154 return None, None, None 

155 

156 

157def _check_mismatches( 

158 path: VirtualPath, 

159 definition: PackagerProvidedFileClassSpec, 

160 owning_package: BinaryPackage, 

161 install_as_name: str | None, 

162 had_arch: bool, 

163) -> None: 

164 if install_as_name is not None and not definition.allow_name_segment: 164 ↛ 165line 164 didn't jump to line 165 because the condition on line 164 was never true

165 _error( 

166 f'The file "{path.fs_path}" looks like a packager provided file for' 

167 f' {owning_package.name} of type {definition.stem} with the custom name "{install_as_name}".' 

168 " However, this file type does not allow custom naming. The file type was registered" 

169 f" by {definition.debputy_plugin_metadata.plugin_name} in case you disagree and want" 

170 " to file a bug/feature request." 

171 ) 

172 if had_arch: 

173 if owning_package.is_arch_all: 173 ↛ 174line 173 didn't jump to line 174 because the condition on line 173 was never true

174 _error( 

175 f'The file "{path.fs_path}" looks like an architecture specific packager provided file for' 

176 f" {owning_package.name} of type {definition.stem}." 

177 " However, the package in question is arch:all. The use of architecture specific files" 

178 " for arch:all packages does not make sense." 

179 ) 

180 if not definition.allow_architecture_segment: 180 ↛ 181line 180 didn't jump to line 181 because the condition on line 180 was never true

181 _error( 

182 f'The file "{path.fs_path}" looks like an architecture specific packager provided file for' 

183 f" {owning_package.name} of type {definition.stem}." 

184 " However, this file type does not allow architecture specific variants. The file type was registered" 

185 f" by {definition.debputy_plugin_metadata.plugin_name} in case you disagree and want" 

186 " to file a bug/feature request." 

187 ) 

188 

189 

190def _split_basename( 

191 basename: str, 

192 owning_package: BinaryPackage, 

193 *, 

194 has_explicit_package: bool = False, 

195 allow_fuzzy_matches: bool = False, 

196 match_arch: frozenset[str] | set[str] | None = None, 

197 match_arch_os: frozenset[str] | set[str] | None = None, 

198) -> tuple[str, int, str | None, bool]: 

199 match_priority = 1 if has_explicit_package else 0 

200 fuzzy_match = False 

201 arch_restriction: str | None = None 

202 if allow_fuzzy_matches and basename.endswith(".in") and len(basename) > 3: 202 ↛ 203line 202 didn't jump to line 203 because the condition on line 202 was never true

203 basename = basename[:-3] 

204 fuzzy_match = True 

205 

206 if "." in basename: 

207 remaining, last_word = basename.rsplit(".", 1) 

208 if match_arch is None: 208 ↛ 212line 208 didn't jump to line 212 because the condition on line 208 was always true

209 match_arch = frozenset( 

210 [owning_package.package_deb_architecture_variable("ARCH")] 

211 ) 

212 if match_arch_os is None: 212 ↛ 217line 212 didn't jump to line 217 because the condition on line 212 was always true

213 match_arch_os = frozenset( 

214 [owning_package.package_deb_architecture_variable("ARCH_OS")] 

215 ) 

216 # We cannot use "resolved_architecture" as it would return "all". 

217 if last_word in match_arch: 

218 match_priority = 3 

219 basename = remaining 

220 arch_restriction = last_word 

221 elif last_word in match_arch_os: 221 ↛ 222line 221 didn't jump to line 222 because the condition on line 221 was never true

222 match_priority = 2 

223 basename = remaining 

224 arch_restriction = last_word 

225 elif last_word == "all" and owning_package.is_arch_all: 225 ↛ 228line 225 didn't jump to line 228 because the condition on line 225 was never true

226 # This case does not make sense, but we detect it, so we can report an error 

227 # via _check_mismatches. 

228 match_priority = -1 

229 basename = remaining 

230 arch_restriction = last_word 

231 

232 return basename, match_priority, arch_restriction, fuzzy_match 

233 

234 

235def _split_path( 

236 packager_provided_files: Mapping[str, PackagerProvidedFileClassSpec], 

237 binary_packages: Mapping[str, BinaryPackage], 

238 main_binary_package: str, 

239 max_periods_in_package_name: int, 

240 path: VirtualPath, 

241 *, 

242 allow_fuzzy_matches: bool = False, 

243 period2stems: Mapping[int, Sequence[str]] | None = None, 

244 known_static_non_ppf_names=frozenset(), 

245 match_arch: frozenset[str] | set[str] | None = None, 

246 match_arch_os: frozenset[str] | set[str] | None = None, 

247) -> Iterable[PackagerProvidedFile]: 

248 owning_package_name = main_binary_package 

249 basename = path.name 

250 match_priority = 0 

251 had_arch = False 

252 if "." not in basename: 

253 definition = packager_provided_files.get(basename) 

254 if definition is None: 

255 return 

256 if definition.packageless_is_fallback_for_all_packages: 

257 yield from ( 

258 PackagerProvidedFile( 

259 path=path, 

260 package_name=n, 

261 installed_as_basename=n, 

262 provided_key=".UNNAMED.", 

263 definition=definition, 

264 match_priority=match_priority, 

265 fuzzy_match=False, 

266 uses_explicit_package_name=False, 

267 name_segment=None, 

268 architecture_restriction=None, 

269 ) 

270 for n, p in binary_packages.items() 

271 if p.package_type in definition.package_types 

272 ) 

273 elif ( 273 ↛ 290line 273 didn't jump to line 290 because the condition on line 273 was always true

274 binary_packages[owning_package_name].package_type 

275 in definition.package_types 

276 ): 

277 # TODO: raise some noise (a la `not allow fuzzy_matches`) if not matching the package type. 

278 yield PackagerProvidedFile( 

279 path=path, 

280 package_name=owning_package_name, 

281 installed_as_basename=owning_package_name, 

282 provided_key=".UNNAMED.", 

283 definition=definition, 

284 match_priority=match_priority, 

285 fuzzy_match=False, 

286 uses_explicit_package_name=False, 

287 name_segment=None, 

288 architecture_restriction=None, 

289 ) 

290 return 

291 if f"debian/{path.name}" in known_static_non_ppf_names: 

292 return 

293 

294 for ( 

295 owning_package_name, 

296 basename, 

297 explicit_package, 

298 bug_950723, 

299 ) in _find_package_name_prefix( 

300 binary_packages, 

301 main_binary_package, 

302 max_periods_in_package_name, 

303 path, 

304 allow_fuzzy_matches=allow_fuzzy_matches, 

305 ): 

306 owning_package = binary_packages[owning_package_name] 

307 

308 basename, match_priority, arch_restriction, fuzzy_match = _split_basename( 

309 basename, 

310 owning_package, 

311 has_explicit_package=explicit_package, 

312 allow_fuzzy_matches=allow_fuzzy_matches, 

313 match_arch=match_arch, 

314 match_arch_os=match_arch_os, 

315 ) 

316 

317 install_as_name, definition, typoed_stem = _find_definition( 

318 packager_provided_files, 

319 basename, 

320 period2stems=period2stems, 

321 had_arch=bool(arch_restriction), 

322 ) 

323 if definition is None: 

324 continue 

325 

326 # Note: bug_950723 implies allow_fuzzy_matches 

327 if bug_950723 and not definition.bug_950723: 327 ↛ 328line 327 didn't jump to line 328 because the condition on line 327 was never true

328 continue 

329 

330 if owning_package.package_type not in definition.package_types: 330 ↛ 332line 330 didn't jump to line 332 because the condition on line 330 was never true

331 # TODO: raise some noise (a la `not allow fuzzy_matches`) 

332 continue 

333 

334 if not allow_fuzzy_matches: 

335 # LSP/Lint checks here but should not use `_check_mismatches` as 

336 # the hard error disrupts them. 

337 _check_mismatches( 

338 path, 

339 definition, 

340 owning_package, 

341 install_as_name, 

342 arch_restriction is not None, 

343 ) 

344 

345 expected_path: str | None = None 

346 if ( 

347 definition.packageless_is_fallback_for_all_packages 

348 and install_as_name is None 

349 and not had_arch 

350 and not explicit_package 

351 and arch_restriction is None 

352 ): 

353 if typoed_stem is not None: 353 ↛ 354line 353 didn't jump to line 354 because the condition on line 353 was never true

354 parent_path = ( 

355 path.parent_dir.path + "/" if path.parent_dir is not None else "" 

356 ) 

357 expected_path = f"{parent_path}{definition.stem}" 

358 if fuzzy_match and path.name.endswith(".in"): 

359 expected_path += ".in" 

360 yield from ( 

361 PackagerProvidedFile( 

362 path=path, 

363 package_name=n, 

364 installed_as_basename=f"{n}@" if bug_950723 else n, 

365 provided_key=".UNNAMED." if bug_950723 else ".UNNAMED@.", 

366 definition=definition, 

367 match_priority=match_priority, 

368 fuzzy_match=fuzzy_match, 

369 uses_explicit_package_name=False, 

370 name_segment=None, 

371 architecture_restriction=None, 

372 expected_path=expected_path, 

373 ) 

374 for n in binary_packages 

375 ) 

376 else: 

377 provided_key = ( 

378 install_as_name if install_as_name is not None else ".UNNAMED." 

379 ) 

380 basename = ( 

381 install_as_name if install_as_name is not None else owning_package_name 

382 ) 

383 if bug_950723: 

384 provided_key = f"{provided_key}@" 

385 basename = f"{basename}@" 

386 package_prefix = f"{owning_package_name}@" 

387 else: 

388 package_prefix = owning_package_name 

389 if typoed_stem: 

390 parent_path = ( 

391 path.parent_dir.path + "/" if path.parent_dir is not None else "" 

392 ) 

393 basename = definition.stem 

394 if install_as_name is not None: 394 ↛ 395line 394 didn't jump to line 395 because the condition on line 394 was never true

395 basename = f"{install_as_name}.{basename}" 

396 if explicit_package: 

397 basename = f"{package_prefix}.{basename}" 

398 if arch_restriction is not None and arch_restriction != "all": 398 ↛ 399line 398 didn't jump to line 399 because the condition on line 398 was never true

399 basename = f"{basename}.{arch_restriction}" 

400 expected_path = f"{parent_path}{basename}" 

401 if fuzzy_match and path.name.endswith(".in"): 401 ↛ 402line 401 didn't jump to line 402 because the condition on line 401 was never true

402 expected_path += ".in" 

403 yield PackagerProvidedFile( 

404 path=path, 

405 package_name=owning_package_name, 

406 installed_as_basename=basename, 

407 provided_key=provided_key, 

408 definition=definition, 

409 match_priority=match_priority, 

410 fuzzy_match=fuzzy_match, 

411 uses_explicit_package_name=bool(explicit_package), 

412 name_segment=install_as_name, 

413 architecture_restriction=arch_restriction, 

414 expected_path=expected_path, 

415 ) 

416 return 

417 

418 

419def _period_stem(stems: Iterable[str]) -> Mapping[int, Sequence[str]]: 

420 result: dict[int, list[str]] = {} 

421 for stem in stems: 

422 period_count = stem.count(".") 

423 matched_stems = result.get(period_count) 

424 if not matched_stems: 

425 matched_stems = [stem] 

426 result[period_count] = matched_stems 

427 else: 

428 matched_stems.append(stem) 

429 return result 

430 

431 

432def _find_main_package_name( 

433 binary_packages: Mapping[str, BinaryPackage], 

434 *, 

435 allow_fuzzy_matches: bool = False, 

436) -> str | None: 

437 main_packages = [p.name for p in binary_packages.values() if p.is_main_package] 

438 if not main_packages: 438 ↛ 439line 438 didn't jump to line 439 because the condition on line 438 was never true

439 assert allow_fuzzy_matches 

440 return next( 

441 iter(p.name for p in binary_packages.values() if "Package" in p.fields), 

442 None, 

443 ) 

444 return main_packages[0] 

445 

446 

447@dataclasses.dataclass(slots=True, frozen=True) 

448class PackagingFileClassification: 

449 path: VirtualPath 

450 packager_provided_files_per_package: None | ( 

451 Mapping[str, Sequence[PackagerProvidedFile]] 

452 ) 

453 

454 

455def classify_debian_packaging_files( 

456 plugin_feature_set: "PluginProvidedFeatureSet", 

457 debian_dir: VirtualPath, 

458 binary_packages: Mapping[str, BinaryPackage], 

459 *, 

460 allow_fuzzy_matches: bool = False, 

461 detect_typos: bool = False, 

462 ignore_paths: Container[str] = frozenset(), 

463 match_arch: frozenset[str] | set[str] | None = None, 

464 match_arch_os: frozenset[str] | set[str] | None = None, 

465 multi_match: bool = False, 

466) -> Iterable[PackagingFileClassification]: 

467 packager_provided_files = plugin_feature_set.packager_provided_files 

468 known_static_non_ppf_names: frozenset[str] = frozenset( 

469 { 

470 p.detection_value 

471 for p in plugin_feature_set.known_packaging_files.values() 

472 if p.detection_method == "path" 

473 } 

474 ) 

475 main_binary_package = _find_main_package_name( 

476 binary_packages, 

477 allow_fuzzy_matches=allow_fuzzy_matches, 

478 ) 

479 if main_binary_package is None: 479 ↛ 480line 479 didn't jump to line 480 because the condition on line 479 was never true

480 return 

481 provided_files_by_key: dict[tuple[str | None, ...], PackagerProvidedFile] = {} 

482 max_periods_in_package_name = max(name.count(".") for name in binary_packages) 

483 if detect_typos and CAN_DETECT_TYPOS: 

484 period2stems = _period_stem(packager_provided_files.keys()) 

485 else: 

486 period2stems = {} 

487 

488 paths = [] 

489 

490 for entry in debian_dir.iterdir(): 

491 if entry.is_dir or entry.name.startswith("."): 

492 continue 

493 if entry.path in ignore_paths: 

494 continue 

495 paths.append(entry) 

496 matching_ppfs = _split_path( 

497 packager_provided_files, 

498 binary_packages, 

499 main_binary_package, 

500 max_periods_in_package_name, 

501 entry, 

502 allow_fuzzy_matches=allow_fuzzy_matches, 

503 period2stems=period2stems, 

504 known_static_non_ppf_names=known_static_non_ppf_names, 

505 match_arch=match_arch, 

506 match_arch_os=match_arch_os, 

507 ) 

508 for packager_provided_file in matching_ppfs: 

509 match_key = ( 

510 packager_provided_file.package_name, 

511 packager_provided_file.definition.stem, 

512 packager_provided_file.provided_key, 

513 ) 

514 if multi_match: 

515 match_key = ( 

516 *match_key, 

517 packager_provided_file.architecture_restriction, 

518 ) 

519 existing = provided_files_by_key.get(match_key) 

520 if ( 

521 existing is not None 

522 and existing.match_priority > packager_provided_file.match_priority 

523 ): 

524 continue 

525 provided_files_by_key[match_key] = packager_provided_file 

526 

527 paths2ppfs_per_package = dict[str, dict[str, list[PackagerProvidedFile]]]() 

528 for packager_provided_file in provided_files_by_key.values(): 

529 package_name = packager_provided_file.package_name 

530 path_name = packager_provided_file.path.path 

531 ppfs_per_package = paths2ppfs_per_package.get(path_name) 

532 if ppfs_per_package is None: 

533 ppfs_per_package = collections.defaultdict(list) 

534 paths2ppfs_per_package[path_name] = ppfs_per_package 

535 ppfs_per_package[package_name].append(packager_provided_file) 

536 

537 for entry in paths: 

538 yield PackagingFileClassification( 

539 entry, 

540 paths2ppfs_per_package.get(entry.path), 

541 ) 

542 

543 

544def detect_all_packager_provided_files( 

545 plugin_feature_set: "PluginProvidedFeatureSet", 

546 debian_dir: VirtualPath, 

547 binary_packages: Mapping[str, BinaryPackage], 

548 *, 

549 allow_fuzzy_matches: bool = False, 

550 detect_typos: bool = False, 

551 ignore_paths: Container[str] = frozenset(), 

552 match_arch: frozenset[str] | set[str] | None = None, 

553 match_arch_os: frozenset[str] | set[str] | None = None, 

554 multi_match: bool = False, 

555) -> dict[str, PerPackagePackagerProvidedResult]: 

556 result = { 

557 n: PerPackagePackagerProvidedResult([], collections.defaultdict(list)) 

558 for n in binary_packages 

559 } 

560 for classified_path in classify_debian_packaging_files( 

561 plugin_feature_set, 

562 debian_dir, 

563 binary_packages, 

564 allow_fuzzy_matches=allow_fuzzy_matches, 

565 detect_typos=detect_typos, 

566 ignore_paths=ignore_paths, 

567 match_arch=match_arch, 

568 match_arch_os=match_arch_os, 

569 multi_match=multi_match, 

570 ): 

571 provided_files = classified_path.packager_provided_files_per_package 

572 if not provided_files: 

573 continue 

574 for package_name, provided_file_data in provided_files.items(): 

575 per_package_result = result[package_name] 

576 per_package_result.auto_installable.extend( 

577 x for x in provided_file_data if not x.definition.reservation_only 

578 ) 

579 reservation_only = per_package_result.reserved_only 

580 for packager_provided_file in provided_file_data: 

581 if not packager_provided_file.definition.reservation_only: 

582 continue 

583 reservation_only[packager_provided_file.definition.stem].append( 

584 packager_provided_file 

585 ) 

586 

587 return result