Coverage for src/debputy/packager_provided_files.py: 87%
234 statements
« prev ^ index » next coverage.py v7.16.1, created at 2026-10-28 20:14 +0000
« prev ^ index » next coverage.py v7.16.1, created at 2026-10-28 20:14 +0000
1import collections.abc
2import dataclasses
3from typing import (
4 TYPE_CHECKING,
5)
6from collections.abc import Mapping, Iterable, Sequence, Container
8from debputy.packages import BinaryPackage
9from debputy.plugin.api import VirtualPath
10from debputy.plugin.api.impl_types import (
11 PackagerProvidedFileClassSpec,
12)
13from debputy.util import _error, CAN_DETECT_TYPOS, detect_possible_typo
15if TYPE_CHECKING:
16 from debputy.plugin.api.feature_set import PluginProvidedFeatureSet
19_KNOWN_NON_TYPO_EXTENSIONS = frozenset(
20 {
21 "conf",
22 "config",
23 "sh",
24 "yml",
25 "yaml",
26 "json",
27 "bash",
28 "pl",
29 "py",
30 "md",
31 "rst",
32 # Fairly common image format in older packages
33 "xpm",
34 # Cruft of various kind (for unclean work directories). Just to avoid stupid false-positives.
35 "bak",
36 "tmp",
37 "temp",
38 "orig",
39 "rej",
40 }
41)
44@dataclasses.dataclass(frozen=True, slots=True)
45class PackagerProvidedFile:
46 path: VirtualPath
47 package_name: str
48 installed_as_basename: str
49 provided_key: str
50 definition: PackagerProvidedFileClassSpec
51 match_priority: int = 0
52 fuzzy_match: bool = False
53 uses_explicit_package_name: bool = False
54 name_segment: str | None = None
55 architecture_restriction: str | None = None
56 expected_path: str | None = None
58 def compute_dest(self) -> tuple[str, str]:
59 return self.definition.compute_dest(
60 self.installed_as_basename,
61 owning_package=self.package_name,
62 path=self.path,
63 )
66@dataclasses.dataclass(slots=True)
67class PerPackagePackagerProvidedResult:
68 auto_installable: list[PackagerProvidedFile]
69 reserved_only: dict[str, list[PackagerProvidedFile]]
72def _find_package_name_prefix(
73 binary_packages: Mapping[str, BinaryPackage],
74 main_binary_package: str,
75 max_periods_in_package_name: int,
76 path: VirtualPath,
77 *,
78 allow_fuzzy_matches: bool = False,
79) -> Iterable[tuple[str, str, bool, bool]]:
80 if max_periods_in_package_name < 1:
81 prefix, remaining = path.name.split(".", 1)
82 package_name = prefix
83 bug_950723 = False
84 if allow_fuzzy_matches and package_name.endswith("@"):
85 package_name = package_name[:-1]
86 bug_950723 = True
87 if package_name in binary_packages:
88 yield package_name, remaining, True, bug_950723
89 else:
90 yield main_binary_package, path.name, False, False
91 return
93 parts = path.name.split(".", max_periods_in_package_name + 1)
94 for p in range(len(parts) - 1, 0, -1):
95 name = ".".join(parts[0:p])
96 bug_950723 = False
97 if allow_fuzzy_matches and name.endswith("@"): 97 ↛ 98line 97 didn't jump to line 98 because the condition on line 97 was never true
98 name = name[:-1]
99 bug_950723 = True
101 if name in binary_packages:
102 remaining = ".".join(parts[p:])
103 yield name, remaining, True, bug_950723
104 # main package case
105 yield main_binary_package, path.name, False, False
108def _iterate_stem_splits(
109 basename: str,
110) -> collections.abc.Iterator[tuple[str, str | None, int]]:
111 stem = basename
112 period_count = stem.count(".")
113 yield stem, None, period_count
114 install_as_name = ""
115 while period_count > 0:
116 period_count -= 1
117 install_as_name_part, stem = stem.split(".", 1)
118 install_as_name = (
119 install_as_name + "." + install_as_name_part
120 if install_as_name != ""
121 else install_as_name_part
122 )
123 yield stem, install_as_name, period_count
126def _find_definition(
127 packager_provided_files: Mapping[str, PackagerProvidedFileClassSpec],
128 basename: str,
129 *,
130 period2stems: Mapping[int, Sequence[str]] | None = None,
131 had_arch: bool = False,
132) -> tuple[str | None, PackagerProvidedFileClassSpec | None, str | None]:
133 for stem, install_as_name, period_count in _iterate_stem_splits(basename):
134 definition = packager_provided_files.get(stem)
135 if definition is not None:
136 return install_as_name, definition, None
137 if not period2stems:
138 continue
139 stems = period2stems.get(period_count)
141 if not stems:
142 continue
143 # If the stem is also the extension and a known one at that, then
144 # we do not consider it a typo match (to avoid false positives).
145 #
146 # We also ignore "foo.1" since manpages are kind of common.
147 if not had_arch and (stem in _KNOWN_NON_TYPO_EXTENSIONS or stem.isdigit()):
148 continue
149 max_edit_distance = 2 if len(stem) > 3 else 1
150 matches = detect_possible_typo(stem, stems, max_edit_distance=max_edit_distance)
151 if matches is not None and len(matches) == 1:
152 definition = packager_provided_files[matches[0]]
153 return install_as_name, definition, stem
154 return None, None, None
157def _check_mismatches(
158 path: VirtualPath,
159 definition: PackagerProvidedFileClassSpec,
160 owning_package: BinaryPackage,
161 install_as_name: str | None,
162 had_arch: bool,
163) -> None:
164 if install_as_name is not None and not definition.allow_name_segment: 164 ↛ 165line 164 didn't jump to line 165 because the condition on line 164 was never true
165 _error(
166 f'The file "{path.fs_path}" looks like a packager provided file for'
167 f' {owning_package.name} of type {definition.stem} with the custom name "{install_as_name}".'
168 " However, this file type does not allow custom naming. The file type was registered"
169 f" by {definition.debputy_plugin_metadata.plugin_name} in case you disagree and want"
170 " to file a bug/feature request."
171 )
172 if had_arch:
173 if owning_package.is_arch_all: 173 ↛ 174line 173 didn't jump to line 174 because the condition on line 173 was never true
174 _error(
175 f'The file "{path.fs_path}" looks like an architecture specific packager provided file for'
176 f" {owning_package.name} of type {definition.stem}."
177 " However, the package in question is arch:all. The use of architecture specific files"
178 " for arch:all packages does not make sense."
179 )
180 if not definition.allow_architecture_segment: 180 ↛ 181line 180 didn't jump to line 181 because the condition on line 180 was never true
181 _error(
182 f'The file "{path.fs_path}" looks like an architecture specific packager provided file for'
183 f" {owning_package.name} of type {definition.stem}."
184 " However, this file type does not allow architecture specific variants. The file type was registered"
185 f" by {definition.debputy_plugin_metadata.plugin_name} in case you disagree and want"
186 " to file a bug/feature request."
187 )
190def _split_basename(
191 basename: str,
192 owning_package: BinaryPackage,
193 *,
194 has_explicit_package: bool = False,
195 allow_fuzzy_matches: bool = False,
196 match_arch: frozenset[str] | set[str] | None = None,
197 match_arch_os: frozenset[str] | set[str] | None = None,
198) -> tuple[str, int, str | None, bool]:
199 match_priority = 1 if has_explicit_package else 0
200 fuzzy_match = False
201 arch_restriction: str | None = None
202 if allow_fuzzy_matches and basename.endswith(".in") and len(basename) > 3: 202 ↛ 203line 202 didn't jump to line 203 because the condition on line 202 was never true
203 basename = basename[:-3]
204 fuzzy_match = True
206 if "." in basename:
207 remaining, last_word = basename.rsplit(".", 1)
208 if match_arch is None: 208 ↛ 212line 208 didn't jump to line 212 because the condition on line 208 was always true
209 match_arch = frozenset(
210 [owning_package.package_deb_architecture_variable("ARCH")]
211 )
212 if match_arch_os is None: 212 ↛ 217line 212 didn't jump to line 217 because the condition on line 212 was always true
213 match_arch_os = frozenset(
214 [owning_package.package_deb_architecture_variable("ARCH_OS")]
215 )
216 # We cannot use "resolved_architecture" as it would return "all".
217 if last_word in match_arch:
218 match_priority = 3
219 basename = remaining
220 arch_restriction = last_word
221 elif last_word in match_arch_os: 221 ↛ 222line 221 didn't jump to line 222 because the condition on line 221 was never true
222 match_priority = 2
223 basename = remaining
224 arch_restriction = last_word
225 elif last_word == "all" and owning_package.is_arch_all: 225 ↛ 228line 225 didn't jump to line 228 because the condition on line 225 was never true
226 # This case does not make sense, but we detect it, so we can report an error
227 # via _check_mismatches.
228 match_priority = -1
229 basename = remaining
230 arch_restriction = last_word
232 return basename, match_priority, arch_restriction, fuzzy_match
235def _split_path(
236 packager_provided_files: Mapping[str, PackagerProvidedFileClassSpec],
237 binary_packages: Mapping[str, BinaryPackage],
238 main_binary_package: str,
239 max_periods_in_package_name: int,
240 path: VirtualPath,
241 *,
242 allow_fuzzy_matches: bool = False,
243 period2stems: Mapping[int, Sequence[str]] | None = None,
244 known_static_non_ppf_names=frozenset(),
245 match_arch: frozenset[str] | set[str] | None = None,
246 match_arch_os: frozenset[str] | set[str] | None = None,
247) -> Iterable[PackagerProvidedFile]:
248 owning_package_name = main_binary_package
249 basename = path.name
250 match_priority = 0
251 had_arch = False
252 if "." not in basename:
253 definition = packager_provided_files.get(basename)
254 if definition is None:
255 return
256 if definition.packageless_is_fallback_for_all_packages:
257 yield from (
258 PackagerProvidedFile(
259 path=path,
260 package_name=n,
261 installed_as_basename=n,
262 provided_key=".UNNAMED.",
263 definition=definition,
264 match_priority=match_priority,
265 fuzzy_match=False,
266 uses_explicit_package_name=False,
267 name_segment=None,
268 architecture_restriction=None,
269 )
270 for n, p in binary_packages.items()
271 if p.package_type in definition.package_types
272 )
273 elif ( 273 ↛ 290line 273 didn't jump to line 290 because the condition on line 273 was always true
274 binary_packages[owning_package_name].package_type
275 in definition.package_types
276 ):
277 # TODO: raise some noise (a la `not allow fuzzy_matches`) if not matching the package type.
278 yield PackagerProvidedFile(
279 path=path,
280 package_name=owning_package_name,
281 installed_as_basename=owning_package_name,
282 provided_key=".UNNAMED.",
283 definition=definition,
284 match_priority=match_priority,
285 fuzzy_match=False,
286 uses_explicit_package_name=False,
287 name_segment=None,
288 architecture_restriction=None,
289 )
290 return
291 if f"debian/{path.name}" in known_static_non_ppf_names:
292 return
294 for (
295 owning_package_name,
296 basename,
297 explicit_package,
298 bug_950723,
299 ) in _find_package_name_prefix(
300 binary_packages,
301 main_binary_package,
302 max_periods_in_package_name,
303 path,
304 allow_fuzzy_matches=allow_fuzzy_matches,
305 ):
306 owning_package = binary_packages[owning_package_name]
308 basename, match_priority, arch_restriction, fuzzy_match = _split_basename(
309 basename,
310 owning_package,
311 has_explicit_package=explicit_package,
312 allow_fuzzy_matches=allow_fuzzy_matches,
313 match_arch=match_arch,
314 match_arch_os=match_arch_os,
315 )
317 install_as_name, definition, typoed_stem = _find_definition(
318 packager_provided_files,
319 basename,
320 period2stems=period2stems,
321 had_arch=bool(arch_restriction),
322 )
323 if definition is None:
324 continue
326 # Note: bug_950723 implies allow_fuzzy_matches
327 if bug_950723 and not definition.bug_950723: 327 ↛ 328line 327 didn't jump to line 328 because the condition on line 327 was never true
328 continue
330 if owning_package.package_type not in definition.package_types: 330 ↛ 332line 330 didn't jump to line 332 because the condition on line 330 was never true
331 # TODO: raise some noise (a la `not allow fuzzy_matches`)
332 continue
334 if not allow_fuzzy_matches:
335 # LSP/Lint checks here but should not use `_check_mismatches` as
336 # the hard error disrupts them.
337 _check_mismatches(
338 path,
339 definition,
340 owning_package,
341 install_as_name,
342 arch_restriction is not None,
343 )
345 expected_path: str | None = None
346 if (
347 definition.packageless_is_fallback_for_all_packages
348 and install_as_name is None
349 and not had_arch
350 and not explicit_package
351 and arch_restriction is None
352 ):
353 if typoed_stem is not None: 353 ↛ 354line 353 didn't jump to line 354 because the condition on line 353 was never true
354 parent_path = (
355 path.parent_dir.path + "/" if path.parent_dir is not None else ""
356 )
357 expected_path = f"{parent_path}{definition.stem}"
358 if fuzzy_match and path.name.endswith(".in"):
359 expected_path += ".in"
360 yield from (
361 PackagerProvidedFile(
362 path=path,
363 package_name=n,
364 installed_as_basename=f"{n}@" if bug_950723 else n,
365 provided_key=".UNNAMED." if bug_950723 else ".UNNAMED@.",
366 definition=definition,
367 match_priority=match_priority,
368 fuzzy_match=fuzzy_match,
369 uses_explicit_package_name=False,
370 name_segment=None,
371 architecture_restriction=None,
372 expected_path=expected_path,
373 )
374 for n in binary_packages
375 )
376 else:
377 provided_key = (
378 install_as_name if install_as_name is not None else ".UNNAMED."
379 )
380 basename = (
381 install_as_name if install_as_name is not None else owning_package_name
382 )
383 if bug_950723:
384 provided_key = f"{provided_key}@"
385 basename = f"{basename}@"
386 package_prefix = f"{owning_package_name}@"
387 else:
388 package_prefix = owning_package_name
389 if typoed_stem:
390 parent_path = (
391 path.parent_dir.path + "/" if path.parent_dir is not None else ""
392 )
393 basename = definition.stem
394 if install_as_name is not None: 394 ↛ 395line 394 didn't jump to line 395 because the condition on line 394 was never true
395 basename = f"{install_as_name}.{basename}"
396 if explicit_package:
397 basename = f"{package_prefix}.{basename}"
398 if arch_restriction is not None and arch_restriction != "all": 398 ↛ 399line 398 didn't jump to line 399 because the condition on line 398 was never true
399 basename = f"{basename}.{arch_restriction}"
400 expected_path = f"{parent_path}{basename}"
401 if fuzzy_match and path.name.endswith(".in"): 401 ↛ 402line 401 didn't jump to line 402 because the condition on line 401 was never true
402 expected_path += ".in"
403 yield PackagerProvidedFile(
404 path=path,
405 package_name=owning_package_name,
406 installed_as_basename=basename,
407 provided_key=provided_key,
408 definition=definition,
409 match_priority=match_priority,
410 fuzzy_match=fuzzy_match,
411 uses_explicit_package_name=bool(explicit_package),
412 name_segment=install_as_name,
413 architecture_restriction=arch_restriction,
414 expected_path=expected_path,
415 )
416 return
419def _period_stem(stems: Iterable[str]) -> Mapping[int, Sequence[str]]:
420 result: dict[int, list[str]] = {}
421 for stem in stems:
422 period_count = stem.count(".")
423 matched_stems = result.get(period_count)
424 if not matched_stems:
425 matched_stems = [stem]
426 result[period_count] = matched_stems
427 else:
428 matched_stems.append(stem)
429 return result
432def _find_main_package_name(
433 binary_packages: Mapping[str, BinaryPackage],
434 *,
435 allow_fuzzy_matches: bool = False,
436) -> str | None:
437 main_packages = [p.name for p in binary_packages.values() if p.is_main_package]
438 if not main_packages: 438 ↛ 439line 438 didn't jump to line 439 because the condition on line 438 was never true
439 assert allow_fuzzy_matches
440 return next(
441 iter(p.name for p in binary_packages.values() if "Package" in p.fields),
442 None,
443 )
444 return main_packages[0]
447@dataclasses.dataclass(slots=True, frozen=True)
448class PackagingFileClassification:
449 path: VirtualPath
450 packager_provided_files_per_package: None | (
451 Mapping[str, Sequence[PackagerProvidedFile]]
452 )
455def classify_debian_packaging_files(
456 plugin_feature_set: "PluginProvidedFeatureSet",
457 debian_dir: VirtualPath,
458 binary_packages: Mapping[str, BinaryPackage],
459 *,
460 allow_fuzzy_matches: bool = False,
461 detect_typos: bool = False,
462 ignore_paths: Container[str] = frozenset(),
463 match_arch: frozenset[str] | set[str] | None = None,
464 match_arch_os: frozenset[str] | set[str] | None = None,
465 multi_match: bool = False,
466) -> Iterable[PackagingFileClassification]:
467 packager_provided_files = plugin_feature_set.packager_provided_files
468 known_static_non_ppf_names: frozenset[str] = frozenset(
469 {
470 p.detection_value
471 for p in plugin_feature_set.known_packaging_files.values()
472 if p.detection_method == "path"
473 }
474 )
475 main_binary_package = _find_main_package_name(
476 binary_packages,
477 allow_fuzzy_matches=allow_fuzzy_matches,
478 )
479 if main_binary_package is None: 479 ↛ 480line 479 didn't jump to line 480 because the condition on line 479 was never true
480 return
481 provided_files_by_key: dict[tuple[str | None, ...], PackagerProvidedFile] = {}
482 max_periods_in_package_name = max(name.count(".") for name in binary_packages)
483 if detect_typos and CAN_DETECT_TYPOS:
484 period2stems = _period_stem(packager_provided_files.keys())
485 else:
486 period2stems = {}
488 paths = []
490 for entry in debian_dir.iterdir():
491 if entry.is_dir or entry.name.startswith("."):
492 continue
493 if entry.path in ignore_paths:
494 continue
495 paths.append(entry)
496 matching_ppfs = _split_path(
497 packager_provided_files,
498 binary_packages,
499 main_binary_package,
500 max_periods_in_package_name,
501 entry,
502 allow_fuzzy_matches=allow_fuzzy_matches,
503 period2stems=period2stems,
504 known_static_non_ppf_names=known_static_non_ppf_names,
505 match_arch=match_arch,
506 match_arch_os=match_arch_os,
507 )
508 for packager_provided_file in matching_ppfs:
509 match_key = (
510 packager_provided_file.package_name,
511 packager_provided_file.definition.stem,
512 packager_provided_file.provided_key,
513 )
514 if multi_match:
515 match_key = (
516 *match_key,
517 packager_provided_file.architecture_restriction,
518 )
519 existing = provided_files_by_key.get(match_key)
520 if (
521 existing is not None
522 and existing.match_priority > packager_provided_file.match_priority
523 ):
524 continue
525 provided_files_by_key[match_key] = packager_provided_file
527 paths2ppfs_per_package = dict[str, dict[str, list[PackagerProvidedFile]]]()
528 for packager_provided_file in provided_files_by_key.values():
529 package_name = packager_provided_file.package_name
530 path_name = packager_provided_file.path.path
531 ppfs_per_package = paths2ppfs_per_package.get(path_name)
532 if ppfs_per_package is None:
533 ppfs_per_package = collections.defaultdict(list)
534 paths2ppfs_per_package[path_name] = ppfs_per_package
535 ppfs_per_package[package_name].append(packager_provided_file)
537 for entry in paths:
538 yield PackagingFileClassification(
539 entry,
540 paths2ppfs_per_package.get(entry.path),
541 )
544def detect_all_packager_provided_files(
545 plugin_feature_set: "PluginProvidedFeatureSet",
546 debian_dir: VirtualPath,
547 binary_packages: Mapping[str, BinaryPackage],
548 *,
549 allow_fuzzy_matches: bool = False,
550 detect_typos: bool = False,
551 ignore_paths: Container[str] = frozenset(),
552 match_arch: frozenset[str] | set[str] | None = None,
553 match_arch_os: frozenset[str] | set[str] | None = None,
554 multi_match: bool = False,
555) -> dict[str, PerPackagePackagerProvidedResult]:
556 result = {
557 n: PerPackagePackagerProvidedResult([], collections.defaultdict(list))
558 for n in binary_packages
559 }
560 for classified_path in classify_debian_packaging_files(
561 plugin_feature_set,
562 debian_dir,
563 binary_packages,
564 allow_fuzzy_matches=allow_fuzzy_matches,
565 detect_typos=detect_typos,
566 ignore_paths=ignore_paths,
567 match_arch=match_arch,
568 match_arch_os=match_arch_os,
569 multi_match=multi_match,
570 ):
571 provided_files = classified_path.packager_provided_files_per_package
572 if not provided_files:
573 continue
574 for package_name, provided_file_data in provided_files.items():
575 per_package_result = result[package_name]
576 per_package_result.auto_installable.extend(
577 x for x in provided_file_data if not x.definition.reservation_only
578 )
579 reservation_only = per_package_result.reserved_only
580 for packager_provided_file in provided_file_data:
581 if not packager_provided_file.definition.reservation_only:
582 continue
583 reservation_only[packager_provided_file.definition.stem].append(
584 packager_provided_file
585 )
587 return result