Coverage for src/debputy/analysis/debian_dir.py: 27%
308 statements
« prev ^ index » next coverage.py v7.6.0, created at 2025-01-27 13:59 +0000
1import json
2import os
3import stat
4import subprocess
5from typing import (
6 AbstractSet,
7 List,
8 Mapping,
9 Iterable,
10 Tuple,
11 Optional,
12 Sequence,
13 Dict,
14 Any,
15 Union,
16 Iterator,
17 TypedDict,
18 NotRequired,
19 Container,
20)
22from debputy.analysis import REFERENCE_DATA_TABLE
23from debputy.analysis.analysis_util import flatten_ppfs
24from debputy.dh.dh_assistant import (
25 resolve_active_and_inactive_dh_commands,
26 read_dh_addon_sequences,
27 extract_dh_compat_level,
28)
29from debputy.packager_provided_files import (
30 PackagerProvidedFile,
31 detect_all_packager_provided_files,
32)
33from debputy.packages import BinaryPackage
34from debputy.plugin.api import (
35 VirtualPath,
36 packager_provided_file_reference_documentation,
37)
38from debputy.plugin.api.feature_set import PluginProvidedFeatureSet
39from debputy.plugin.api.impl import plugin_metadata_for_debputys_own_plugin
40from debputy.plugin.api.impl_types import (
41 PluginProvidedKnownPackagingFile,
42 DebputyPluginMetadata,
43 KnownPackagingFileInfo,
44 InstallPatternDHCompatRule,
45 PackagerProvidedFileClassSpec,
46 expand_known_packaging_config_features,
47)
48from debputy.util import assume_not_none, escape_shell
50PackagingFileInfo = TypedDict(
51 "PackagingFileInfo",
52 {
53 "path": str,
54 "binary-package": NotRequired[str],
55 "install-path": NotRequired[str],
56 "install-pattern": NotRequired[str],
57 "file-categories": NotRequired[List[str]],
58 "config-features": NotRequired[List[str]],
59 "pkgfile-is-active-in-build": NotRequired[bool],
60 "pkgfile-stem": NotRequired[str],
61 "pkgfile-explicit-package-name": NotRequired[bool],
62 "pkgfile-name-segment": NotRequired[str],
63 "pkgfile-architecture-restriction": NotRequired[str],
64 "likely-typo-of": NotRequired[str],
65 "likely-generated-from": NotRequired[List[str]],
66 "related-tools": NotRequired[List[str]],
67 "documentation-uris": NotRequired[List[str]],
68 "debputy-cmd-templates": NotRequired[List[List[str]]],
69 "generates": NotRequired[str],
70 "generated-from": NotRequired[str],
71 },
72)
75def scan_debian_dir(
76 feature_set: PluginProvidedFeatureSet,
77 binary_packages: Mapping[str, BinaryPackage],
78 debian_dir: VirtualPath,
79 *,
80 uses_dh_sequencer: bool = True,
81 dh_sequences: Optional[AbstractSet[str]] = None,
82) -> Tuple[List[PackagingFileInfo], List[str], int, Optional[object]]:
83 known_packaging_files = feature_set.known_packaging_files
84 debputy_plugin_metadata = plugin_metadata_for_debputys_own_plugin()
86 reference_data_set_names = [
87 "config-features",
88 "file-categories",
89 ]
90 for n in reference_data_set_names:
91 assert n in REFERENCE_DATA_TABLE
93 annotated: List[PackagingFileInfo] = []
94 seen_paths: Dict[str, PackagingFileInfo] = {}
96 if dh_sequences is None:
97 r = read_dh_addon_sequences(debian_dir)
98 if r is not None:
99 bd_sequences, dr_sequences, uses_dh_sequencer = r
100 dh_sequences = bd_sequences | dr_sequences
101 else:
102 dh_sequences = set()
103 uses_dh_sequencer = False
104 is_debputy_package = (
105 "debputy" in dh_sequences
106 or "zz-debputy" in dh_sequences
107 or "zz_debputy" in dh_sequences
108 or "zz-debputy-rrr" in dh_sequences
109 )
110 dh_compat_level, dh_assistant_exit_code = extract_dh_compat_level()
111 dh_issues = []
113 static_packaging_files = {
114 kpf.detection_value: kpf
115 for kpf in known_packaging_files.values()
116 if kpf.detection_method == "path"
117 }
118 dh_pkgfile_docs = {
119 kpf.detection_value: kpf
120 for kpf in known_packaging_files.values()
121 if kpf.detection_method == "dh.pkgfile"
122 }
124 if is_debputy_package:
125 all_debputy_ppfs = list(
126 flatten_ppfs(
127 detect_all_packager_provided_files(
128 feature_set.packager_provided_files,
129 debian_dir,
130 binary_packages,
131 allow_fuzzy_matches=True,
132 detect_typos=True,
133 ignore_paths=static_packaging_files,
134 )
135 )
136 )
137 else:
138 all_debputy_ppfs = []
140 if dh_compat_level is not None:
141 (
142 all_dh_ppfs,
143 dh_issues,
144 dh_assistant_exit_code,
145 ) = resolve_debhelper_config_files(
146 debian_dir,
147 binary_packages,
148 debputy_plugin_metadata,
149 dh_pkgfile_docs,
150 dh_sequences,
151 dh_compat_level,
152 uses_dh_sequencer,
153 ignore_paths=static_packaging_files,
154 )
156 else:
157 all_dh_ppfs = []
159 for ppf in all_debputy_ppfs:
160 key = ppf.path.path
161 ref_doc = ppf.definition.reference_documentation
162 documentation_uris = (
163 ref_doc.format_documentation_uris if ref_doc is not None else None
164 )
165 details: PackagingFileInfo = {
166 "path": key,
167 "binary-package": ppf.package_name,
168 "pkgfile-stem": ppf.definition.stem,
169 "pkgfile-explicit-package-name": ppf.uses_explicit_package_name,
170 "pkgfile-is-active-in-build": ppf.definition.has_active_command,
171 "debputy-cmd-templates": [
172 ["debputy", "plugin", "show", "p-p-f", ppf.definition.stem]
173 ],
174 }
175 if ppf.fuzzy_match and key.endswith(".in"):
176 _merge_list(details, "file-categories", ["generic-template"])
177 details["generates"] = key[:-3]
178 elif assume_not_none(ppf.path.parent_dir).get(ppf.path.name + ".in"):
179 _merge_list(details, "file-categories", ["generated"])
180 details["generated-from"] = key + ".in"
181 name_segment = ppf.name_segment
182 arch_restriction = ppf.architecture_restriction
183 if name_segment is not None:
184 details["pkgfile-name-segment"] = name_segment
185 if arch_restriction:
186 details["pkgfile-architecture-restriction"] = arch_restriction
187 seen_paths[key] = details
188 annotated.append(details)
189 static_details = static_packaging_files.get(key)
190 if static_details is not None:
191 # debhelper compat rules does not apply to debputy files
192 _add_known_packaging_data(details, static_details, None)
193 if documentation_uris:
194 details["documentation-uris"] = list(documentation_uris)
196 _merge_ppfs(annotated, seen_paths, all_dh_ppfs, dh_pkgfile_docs, dh_compat_level)
198 for virtual_path in _scan_debian_dir(debian_dir):
199 key = virtual_path.path
200 if key in seen_paths:
201 continue
202 if virtual_path.is_symlink:
203 try:
204 st = os.stat(virtual_path.fs_path)
205 except FileNotFoundError:
206 continue
207 else:
208 if not stat.S_ISREG(st.st_mode):
209 continue
210 elif not virtual_path.is_file:
211 continue
213 static_match = static_packaging_files.get(virtual_path.path)
214 if static_match is not None:
215 details: PackagingFileInfo = {
216 "path": key,
217 }
218 annotated.append(details)
219 if assume_not_none(virtual_path.parent_dir).get(virtual_path.name + ".in"):
220 details["generated-from"] = key + ".in"
221 _merge_list(details, "file-categories", ["generated"])
222 _add_known_packaging_data(details, static_match, dh_compat_level)
224 return annotated, reference_data_set_names, dh_assistant_exit_code, dh_issues
227def _fake_PPFClassSpec(
228 debputy_plugin_metadata: DebputyPluginMetadata,
229 stem: str,
230 doc_uris: Optional[Sequence[str]],
231 install_pattern: Optional[str],
232 *,
233 default_priority: Optional[int] = None,
234 packageless_is_fallback_for_all_packages: bool = False,
235 post_formatting_rewrite: Optional[str] = None,
236 bug_950723: bool = False,
237 has_active_command: bool = False,
238) -> PackagerProvidedFileClassSpec:
239 if install_pattern is None: 239 ↛ 241line 239 didn't jump to line 241 because the condition on line 239 was always true
240 install_pattern = "not-a-real-ppf"
241 if post_formatting_rewrite is not None: 241 ↛ 242line 241 didn't jump to line 242 because the condition on line 241 was never true
242 formatting_hook = _POST_FORMATTING_REWRITE[post_formatting_rewrite]
243 else:
244 formatting_hook = None
245 return PackagerProvidedFileClassSpec(
246 debputy_plugin_metadata,
247 stem,
248 install_pattern,
249 allow_architecture_segment=True,
250 allow_name_segment=True,
251 default_priority=default_priority,
252 default_mode=0o644,
253 post_formatting_rewrite=formatting_hook,
254 packageless_is_fallback_for_all_packages=packageless_is_fallback_for_all_packages,
255 reservation_only=False,
256 formatting_callback=None,
257 bug_950723=bug_950723,
258 has_active_command=has_active_command,
259 reference_documentation=packager_provided_file_reference_documentation(
260 format_documentation_uris=doc_uris,
261 ),
262 )
265def _relevant_dh_compat_rules(
266 compat_level: Optional[int],
267 info: KnownPackagingFileInfo,
268) -> Iterable[InstallPatternDHCompatRule]:
269 if compat_level is None:
270 return
271 dh_compat_rules = info.get("dh_compat_rules")
272 if not dh_compat_rules:
273 return
274 for dh_compat_rule in dh_compat_rules:
275 rule_compat_level = dh_compat_rule.get("starting_with_compat_level")
276 if rule_compat_level is not None and compat_level < rule_compat_level:
277 continue
278 yield dh_compat_rule
281def _kpf_install_pattern(
282 compat_level: Optional[int],
283 ppkpf: PluginProvidedKnownPackagingFile,
284) -> Optional[str]:
285 for compat_rule in _relevant_dh_compat_rules(compat_level, ppkpf.info):
286 install_pattern = compat_rule.get("install_pattern")
287 if install_pattern is not None:
288 return install_pattern
289 return ppkpf.info.get("install_pattern")
292def resolve_debhelper_config_files(
293 debian_dir: VirtualPath,
294 binary_packages: Mapping[str, BinaryPackage],
295 debputy_plugin_metadata: DebputyPluginMetadata,
296 dh_ppf_docs: Dict[str, PluginProvidedKnownPackagingFile],
297 dh_rules_addons: AbstractSet[str],
298 dh_compat_level: int,
299 saw_dh: bool,
300 ignore_paths: Container[str] = frozenset(),
301) -> Tuple[List[PackagerProvidedFile], Optional[object], int]:
302 dh_ppfs = {}
303 commands, exit_code = _relevant_dh_commands(dh_rules_addons)
305 cmd = ["dh_assistant", "list-guessed-dh-config-files"]
306 if dh_rules_addons:
307 addons = ",".join(dh_rules_addons)
308 cmd.append(f"--with={addons}")
309 try:
310 output = subprocess.check_output(
311 cmd,
312 stderr=subprocess.DEVNULL,
313 )
314 except (subprocess.CalledProcessError, FileNotFoundError) as e:
315 config_files = []
316 issues = None
317 if isinstance(e, subprocess.CalledProcessError):
318 exit_code = e.returncode
319 else:
320 exit_code = 127
321 else:
322 result = json.loads(output)
323 config_files: List[Union[Mapping[str, Any], object]] = result.get(
324 "config-files", []
325 )
326 issues = result.get("issues")
327 dh_commands = resolve_active_and_inactive_dh_commands(dh_rules_addons)
328 for config_file in config_files:
329 if not isinstance(config_file, dict): 329 ↛ 330line 329 didn't jump to line 330 because the condition on line 329 was never true
330 continue
331 if config_file.get("file-type") != "pkgfile": 331 ↛ 332line 331 didn't jump to line 332 because the condition on line 331 was never true
332 continue
333 stem = config_file.get("pkgfile")
334 if stem is None: 334 ↛ 335line 334 didn't jump to line 335 because the condition on line 334 was never true
335 continue
336 internal = config_file.get("internal")
337 if isinstance(internal, dict):
338 bug_950723 = internal.get("bug#950723", False) is True
339 else:
340 bug_950723 = False
341 commands = config_file.get("commands")
342 documentation_uris = []
343 related_tools = []
344 seen_commands = set()
345 seen_docs = set()
346 ppkpf = dh_ppf_docs.get(stem)
348 if ppkpf: 348 ↛ 349line 348 didn't jump to line 349 because the condition on line 348 was never true
349 dh_cmds = ppkpf.info.get("debhelper_commands")
350 doc_uris = ppkpf.info.get("documentation_uris")
351 default_priority = ppkpf.info.get("default_priority")
352 if doc_uris is not None:
353 seen_docs.update(doc_uris)
354 documentation_uris.extend(doc_uris)
355 if dh_cmds is not None:
356 seen_commands.update(dh_cmds)
357 related_tools.extend(dh_cmds)
358 install_pattern = _kpf_install_pattern(dh_compat_level, ppkpf)
359 post_formatting_rewrite = ppkpf.info.get("post_formatting_rewrite")
360 packageless_is_fallback_for_all_packages = ppkpf.info.get(
361 "packageless_is_fallback_for_all_packages",
362 False,
363 )
364 # If it is a debhelper PPF, then `has_active_command` is false by default.
365 has_active_command = ppkpf.info.get("has_active_command", False)
366 else:
367 install_pattern = None
368 default_priority = None
369 post_formatting_rewrite = None
370 packageless_is_fallback_for_all_packages = False
371 has_active_command = False
372 for command in commands:
373 if isinstance(command, dict): 373 ↛ 372line 373 didn't jump to line 372 because the condition on line 373 was always true
374 command_name = command.get("command")
375 if isinstance(command_name, str) and command_name: 375 ↛ 384line 375 didn't jump to line 384 because the condition on line 375 was always true
376 if command_name not in seen_commands: 376 ↛ 379line 376 didn't jump to line 379 because the condition on line 376 was always true
377 related_tools.append(command_name)
378 seen_commands.add(command_name)
379 manpage = f"man:{command_name}(1)"
380 if manpage not in seen_docs: 380 ↛ 385line 380 didn't jump to line 385 because the condition on line 380 was always true
381 documentation_uris.append(manpage)
382 seen_docs.add(manpage)
383 else:
384 continue
385 is_active = command.get("is-active", True)
386 if is_active is None and command_name in dh_commands.active_commands: 386 ↛ 387line 386 didn't jump to line 387 because the condition on line 386 was never true
387 is_active = True
388 if not isinstance(is_active, bool): 388 ↛ 389line 388 didn't jump to line 389 because the condition on line 388 was never true
389 continue
390 if is_active:
391 has_active_command = True
392 dh_ppfs[stem] = _fake_PPFClassSpec(
393 debputy_plugin_metadata,
394 stem,
395 documentation_uris,
396 install_pattern,
397 default_priority=default_priority,
398 post_formatting_rewrite=post_formatting_rewrite,
399 packageless_is_fallback_for_all_packages=packageless_is_fallback_for_all_packages,
400 bug_950723=bug_950723,
401 has_active_command=has_active_command if saw_dh else True,
402 )
403 for ppkpf in dh_ppf_docs.values(): 403 ↛ 404line 403 didn't jump to line 404 because the loop on line 403 never started
404 stem = ppkpf.detection_value
405 if stem in dh_ppfs:
406 continue
408 default_priority = ppkpf.info.get("default_priority")
409 install_pattern = _kpf_install_pattern(dh_compat_level, ppkpf)
410 post_formatting_rewrite = ppkpf.info.get("post_formatting_rewrite")
411 packageless_is_fallback_for_all_packages = ppkpf.info.get(
412 "packageless_is_fallback_for_all_packages",
413 False,
414 )
415 has_active_command = (
416 ppkpf.info.get("has_active_command", False) if saw_dh else False
417 )
418 if not has_active_command:
419 dh_cmds = ppkpf.info.get("debhelper_commands")
420 if dh_cmds:
421 has_active_command = any(
422 c in dh_commands.active_commands for c in dh_cmds
423 )
424 dh_ppfs[stem] = _fake_PPFClassSpec(
425 debputy_plugin_metadata,
426 stem,
427 ppkpf.info.get("documentation_uris"),
428 install_pattern,
429 default_priority=default_priority,
430 post_formatting_rewrite=post_formatting_rewrite,
431 packageless_is_fallback_for_all_packages=packageless_is_fallback_for_all_packages,
432 has_active_command=has_active_command,
433 )
434 all_dh_ppfs = list(
435 flatten_ppfs(
436 detect_all_packager_provided_files(
437 dh_ppfs,
438 debian_dir,
439 binary_packages,
440 allow_fuzzy_matches=True,
441 detect_typos=True,
442 ignore_paths=ignore_paths,
443 )
444 )
445 )
446 return all_dh_ppfs, issues, exit_code
449def _merge_list(
450 existing_table: Dict[str, Any],
451 key: str,
452 new_data: Optional[Sequence[str]],
453) -> None:
454 if not new_data:
455 return
456 existing_values = existing_table.get(key, [])
457 if isinstance(existing_values, tuple):
458 existing_values = list(existing_values)
459 assert isinstance(existing_values, list)
460 seen = set(existing_values)
461 existing_values.extend(x for x in new_data if x not in seen)
462 existing_table[key] = existing_values
465def _merge_ppfs(
466 identified: List[PackagingFileInfo],
467 seen_paths: Dict[str, PackagingFileInfo],
468 ppfs: List[PackagerProvidedFile],
469 context: Mapping[str, PluginProvidedKnownPackagingFile],
470 dh_compat_level: Optional[int],
471) -> None:
472 for ppf in ppfs:
473 key = ppf.path.path
474 ref_doc = ppf.definition.reference_documentation
475 documentation_uris = (
476 ref_doc.format_documentation_uris if ref_doc is not None else None
477 )
478 if not ppf.definition.installed_as_format.startswith("not-a-real-ppf"):
479 try:
480 parts = ppf.compute_dest()
481 except RuntimeError:
482 dest = None
483 else:
484 dest = "/".join(parts).lstrip(".")
485 else:
486 dest = None
487 orig_details = seen_paths.get(key)
488 if orig_details is None:
489 details: PackagingFileInfo = {
490 "path": key,
491 "pkgfile-stem": ppf.definition.stem,
492 "pkgfile-is-active-in-build": ppf.definition.has_active_command,
493 "pkgfile-explicit-package-name": ppf.uses_explicit_package_name,
494 "binary-package": ppf.package_name,
495 }
496 if ppf.expected_path is not None:
497 details["likely-typo-of"] = ppf.expected_path
498 identified.append(details)
499 else:
500 details = orig_details
501 # We do not merge the "is typo" field; if the original
502 for k, v in [
503 ("pkgfile-stem", ppf.definition.stem),
504 ("pkgfile-explicit-package-name", ppf.definition.has_active_command),
505 ("binary-package", ppf.package_name),
506 ]:
507 if k not in details:
508 details[k] = v
509 if ppf.definition.has_active_command and details.get(
510 "pkgfile-is-active-in-build", False
511 ):
512 details["pkgfile-is-active-in-build"] = True
513 if ppf.expected_path is None and "likely-typo-of" in details:
514 del details["likely-typo-of"]
516 name_segment = ppf.name_segment
517 arch_restriction = ppf.architecture_restriction
518 if name_segment is not None and "pkgfile-name-segment" not in details:
519 details["pkgfile-name-segment"] = name_segment
520 if (
521 arch_restriction is not None
522 and "pkgfile-architecture-restriction" not in details
523 ):
524 details["pkgfile-architecture-restriction"] = arch_restriction
525 if ppf.fuzzy_match and key.endswith(".in"):
526 _merge_list(details, "file-categories", ["generic-template"])
527 details["generates"] = key[:-3]
528 elif assume_not_none(ppf.path.parent_dir).get(ppf.path.name + ".in"):
529 _merge_list(details, "file-categories", ["generated"])
530 details["generated-from"] = key + ".in"
531 if dest is not None and "install-path" not in details:
532 details["install-path"] = dest
534 extra_details = context.get(ppf.definition.stem)
535 if extra_details is not None:
536 _add_known_packaging_data(details, extra_details, dh_compat_level)
538 _merge_list(details, "documentation-uris", documentation_uris)
541def _relevant_dh_commands(dh_rules_addons: Iterable[str]) -> Tuple[List[str], int]:
542 cmd = ["dh_assistant", "list-commands", "--output-format=json"]
543 if dh_rules_addons:
544 addons = ",".join(dh_rules_addons)
545 cmd.append(f"--with={addons}")
546 try:
547 output = subprocess.check_output(
548 cmd,
549 stderr=subprocess.DEVNULL,
550 )
551 except (FileNotFoundError, subprocess.CalledProcessError) as e:
552 exit_code = 127
553 if isinstance(e, subprocess.CalledProcessError):
554 exit_code = e.returncode
555 return [], exit_code
556 else:
557 data = json.loads(output)
558 commands_json = data.get("commands")
559 commands = []
560 for command in commands_json:
561 if isinstance(command, dict): 561 ↛ 560line 561 didn't jump to line 560 because the condition on line 561 was always true
562 command_name = command.get("command")
563 if isinstance(command_name, str) and command_name: 563 ↛ 560line 563 didn't jump to line 560 because the condition on line 563 was always true
564 commands.append(command_name)
565 return commands, 0
568def _add_known_packaging_data(
569 details: PackagingFileInfo,
570 plugin_data: PluginProvidedKnownPackagingFile,
571 dh_compat_level: Optional[int],
572):
573 install_pattern = _kpf_install_pattern(
574 dh_compat_level,
575 plugin_data,
576 )
577 config_features = plugin_data.info.get("config_features")
578 if config_features:
579 config_features = expand_known_packaging_config_features(
580 dh_compat_level or 0,
581 config_features,
582 )
583 _merge_list(details, "config-features", config_features)
585 if dh_compat_level is not None:
586 extra_config_features = []
587 for dh_compat_rule in _relevant_dh_compat_rules(
588 dh_compat_level, plugin_data.info
589 ):
590 cf = dh_compat_rule.get("add_config_features")
591 if cf:
592 extra_config_features.extend(cf)
593 if extra_config_features:
594 extra_config_features = expand_known_packaging_config_features(
595 dh_compat_level,
596 extra_config_features,
597 )
598 _merge_list(details, "config-features", extra_config_features)
599 if "install-pattern" not in details and install_pattern is not None:
600 details["install-pattern"] = install_pattern
601 for mk, ok in [
602 ("file_categories", "file-categories"),
603 ("documentation_uris", "documentation-uris"),
604 ("debputy_cmd_templates", "debputy-cmd-templates"),
605 ]:
606 value = plugin_data.info.get(mk)
607 if value and ok == "debputy-cmd-templates":
608 value = [escape_shell(*c) for c in value]
609 _merge_list(details, ok, value)
612def _scan_debian_dir(debian_dir: VirtualPath) -> Iterator[VirtualPath]:
613 for p in debian_dir.iterdir:
614 yield p
615 if p.is_dir and p.path in ("debian/source", "debian/tests"):
616 yield from p.iterdir
619_POST_FORMATTING_REWRITE = { 619 ↛ exitline 619 didn't jump to the function exit
620 "period-to-underscore": lambda n: n.replace(".", "_"),
621}