Coverage for src/debputy/highlevel_manifest_parser.py: 73%
315 statements
« prev ^ index » next coverage.py v7.8.2, created at 2026-07-22 10:58 +0000
1import collections.abc
2import contextlib
3from collections.abc import Callable, Mapping, Iterator
4from typing import (
5 Any,
6 IO,
7 cast,
8 TYPE_CHECKING,
9)
11from debian.debian_support import DpkgArchTable
13import debputy.util
14from debputy.highlevel_manifest import (
15 HighLevelManifest,
16 PackageTransformationDefinition,
17 MutableYAMLManifest,
18)
19from debputy.maintscript_snippet import (
20 UnboundMaintscriptSnippet,
21 DPKG_DEB_CONTROL_SCRIPTS,
22 SnippetResolver,
23 PackageMaintscriptSnippetContainer,
24 SUPPORTED_UDEB_SCRIPTS,
25)
26from debputy.packages import BinaryPackage, SourcePackage
27from debputy.path_matcher import (
28 MatchRuleType,
29 ExactFileSystemPath,
30 MatchRule,
31)
32from debputy.plugin.api.parser_tables import OPARSER_MANIFEST_ROOT
33from debputy.plugins.debputy.build_system_rules import BuildRule
34from debputy.substitution import Substitution
35from debputy.util import (
36 _normalize_path,
37 escape_shell,
38 assume_not_none,
39)
40from debputy.util import _warn, _info
41from ._deb_options_profiles import DebBuildOptionsAndProfiles
42from ._manifest_constants import (
43 MK_CONFFILE_MANAGEMENT,
44 MK_INSTALLATIONS,
45 MK_PACKAGES,
46 MK_INSTALLATION_SEARCH_DIRS,
47 MK_TRANSFORMATIONS,
48 MK_BINARY_VERSION,
49 MK_SERVICES,
50 MK_MANIFEST_REMOVE_DURING_CLEAN,
51)
52from .architecture_support import DpkgArchitectureBuildProcessValuesTable
53from .filesystem_scan import OSFSROOverlay
54from .installations import InstallRule, PPFInstallRule
55from .manifest_parser.base_types import (
56 BuildEnvironments,
57 BuildEnvironmentDefinition,
58 FileSystemMatchRule,
59)
60from .manifest_parser.exceptions import ManifestParseException
61from .manifest_parser.parser_data import ParserContextData
62from .manifest_parser.util import AttributePath
63from .packager_provided_files import detect_all_packager_provided_files
64from .plugin.api import VirtualPath
65from .plugin.api.feature_set import PluginProvidedFeatureSet
66from .plugin.api.impl_types import (
67 TP,
68 TTP,
69 DispatchingTableParser,
70 PackageContextData,
71)
72from .plugin.api.spec import DebputyIntegrationMode
73from .plugin.plugin_state import with_binary_pkg_parsing_context, begin_parsing_context
74from .yaml import YAMLError, MANIFEST_YAML
77def _detect_possible_typo(
78 d: collections.abc.Iterable[str],
79 key: str,
80 attribute_parent_path: AttributePath,
81 required: bool,
82) -> None:
83 if debputy.util.CAN_DETECT_TYPOS:
84 k_len = len(key)
85 for actual_key in d:
86 if abs(k_len - len(actual_key)) > 2:
87 continue
88 if debputy.util.distance(key, actual_key) > 2:
89 continue
90 path = attribute_parent_path.path
91 ref = f'at "{path}"' if path else "at the manifest root level"
92 _warn(
93 f'Possible typo: The key "{actual_key}" should probably have been "{key}" {ref}'
94 )
95 elif required:
96 _info(
97 "Install python3-levenshtein to have debputy try to detect typos in the manifest."
98 )
101def _per_package_subst_variables(
102 p: BinaryPackage,
103 *,
104 name: str | None = None,
105) -> dict[str, str]:
106 return {
107 "PACKAGE": name if name is not None else p.name,
108 }
111class HighLevelManifestParser(ParserContextData):
112 def __init__(
113 self,
114 manifest_path: str,
115 source_package: SourcePackage,
116 binary_packages: Mapping[str, BinaryPackage],
117 substitution: Substitution,
118 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
119 dpkg_arch_query_table: DpkgArchTable,
120 build_env: DebBuildOptionsAndProfiles,
121 plugin_provided_feature_set: PluginProvidedFeatureSet,
122 debputy_integration_mode: DebputyIntegrationMode,
123 *,
124 # Available for testing purposes only
125 debian_dir: str | VirtualPath = "./debian",
126 ):
127 self.manifest_path = manifest_path
128 self._source_package = source_package
129 self._binary_packages = binary_packages
130 self._mutable_yaml_manifest: MutableYAMLManifest | None = None
131 # In source context, some variables are known to be unresolvable. Record this, so
132 # we can give better error messages.
133 self._substitution = substitution
134 self._dpkg_architecture_variables = dpkg_architecture_variables
135 self._dpkg_arch_query_table = dpkg_arch_query_table
136 self._deb_options_and_profiles = build_env
137 self._package_state_stack: list[PackageTransformationDefinition] = []
138 self._plugin_provided_feature_set = plugin_provided_feature_set
139 self._debputy_integration_mode = debputy_integration_mode
140 self._declared_variables = dict[str, AttributePath]()
141 self._used_named_envs = set[str]()
142 self._build_environments: BuildEnvironments | None = BuildEnvironments(
143 {},
144 None,
145 )
146 self._has_set_default_build_environment = False
147 self._read_build_environment = False
148 self._build_rules: list[BuildRule] | None = None
149 self._value_table: dict[
150 tuple[SourcePackage | BinaryPackage, type[Any]],
151 Any,
152 ] = {}
154 if isinstance(debian_dir, str): 154 ↛ 155line 154 didn't jump to line 155 because the condition on line 154 was never true
155 debian_dir = OSFSROOverlay.create_root_dir("debian", debian_dir)
157 self._debian_dir = debian_dir
159 # Delayed initialized; we rely on this delay to parse the variables.
160 self._all_package_states: dict[str, PackageTransformationDefinition] | None = (
161 None
162 )
164 self._install_rules: list[InstallRule] | None = None
165 self._remove_during_clean_rules: list[FileSystemMatchRule] = []
166 self._ownership_caches_loaded = False
167 self._used = False
169 def _ensure_package_states_is_initialized(self) -> None:
170 if self._all_package_states is not None:
171 return
172 substitution = self._substitution
173 binary_packages = self._binary_packages
175 self._all_package_states = {
176 n: PackageTransformationDefinition(
177 binary_package=p,
178 substitution=substitution.with_extra_substitutions(
179 **_per_package_subst_variables(p)
180 ),
181 is_auto_generated_package=False,
182 maintscript_snippets=PackageMaintscriptSnippetContainer(
183 SUPPORTED_UDEB_SCRIPTS if p.is_udeb else DPKG_DEB_CONTROL_SCRIPTS
184 ),
185 )
186 for n, p in binary_packages.items()
187 }
188 for n, p in binary_packages.items():
189 dbgsym_name = f"{n}-dbgsym"
190 if dbgsym_name in self._all_package_states: 190 ↛ 191line 190 didn't jump to line 191 because the condition on line 190 was never true
191 continue
192 self._all_package_states[dbgsym_name] = PackageTransformationDefinition(
193 binary_package=p,
194 substitution=substitution.with_extra_substitutions(
195 **_per_package_subst_variables(p, name=dbgsym_name)
196 ),
197 is_auto_generated_package=True,
198 maintscript_snippets=PackageMaintscriptSnippetContainer(frozenset()),
199 )
201 @property
202 def source_package(self) -> SourcePackage:
203 return self._source_package
205 @property
206 def binary_packages(self) -> Mapping[str, BinaryPackage]:
207 return self._binary_packages
209 @property
210 def _package_states(self) -> Mapping[str, PackageTransformationDefinition]:
211 assert self._all_package_states is not None
212 return self._all_package_states
214 @property
215 def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable:
216 return self._dpkg_architecture_variables
218 @property
219 def dpkg_arch_query_table(self) -> DpkgArchTable:
220 return self._dpkg_arch_query_table
222 @property
223 def deb_options_and_profiles(self) -> DebBuildOptionsAndProfiles:
224 return self._deb_options_and_profiles
226 def _self_check(self) -> None:
227 unused_envs = (
228 self._build_environments.environments.keys() - self._used_named_envs
229 )
230 if unused_envs:
231 unused_env_names = ", ".join(unused_envs)
232 raise ManifestParseException(
233 f"The following named environments were never referenced: {unused_env_names}"
234 )
236 def build_manifest(self) -> HighLevelManifest:
237 self._self_check()
238 if self._used: 238 ↛ 239line 238 didn't jump to line 239 because the condition on line 238 was never true
239 raise TypeError("build_manifest can only be called once!")
240 self._used = True
241 return begin_parsing_context(
242 self._value_table,
243 self._source_package,
244 self._build_manifest,
245 )
247 def _build_manifest(self) -> HighLevelManifest:
248 self._ensure_package_states_is_initialized()
249 for var, attribute_path in self._declared_variables.items():
250 if not self.substitution.is_used(var):
251 raise ManifestParseException(
252 f'The variable "{var}" is unused. Either use it or remove it.'
253 f" The variable was declared at {attribute_path.path_key_lc}."
254 )
255 if isinstance(self, YAMLManifestParser) and self._mutable_yaml_manifest is None:
256 self._mutable_yaml_manifest = MutableYAMLManifest.empty_manifest()
257 all_packager_provided_files = detect_all_packager_provided_files(
258 self._plugin_provided_feature_set,
259 self._debian_dir,
260 self.binary_packages,
261 )
263 for package in self._package_states:
264 with self.binary_package_context(package) as context:
265 if not context.is_auto_generated_package:
266 ppf_result = all_packager_provided_files[package]
267 if ppf_result.auto_installable: 267 ↛ 268line 267 didn't jump to line 268 because the condition on line 267 was never true
268 context.install_rules.append(
269 PPFInstallRule(
270 context.binary_package,
271 context.substitution,
272 ppf_result.auto_installable,
273 )
274 )
275 context.reserved_packager_provided_files.update(
276 ppf_result.reserved_only
277 )
278 self._transform_dpkg_maintscript_helpers_to_snippets()
279 build_environments = self.build_environments()
280 assert build_environments is not None
282 return HighLevelManifest(
283 self.manifest_path,
284 self._mutable_yaml_manifest,
285 self._remove_during_clean_rules,
286 self._install_rules,
287 self._source_package,
288 self.binary_packages,
289 self.substitution,
290 self._package_states,
291 self._dpkg_architecture_variables,
292 self._dpkg_arch_query_table,
293 self._deb_options_and_profiles,
294 build_environments,
295 self._build_rules,
296 self._value_table,
297 self._plugin_provided_feature_set,
298 self._debian_dir,
299 )
301 @contextlib.contextmanager
302 def binary_package_context(
303 self,
304 package_name: str,
305 ) -> Iterator[PackageTransformationDefinition]:
306 if package_name not in self._package_states:
307 self._error(
308 f'The package "{package_name}" is not present in the debian/control file (could not find'
309 f' "Package: {package_name}" in a binary stanza) nor is it a -dbgsym package for one'
310 " for a package in debian/control."
311 )
312 package_state = self._package_states[package_name]
313 self._package_state_stack.append(package_state)
314 ps_len = len(self._package_state_stack)
315 with with_binary_pkg_parsing_context(package_state.binary_package):
316 yield package_state
317 if ps_len != len(self._package_state_stack): 317 ↛ 318line 317 didn't jump to line 318 because the condition on line 317 was never true
318 raise RuntimeError("Internal error: Unbalanced stack manipulation detected")
319 self._package_state_stack.pop()
321 def dispatch_parser_table_for(self, rule_type: TTP) -> DispatchingTableParser[TP]:
322 t = self._plugin_provided_feature_set.manifest_parser_generator.dispatch_parser_table_for(
323 rule_type
324 )
325 if t is None:
326 raise AssertionError(
327 f"Internal error: No dispatching parser for {rule_type.__name__}"
328 )
329 return t
331 @property
332 def substitution(self) -> Substitution:
333 if self._package_state_stack:
334 return self._package_state_stack[-1].substitution
335 return self._substitution
337 def add_extra_substitution_variables(
338 self,
339 **extra_substitutions: tuple[str, AttributePath],
340 ) -> Substitution:
341 if self._package_state_stack or self._all_package_states is not None: 341 ↛ 346line 341 didn't jump to line 346 because the condition on line 341 was never true
342 # For one, it would not "bubble up" correctly when added to the lowest stack.
343 # And if it is not added to the lowest stack, then you get errors about it being
344 # unknown as soon as you leave the stack (which is weird for the user when
345 # the variable is something known, sometimes not)
346 raise RuntimeError("Cannot use add_extra_substitution from this state")
347 for key, (_, path) in extra_substitutions.items():
348 self._declared_variables[key] = path
349 self._substitution = self._substitution.with_extra_substitutions(
350 **{k: v[0] for k, v in extra_substitutions.items()}
351 )
352 return self._substitution
354 @property
355 def current_binary_package_state(self) -> PackageTransformationDefinition:
356 if not self._package_state_stack: 356 ↛ 357line 356 didn't jump to line 357 because the condition on line 356 was never true
357 raise RuntimeError("Invalid state: Not in a binary package context")
358 return self._package_state_stack[-1]
360 @property
361 def is_in_binary_package_state(self) -> bool:
362 return bool(self._package_state_stack)
364 @property
365 def debputy_integration_mode(self) -> DebputyIntegrationMode:
366 return self._debputy_integration_mode
368 @debputy_integration_mode.setter
369 def debputy_integration_mode(self, new_value: DebputyIntegrationMode) -> None:
370 self._debputy_integration_mode = new_value
372 def _register_build_environment(
373 self,
374 name: str | None,
375 build_environment: BuildEnvironmentDefinition,
376 attribute_path: AttributePath,
377 is_default: bool = False,
378 ) -> None:
379 assert not self._read_build_environment
381 # TODO: Reference the paths of the original environments for the error messages where that is relevant.
382 if is_default:
383 if self._has_set_default_build_environment: 383 ↛ 384line 383 didn't jump to line 384 because the condition on line 383 was never true
384 raise ManifestParseException(
385 f"There cannot be multiple default environments and"
386 f" therefore {attribute_path.path} cannot be a default environment"
387 )
388 self._has_set_default_build_environment = True
389 self._build_environments.default_environment = build_environment
390 if name is None: 390 ↛ 399line 390 didn't jump to line 399 because the condition on line 390 was always true
391 return
392 elif name is None: 392 ↛ 393line 392 didn't jump to line 393 because the condition on line 392 was never true
393 raise ManifestParseException(
394 f"Useless environment defined at {attribute_path.path}. It is neither the"
395 " default environment nor does it have a name (so no rules can reference it"
396 " explicitly)"
397 )
399 if name in self._build_environments.environments: 399 ↛ 400line 399 didn't jump to line 400 because the condition on line 399 was never true
400 raise ManifestParseException(
401 f'The environment defined at {attribute_path.path} reuse the name "{name}".'
402 " The environment name must be unique."
403 )
404 self._build_environments.environments[name] = build_environment
406 def resolve_build_environment(
407 self,
408 name: str | None,
409 attribute_path: AttributePath,
410 ) -> BuildEnvironmentDefinition:
411 if name is None:
412 return self.build_environments().default_environment
413 try:
414 env = self.build_environments().environments[name]
415 except KeyError:
416 raise ManifestParseException(
417 f'The environment "{name}" requested at {attribute_path.path} was not'
418 f" defined in the `build-environments`"
419 )
420 self._used_named_envs.add(name)
421 return env
423 def build_environments(self) -> BuildEnvironments:
424 v = self._build_environments
425 if (
426 not self._read_build_environment
427 and not self._build_environments.environments
428 and self._build_environments.default_environment is None
429 ):
430 self._build_environments.default_environment = BuildEnvironmentDefinition()
431 self._read_build_environment = True
432 return v
434 def _transform_dpkg_maintscript_helpers_to_snippets(self) -> None:
435 package_state = self.current_binary_package_state
436 for dmh in package_state.dpkg_maintscript_helper_snippets:
437 snippet = UnboundMaintscriptSnippet(
438 definition_source=dmh.definition_source,
439 snippet=SnippetResolver.snippet(
440 f'dpkg-maintscript-helper {escape_shell(*dmh.cmdline)} -- "$@"\n'
441 ),
442 )
443 for script in DPKG_DEB_CONTROL_SCRIPTS:
444 package_state.maintscript_snippets[script].append(snippet)
446 def normalize_path(
447 self,
448 path: str,
449 definition_source: AttributePath,
450 *,
451 allow_root_dir_match: bool = False,
452 ) -> ExactFileSystemPath:
453 try:
454 normalized = _normalize_path(path)
455 except ValueError:
456 self._error(
457 f'The path "{path}" provided in {definition_source.path} should be relative to the root of the'
458 ' package and not use any ".." or "." segments.'
459 )
460 if normalized == "." and not allow_root_dir_match:
461 self._error(
462 "Manifests must not change the root directory of the deb file. Please correct"
463 f' "{definition_source.path}" (path: "{path}) in {self.manifest_path}'
464 )
465 return ExactFileSystemPath(
466 self.substitution.substitute(normalized, definition_source.path)
467 )
469 def parse_path_or_glob(
470 self,
471 path_or_glob: str,
472 definition_source: AttributePath,
473 ) -> MatchRule:
474 match_rule = MatchRule.from_path_or_glob(
475 path_or_glob, definition_source.path, substitution=self.substitution
476 )
477 # NB: "." and "/" will be translated to MATCH_ANYTHING by MatchRule.from_path_or_glob,
478 # so there is no need to check for an exact match on "." like in normalize_path.
479 if match_rule.rule_type == MatchRuleType.MATCH_ANYTHING:
480 self._error(
481 f'The chosen match rule "{path_or_glob}" matches everything (including the deb root directory).'
482 f' Please correct "{definition_source.path}" (path: "{path_or_glob}) in {self.manifest_path} to'
483 f' something that matches "less" than everything.'
484 )
485 return match_rule
487 def parse_manifest(self) -> HighLevelManifest:
488 raise NotImplementedError
491class YAMLManifestParser(HighLevelManifestParser):
492 def _optional_key(
493 self,
494 d: Mapping[str, Any],
495 key: str,
496 attribute_parent_path: AttributePath,
497 expected_type=None,
498 default_value=None,
499 ):
500 v = d.get(key)
501 if v is None:
502 _detect_possible_typo(d, key, attribute_parent_path, False)
503 return default_value
504 if expected_type is not None:
505 return self._ensure_value_is_type(
506 v, expected_type, key, attribute_parent_path
507 )
508 return v
510 def _required_key(
511 self,
512 d: Mapping[str, Any],
513 key: str,
514 attribute_parent_path: AttributePath,
515 expected_type=None,
516 extra: str | Callable[[], str] | None = None,
517 ):
518 v = d.get(key)
519 if v is None:
520 _detect_possible_typo(d, key, attribute_parent_path, True)
521 if extra is not None:
522 msg = extra if isinstance(extra, str) else extra()
523 extra_info = " " + msg
524 else:
525 extra_info = ""
526 self._error(
527 f'Missing required key {key} at {attribute_parent_path.path} in manifest "{self.manifest_path}.'
528 f"{extra_info}"
529 )
531 if expected_type is not None:
532 return self._ensure_value_is_type(
533 v, expected_type, key, attribute_parent_path
534 )
535 return v
537 def _ensure_value_is_type(
538 self,
539 v,
540 t,
541 key: str | int | AttributePath,
542 attribute_parent_path: AttributePath | None,
543 ):
544 if v is None:
545 return None
546 if not isinstance(v, t):
547 if isinstance(t, tuple):
548 t_msg = "one of: " + ", ".join(x.__name__ for x in t)
549 else:
550 t_msg = f"a {t.__name__}"
551 key_path = (
552 key.path
553 if isinstance(key, AttributePath)
554 else assume_not_none(attribute_parent_path)[key].path
555 )
556 self._error(
557 f'The key {key_path} must be {t_msg} in manifest "{self.manifest_path}"'
558 )
559 return v
561 def _from_yaml_dict(self, yaml_data: object) -> "HighLevelManifest":
562 attribute_path = AttributePath.root_path(yaml_data)
563 parser_generator = self._plugin_provided_feature_set.manifest_parser_generator
564 dispatchable_object_parsers = parser_generator.dispatchable_object_parsers
565 manifest_root_parser = dispatchable_object_parsers[OPARSER_MANIFEST_ROOT]
566 parsed_data = manifest_root_parser.parse_input(
567 yaml_data,
568 attribute_path,
569 parser_context=self,
570 )
572 packages_dict: Mapping[str, PackageContextData[Mapping[str, Any]]] = cast(
573 "Mapping[str, PackageContextData[Mapping[str, Any]]]",
574 parsed_data.get("packages", {}),
575 )
576 self._remove_during_clean_rules = parsed_data.get(
577 MK_MANIFEST_REMOVE_DURING_CLEAN, []
578 )
579 install_rules = parsed_data.get(MK_INSTALLATIONS)
580 if install_rules:
581 self._install_rules = install_rules
582 packages_parent_path = attribute_path[MK_PACKAGES]
583 for package_name_raw, pcd in packages_dict.items():
584 definition_source = packages_parent_path[package_name_raw]
585 package_name = pcd.resolved_package_name
586 parsed = pcd.value
588 package_state: PackageTransformationDefinition
589 with self.binary_package_context(package_name) as package_state:
590 if package_state.is_auto_generated_package: 590 ↛ 592line 590 didn't jump to line 592 because the condition on line 590 was never true
591 # Maybe lift (part) of this restriction.
592 self._error(
593 f'Cannot define rules for package "{package_name}" (at {definition_source.path}). It is an'
594 " auto-generated package."
595 )
596 binary_version: str | None = parsed.get(MK_BINARY_VERSION)
597 if binary_version is not None:
598 package_state.binary_version = (
599 package_state.substitution.substitute(
600 binary_version,
601 definition_source[MK_BINARY_VERSION].path,
602 )
603 )
604 search_dirs = parsed.get(MK_INSTALLATION_SEARCH_DIRS)
605 if search_dirs is not None: 605 ↛ 606line 605 didn't jump to line 606 because the condition on line 605 was never true
606 package_state.search_dirs = search_dirs
607 transformations = parsed.get(MK_TRANSFORMATIONS)
608 conffile_management = parsed.get(MK_CONFFILE_MANAGEMENT)
609 service_rules = parsed.get(MK_SERVICES)
610 if transformations:
611 package_state.transformations.extend(transformations)
612 if conffile_management:
613 package_state.dpkg_maintscript_helper_snippets.extend(
614 conffile_management
615 )
616 if service_rules: 616 ↛ 617line 616 didn't jump to line 617 because the condition on line 616 was never true
617 package_state.requested_service_rules.extend(service_rules)
618 self._build_rules = parsed_data.get("builds")
620 return self.build_manifest()
622 def _parse_manifest(self, fd: IO[bytes] | str) -> HighLevelManifest:
623 try:
624 data = MANIFEST_YAML.load(fd)
625 except YAMLError as e:
626 msg = str(e)
627 lines = msg.splitlines(keepends=True)
628 i = -1
629 for i, line in enumerate(lines):
630 # Avoid an irrelevant "how do configure the YAML parser" message, which the
631 # user cannot use.
632 if line.startswith("To suppress this check"):
633 break
634 if i > -1 and len(lines) > i + 1:
635 lines = lines[:i]
636 msg = "".join(lines)
637 msg = msg.rstrip()
638 msg += (
639 f"\n\nYou can use `yamllint -d relaxed {escape_shell(self.manifest_path)}` to validate"
640 " the YAML syntax. The yamllint tool also supports style rules for YAML documents"
641 " (such as indentation rules) in case that is of interest."
642 )
643 raise ManifestParseException(
644 f"Could not parse {self.manifest_path} as a YAML document: {msg}"
645 ) from e
646 self._mutable_yaml_manifest = MutableYAMLManifest(data)
648 return begin_parsing_context(
649 self._value_table,
650 self._source_package,
651 self._from_yaml_dict,
652 data,
653 )
655 def parse_manifest(
656 self,
657 *,
658 fd: IO[bytes] | str | None = None,
659 ) -> HighLevelManifest:
660 if fd is None: 660 ↛ 661line 660 didn't jump to line 661 because the condition on line 660 was never true
661 with open(self.manifest_path, "rb") as fd:
662 return self._parse_manifest(fd)
663 else:
664 return self._parse_manifest(fd)