Coverage for src/debputy/highlevel_manifest_parser.py: 73%
315 statements
« prev ^ index » next coverage.py v7.8.2, created at 2026-06-14 08:43 +0000
1import collections.abc
2import contextlib
3from collections.abc import Callable, Mapping, Iterator
4from typing import (
5 Any,
6 IO,
7 cast,
8 TYPE_CHECKING,
9)
11from debian.debian_support import DpkgArchTable
13import debputy.util
14from debputy.highlevel_manifest import (
15 HighLevelManifest,
16 PackageTransformationDefinition,
17 MutableYAMLManifest,
18)
19from debputy.maintscript_snippet import (
20 MaintscriptSnippet,
21 STD_CONTROL_SCRIPTS,
22 MaintscriptSnippetContainer,
23 SnippetResolver,
24)
25from debputy.packages import BinaryPackage, SourcePackage
26from debputy.path_matcher import (
27 MatchRuleType,
28 ExactFileSystemPath,
29 MatchRule,
30)
31from debputy.plugin.api.parser_tables import OPARSER_MANIFEST_ROOT
32from debputy.plugins.debputy.build_system_rules import BuildRule
33from debputy.substitution import Substitution
34from debputy.util import (
35 _normalize_path,
36 escape_shell,
37 assume_not_none,
38)
39from debputy.util import _warn, _info
40from ._deb_options_profiles import DebBuildOptionsAndProfiles
41from ._manifest_constants import (
42 MK_CONFFILE_MANAGEMENT,
43 MK_INSTALLATIONS,
44 MK_PACKAGES,
45 MK_INSTALLATION_SEARCH_DIRS,
46 MK_TRANSFORMATIONS,
47 MK_BINARY_VERSION,
48 MK_SERVICES,
49 MK_MANIFEST_REMOVE_DURING_CLEAN,
50)
51from .architecture_support import DpkgArchitectureBuildProcessValuesTable
52from .filesystem_scan import OSFSROOverlay
53from .installations import InstallRule, PPFInstallRule
54from .manifest_parser.base_types import (
55 BuildEnvironments,
56 BuildEnvironmentDefinition,
57 FileSystemMatchRule,
58)
59from .manifest_parser.exceptions import ManifestParseException
60from .manifest_parser.parser_data import ParserContextData
61from .manifest_parser.util import AttributePath
62from .packager_provided_files import detect_all_packager_provided_files
63from .plugin.api import VirtualPath
64from .plugin.api.feature_set import PluginProvidedFeatureSet
65from .plugin.api.impl_types import (
66 TP,
67 TTP,
68 DispatchingTableParser,
69 PackageContextData,
70)
71from .plugin.api.spec import DebputyIntegrationMode
72from .plugin.plugin_state import with_binary_pkg_parsing_context, begin_parsing_context
73from .yaml import YAMLError, MANIFEST_YAML
76def _detect_possible_typo(
77 d: collections.abc.Iterable[str],
78 key: str,
79 attribute_parent_path: AttributePath,
80 required: bool,
81) -> None:
82 if debputy.util.CAN_DETECT_TYPOS:
83 k_len = len(key)
84 for actual_key in d:
85 if abs(k_len - len(actual_key)) > 2:
86 continue
87 if debputy.util.distance(key, actual_key) > 2:
88 continue
89 path = attribute_parent_path.path
90 ref = f'at "{path}"' if path else "at the manifest root level"
91 _warn(
92 f'Possible typo: The key "{actual_key}" should probably have been "{key}" {ref}'
93 )
94 elif required:
95 _info(
96 "Install python3-levenshtein to have debputy try to detect typos in the manifest."
97 )
100def _per_package_subst_variables(
101 p: BinaryPackage,
102 *,
103 name: str | None = None,
104) -> dict[str, str]:
105 return {
106 "PACKAGE": name if name is not None else p.name,
107 }
110class HighLevelManifestParser(ParserContextData):
111 def __init__(
112 self,
113 manifest_path: str,
114 source_package: SourcePackage,
115 binary_packages: Mapping[str, BinaryPackage],
116 substitution: Substitution,
117 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
118 dpkg_arch_query_table: DpkgArchTable,
119 build_env: DebBuildOptionsAndProfiles,
120 plugin_provided_feature_set: PluginProvidedFeatureSet,
121 debputy_integration_mode: DebputyIntegrationMode,
122 *,
123 # Available for testing purposes only
124 debian_dir: str | VirtualPath = "./debian",
125 ):
126 self.manifest_path = manifest_path
127 self._source_package = source_package
128 self._binary_packages = binary_packages
129 self._mutable_yaml_manifest: MutableYAMLManifest | None = None
130 # In source context, some variables are known to be unresolvable. Record this, so
131 # we can give better error messages.
132 self._substitution = substitution
133 self._dpkg_architecture_variables = dpkg_architecture_variables
134 self._dpkg_arch_query_table = dpkg_arch_query_table
135 self._deb_options_and_profiles = build_env
136 self._package_state_stack: list[PackageTransformationDefinition] = []
137 self._plugin_provided_feature_set = plugin_provided_feature_set
138 self._debputy_integration_mode = debputy_integration_mode
139 self._declared_variables = dict[str, AttributePath]()
140 self._used_named_envs = set[str]()
141 self._build_environments: BuildEnvironments | None = BuildEnvironments(
142 {},
143 None,
144 )
145 self._has_set_default_build_environment = False
146 self._read_build_environment = False
147 self._build_rules: list[BuildRule] | None = None
148 self._value_table: dict[
149 tuple[SourcePackage | BinaryPackage, type[Any]],
150 Any,
151 ] = {}
153 if isinstance(debian_dir, str): 153 ↛ 154line 153 didn't jump to line 154 because the condition on line 153 was never true
154 debian_dir = OSFSROOverlay.create_root_dir("debian", debian_dir)
156 self._debian_dir = debian_dir
158 # Delayed initialized; we rely on this delay to parse the variables.
159 self._all_package_states: dict[str, PackageTransformationDefinition] | None = (
160 None
161 )
163 self._install_rules: list[InstallRule] | None = None
164 self._remove_during_clean_rules: list[FileSystemMatchRule] = []
165 self._ownership_caches_loaded = False
166 self._used = False
168 def _ensure_package_states_is_initialized(self) -> None:
169 if self._all_package_states is not None:
170 return
171 substitution = self._substitution
172 binary_packages = self._binary_packages
174 self._all_package_states = {
175 n: PackageTransformationDefinition(
176 binary_package=p,
177 substitution=substitution.with_extra_substitutions(
178 **_per_package_subst_variables(p)
179 ),
180 is_auto_generated_package=False,
181 maintscript_snippets=collections.defaultdict(
182 MaintscriptSnippetContainer
183 ),
184 )
185 for n, p in binary_packages.items()
186 }
187 for n, p in binary_packages.items():
188 dbgsym_name = f"{n}-dbgsym"
189 if dbgsym_name in self._all_package_states: 189 ↛ 190line 189 didn't jump to line 190 because the condition on line 189 was never true
190 continue
191 self._all_package_states[dbgsym_name] = PackageTransformationDefinition(
192 binary_package=p,
193 substitution=substitution.with_extra_substitutions(
194 **_per_package_subst_variables(p, name=dbgsym_name)
195 ),
196 is_auto_generated_package=True,
197 maintscript_snippets=collections.defaultdict(
198 MaintscriptSnippetContainer
199 ),
200 )
202 @property
203 def source_package(self) -> SourcePackage:
204 return self._source_package
206 @property
207 def binary_packages(self) -> Mapping[str, BinaryPackage]:
208 return self._binary_packages
210 @property
211 def _package_states(self) -> Mapping[str, PackageTransformationDefinition]:
212 assert self._all_package_states is not None
213 return self._all_package_states
215 @property
216 def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable:
217 return self._dpkg_architecture_variables
219 @property
220 def dpkg_arch_query_table(self) -> DpkgArchTable:
221 return self._dpkg_arch_query_table
223 @property
224 def deb_options_and_profiles(self) -> DebBuildOptionsAndProfiles:
225 return self._deb_options_and_profiles
227 def _self_check(self) -> None:
228 unused_envs = (
229 self._build_environments.environments.keys() - self._used_named_envs
230 )
231 if unused_envs:
232 unused_env_names = ", ".join(unused_envs)
233 raise ManifestParseException(
234 f"The following named environments were never referenced: {unused_env_names}"
235 )
237 def build_manifest(self) -> HighLevelManifest:
238 self._self_check()
239 if self._used: 239 ↛ 240line 239 didn't jump to line 240 because the condition on line 239 was never true
240 raise TypeError("build_manifest can only be called once!")
241 self._used = True
242 return begin_parsing_context(
243 self._value_table,
244 self._source_package,
245 self._build_manifest,
246 )
248 def _build_manifest(self) -> HighLevelManifest:
249 self._ensure_package_states_is_initialized()
250 for var, attribute_path in self._declared_variables.items():
251 if not self.substitution.is_used(var):
252 raise ManifestParseException(
253 f'The variable "{var}" is unused. Either use it or remove it.'
254 f" The variable was declared at {attribute_path.path_key_lc}."
255 )
256 if isinstance(self, YAMLManifestParser) and self._mutable_yaml_manifest is None:
257 self._mutable_yaml_manifest = MutableYAMLManifest.empty_manifest()
258 all_packager_provided_files = detect_all_packager_provided_files(
259 self._plugin_provided_feature_set,
260 self._debian_dir,
261 self.binary_packages,
262 )
264 for package in self._package_states:
265 with self.binary_package_context(package) as context:
266 if not context.is_auto_generated_package:
267 ppf_result = all_packager_provided_files[package]
268 if ppf_result.auto_installable: 268 ↛ 269line 268 didn't jump to line 269 because the condition on line 268 was never true
269 context.install_rules.append(
270 PPFInstallRule(
271 context.binary_package,
272 context.substitution,
273 ppf_result.auto_installable,
274 )
275 )
276 context.reserved_packager_provided_files.update(
277 ppf_result.reserved_only
278 )
279 self._transform_dpkg_maintscript_helpers_to_snippets()
280 build_environments = self.build_environments()
281 assert build_environments is not None
283 return HighLevelManifest(
284 self.manifest_path,
285 self._mutable_yaml_manifest,
286 self._remove_during_clean_rules,
287 self._install_rules,
288 self._source_package,
289 self.binary_packages,
290 self.substitution,
291 self._package_states,
292 self._dpkg_architecture_variables,
293 self._dpkg_arch_query_table,
294 self._deb_options_and_profiles,
295 build_environments,
296 self._build_rules,
297 self._value_table,
298 self._plugin_provided_feature_set,
299 self._debian_dir,
300 )
302 @contextlib.contextmanager
303 def binary_package_context(
304 self,
305 package_name: str,
306 ) -> Iterator[PackageTransformationDefinition]:
307 if package_name not in self._package_states:
308 self._error(
309 f'The package "{package_name}" is not present in the debian/control file (could not find'
310 f' "Package: {package_name}" in a binary stanza) nor is it a -dbgsym package for one'
311 " for a package in debian/control."
312 )
313 package_state = self._package_states[package_name]
314 self._package_state_stack.append(package_state)
315 ps_len = len(self._package_state_stack)
316 with with_binary_pkg_parsing_context(package_state.binary_package):
317 yield package_state
318 if ps_len != len(self._package_state_stack): 318 ↛ 319line 318 didn't jump to line 319 because the condition on line 318 was never true
319 raise RuntimeError("Internal error: Unbalanced stack manipulation detected")
320 self._package_state_stack.pop()
322 def dispatch_parser_table_for(self, rule_type: TTP) -> DispatchingTableParser[TP]:
323 t = self._plugin_provided_feature_set.manifest_parser_generator.dispatch_parser_table_for(
324 rule_type
325 )
326 if t is None:
327 raise AssertionError(
328 f"Internal error: No dispatching parser for {rule_type.__name__}"
329 )
330 return t
332 @property
333 def substitution(self) -> Substitution:
334 if self._package_state_stack:
335 return self._package_state_stack[-1].substitution
336 return self._substitution
338 def add_extra_substitution_variables(
339 self,
340 **extra_substitutions: tuple[str, AttributePath],
341 ) -> Substitution:
342 if self._package_state_stack or self._all_package_states is not None: 342 ↛ 347line 342 didn't jump to line 347 because the condition on line 342 was never true
343 # For one, it would not "bubble up" correctly when added to the lowest stack.
344 # And if it is not added to the lowest stack, then you get errors about it being
345 # unknown as soon as you leave the stack (which is weird for the user when
346 # the variable is something known, sometimes not)
347 raise RuntimeError("Cannot use add_extra_substitution from this state")
348 for key, (_, path) in extra_substitutions.items():
349 self._declared_variables[key] = path
350 self._substitution = self._substitution.with_extra_substitutions(
351 **{k: v[0] for k, v in extra_substitutions.items()}
352 )
353 return self._substitution
355 @property
356 def current_binary_package_state(self) -> PackageTransformationDefinition:
357 if not self._package_state_stack: 357 ↛ 358line 357 didn't jump to line 358 because the condition on line 357 was never true
358 raise RuntimeError("Invalid state: Not in a binary package context")
359 return self._package_state_stack[-1]
361 @property
362 def is_in_binary_package_state(self) -> bool:
363 return bool(self._package_state_stack)
365 @property
366 def debputy_integration_mode(self) -> DebputyIntegrationMode:
367 return self._debputy_integration_mode
369 @debputy_integration_mode.setter
370 def debputy_integration_mode(self, new_value: DebputyIntegrationMode) -> None:
371 self._debputy_integration_mode = new_value
373 def _register_build_environment(
374 self,
375 name: str | None,
376 build_environment: BuildEnvironmentDefinition,
377 attribute_path: AttributePath,
378 is_default: bool = False,
379 ) -> None:
380 assert not self._read_build_environment
382 # TODO: Reference the paths of the original environments for the error messages where that is relevant.
383 if is_default:
384 if self._has_set_default_build_environment: 384 ↛ 385line 384 didn't jump to line 385 because the condition on line 384 was never true
385 raise ManifestParseException(
386 f"There cannot be multiple default environments and"
387 f" therefore {attribute_path.path} cannot be a default environment"
388 )
389 self._has_set_default_build_environment = True
390 self._build_environments.default_environment = build_environment
391 if name is None: 391 ↛ 400line 391 didn't jump to line 400 because the condition on line 391 was always true
392 return
393 elif name is None: 393 ↛ 394line 393 didn't jump to line 394 because the condition on line 393 was never true
394 raise ManifestParseException(
395 f"Useless environment defined at {attribute_path.path}. It is neither the"
396 " default environment nor does it have a name (so no rules can reference it"
397 " explicitly)"
398 )
400 if name in self._build_environments.environments: 400 ↛ 401line 400 didn't jump to line 401 because the condition on line 400 was never true
401 raise ManifestParseException(
402 f'The environment defined at {attribute_path.path} reuse the name "{name}".'
403 " The environment name must be unique."
404 )
405 self._build_environments.environments[name] = build_environment
407 def resolve_build_environment(
408 self,
409 name: str | None,
410 attribute_path: AttributePath,
411 ) -> BuildEnvironmentDefinition:
412 if name is None:
413 return self.build_environments().default_environment
414 try:
415 env = self.build_environments().environments[name]
416 except KeyError:
417 raise ManifestParseException(
418 f'The environment "{name}" requested at {attribute_path.path} was not'
419 f" defined in the `build-environments`"
420 )
421 self._used_named_envs.add(name)
422 return env
424 def build_environments(self) -> BuildEnvironments:
425 v = self._build_environments
426 if (
427 not self._read_build_environment
428 and not self._build_environments.environments
429 and self._build_environments.default_environment is None
430 ):
431 self._build_environments.default_environment = BuildEnvironmentDefinition()
432 self._read_build_environment = True
433 return v
435 def _transform_dpkg_maintscript_helpers_to_snippets(self) -> None:
436 package_state = self.current_binary_package_state
437 for dmh in package_state.dpkg_maintscript_helper_snippets:
438 snippet = MaintscriptSnippet(
439 definition_source=dmh.definition_source,
440 snippet=SnippetResolver.snippet_template(
441 f'dpkg-maintscript-helper {escape_shell(*dmh.cmdline)} -- "$@"\n'
442 ),
443 )
444 for script in STD_CONTROL_SCRIPTS:
445 package_state.maintscript_snippets[script].append(snippet)
447 def normalize_path(
448 self,
449 path: str,
450 definition_source: AttributePath,
451 *,
452 allow_root_dir_match: bool = False,
453 ) -> ExactFileSystemPath:
454 try:
455 normalized = _normalize_path(path)
456 except ValueError:
457 self._error(
458 f'The path "{path}" provided in {definition_source.path} should be relative to the root of the'
459 ' package and not use any ".." or "." segments.'
460 )
461 if normalized == "." and not allow_root_dir_match:
462 self._error(
463 "Manifests must not change the root directory of the deb file. Please correct"
464 f' "{definition_source.path}" (path: "{path}) in {self.manifest_path}'
465 )
466 return ExactFileSystemPath(
467 self.substitution.substitute(normalized, definition_source.path)
468 )
470 def parse_path_or_glob(
471 self,
472 path_or_glob: str,
473 definition_source: AttributePath,
474 ) -> MatchRule:
475 match_rule = MatchRule.from_path_or_glob(
476 path_or_glob, definition_source.path, substitution=self.substitution
477 )
478 # NB: "." and "/" will be translated to MATCH_ANYTHING by MatchRule.from_path_or_glob,
479 # so there is no need to check for an exact match on "." like in normalize_path.
480 if match_rule.rule_type == MatchRuleType.MATCH_ANYTHING:
481 self._error(
482 f'The chosen match rule "{path_or_glob}" matches everything (including the deb root directory).'
483 f' Please correct "{definition_source.path}" (path: "{path_or_glob}) in {self.manifest_path} to'
484 f' something that matches "less" than everything.'
485 )
486 return match_rule
488 def parse_manifest(self) -> HighLevelManifest:
489 raise NotImplementedError
492class YAMLManifestParser(HighLevelManifestParser):
493 def _optional_key(
494 self,
495 d: Mapping[str, Any],
496 key: str,
497 attribute_parent_path: AttributePath,
498 expected_type=None,
499 default_value=None,
500 ):
501 v = d.get(key)
502 if v is None:
503 _detect_possible_typo(d, key, attribute_parent_path, False)
504 return default_value
505 if expected_type is not None:
506 return self._ensure_value_is_type(
507 v, expected_type, key, attribute_parent_path
508 )
509 return v
511 def _required_key(
512 self,
513 d: Mapping[str, Any],
514 key: str,
515 attribute_parent_path: AttributePath,
516 expected_type=None,
517 extra: str | Callable[[], str] | None = None,
518 ):
519 v = d.get(key)
520 if v is None:
521 _detect_possible_typo(d, key, attribute_parent_path, True)
522 if extra is not None:
523 msg = extra if isinstance(extra, str) else extra()
524 extra_info = " " + msg
525 else:
526 extra_info = ""
527 self._error(
528 f'Missing required key {key} at {attribute_parent_path.path} in manifest "{self.manifest_path}.'
529 f"{extra_info}"
530 )
532 if expected_type is not None:
533 return self._ensure_value_is_type(
534 v, expected_type, key, attribute_parent_path
535 )
536 return v
538 def _ensure_value_is_type(
539 self,
540 v,
541 t,
542 key: str | int | AttributePath,
543 attribute_parent_path: AttributePath | None,
544 ):
545 if v is None:
546 return None
547 if not isinstance(v, t):
548 if isinstance(t, tuple):
549 t_msg = "one of: " + ", ".join(x.__name__ for x in t)
550 else:
551 t_msg = f"a {t.__name__}"
552 key_path = (
553 key.path
554 if isinstance(key, AttributePath)
555 else assume_not_none(attribute_parent_path)[key].path
556 )
557 self._error(
558 f'The key {key_path} must be {t_msg} in manifest "{self.manifest_path}"'
559 )
560 return v
562 def _from_yaml_dict(self, yaml_data: object) -> "HighLevelManifest":
563 attribute_path = AttributePath.root_path(yaml_data)
564 parser_generator = self._plugin_provided_feature_set.manifest_parser_generator
565 dispatchable_object_parsers = parser_generator.dispatchable_object_parsers
566 manifest_root_parser = dispatchable_object_parsers[OPARSER_MANIFEST_ROOT]
567 parsed_data = manifest_root_parser.parse_input(
568 yaml_data,
569 attribute_path,
570 parser_context=self,
571 )
573 packages_dict: Mapping[str, PackageContextData[Mapping[str, Any]]] = cast(
574 "Mapping[str, PackageContextData[Mapping[str, Any]]]",
575 parsed_data.get("packages", {}),
576 )
577 self._remove_during_clean_rules = parsed_data.get(
578 MK_MANIFEST_REMOVE_DURING_CLEAN, []
579 )
580 install_rules = parsed_data.get(MK_INSTALLATIONS)
581 if install_rules:
582 self._install_rules = install_rules
583 packages_parent_path = attribute_path[MK_PACKAGES]
584 for package_name_raw, pcd in packages_dict.items():
585 definition_source = packages_parent_path[package_name_raw]
586 package_name = pcd.resolved_package_name
587 parsed = pcd.value
589 package_state: PackageTransformationDefinition
590 with self.binary_package_context(package_name) as package_state:
591 if package_state.is_auto_generated_package: 591 ↛ 593line 591 didn't jump to line 593 because the condition on line 591 was never true
592 # Maybe lift (part) of this restriction.
593 self._error(
594 f'Cannot define rules for package "{package_name}" (at {definition_source.path}). It is an'
595 " auto-generated package."
596 )
597 binary_version: str | None = parsed.get(MK_BINARY_VERSION)
598 if binary_version is not None:
599 package_state.binary_version = (
600 package_state.substitution.substitute(
601 binary_version,
602 definition_source[MK_BINARY_VERSION].path,
603 )
604 )
605 search_dirs = parsed.get(MK_INSTALLATION_SEARCH_DIRS)
606 if search_dirs is not None: 606 ↛ 607line 606 didn't jump to line 607 because the condition on line 606 was never true
607 package_state.search_dirs = search_dirs
608 transformations = parsed.get(MK_TRANSFORMATIONS)
609 conffile_management = parsed.get(MK_CONFFILE_MANAGEMENT)
610 service_rules = parsed.get(MK_SERVICES)
611 if transformations:
612 package_state.transformations.extend(transformations)
613 if conffile_management:
614 package_state.dpkg_maintscript_helper_snippets.extend(
615 conffile_management
616 )
617 if service_rules: 617 ↛ 618line 617 didn't jump to line 618 because the condition on line 617 was never true
618 package_state.requested_service_rules.extend(service_rules)
619 self._build_rules = parsed_data.get("builds")
621 return self.build_manifest()
623 def _parse_manifest(self, fd: IO[bytes] | str) -> HighLevelManifest:
624 try:
625 data = MANIFEST_YAML.load(fd)
626 except YAMLError as e:
627 msg = str(e)
628 lines = msg.splitlines(keepends=True)
629 i = -1
630 for i, line in enumerate(lines):
631 # Avoid an irrelevant "how do configure the YAML parser" message, which the
632 # user cannot use.
633 if line.startswith("To suppress this check"):
634 break
635 if i > -1 and len(lines) > i + 1:
636 lines = lines[:i]
637 msg = "".join(lines)
638 msg = msg.rstrip()
639 msg += (
640 f"\n\nYou can use `yamllint -d relaxed {escape_shell(self.manifest_path)}` to validate"
641 " the YAML syntax. The yamllint tool also supports style rules for YAML documents"
642 " (such as indentation rules) in case that is of interest."
643 )
644 raise ManifestParseException(
645 f"Could not parse {self.manifest_path} as a YAML document: {msg}"
646 ) from e
647 self._mutable_yaml_manifest = MutableYAMLManifest(data)
649 return begin_parsing_context(
650 self._value_table,
651 self._source_package,
652 self._from_yaml_dict,
653 data,
654 )
656 def parse_manifest(
657 self,
658 *,
659 fd: IO[bytes] | str | None = None,
660 ) -> HighLevelManifest:
661 if fd is None: 661 ↛ 662line 661 didn't jump to line 662 because the condition on line 661 was never true
662 with open(self.manifest_path, "rb") as fd:
663 return self._parse_manifest(fd)
664 else:
665 return self._parse_manifest(fd)