Coverage for src/debputy/highlevel_manifest_parser.py: 73%
315 statements
« prev ^ index » next coverage.py v7.8.2, created at 2026-01-26 19:30 +0000
1import collections.abc
2import contextlib
3from collections.abc import Callable, Mapping, Iterator
4from typing import (
5 Any,
6 IO,
7 cast,
8 TYPE_CHECKING,
9)
11from debian.debian_support import DpkgArchTable
13import debputy.util
14from debputy.highlevel_manifest import (
15 HighLevelManifest,
16 PackageTransformationDefinition,
17 MutableYAMLManifest,
18)
19from debputy.maintscript_snippet import (
20 MaintscriptSnippet,
21 STD_CONTROL_SCRIPTS,
22 MaintscriptSnippetContainer,
23)
24from debputy.packages import BinaryPackage, SourcePackage
25from debputy.path_matcher import (
26 MatchRuleType,
27 ExactFileSystemPath,
28 MatchRule,
29)
30from debputy.plugin.api.parser_tables import OPARSER_MANIFEST_ROOT
31from debputy.plugins.debputy.build_system_rules import BuildRule
32from debputy.substitution import Substitution
33from debputy.util import (
34 _normalize_path,
35 escape_shell,
36 assume_not_none,
37)
38from debputy.util import _warn, _info
39from ._deb_options_profiles import DebBuildOptionsAndProfiles
40from ._manifest_constants import (
41 MK_CONFFILE_MANAGEMENT,
42 MK_INSTALLATIONS,
43 MK_PACKAGES,
44 MK_INSTALLATION_SEARCH_DIRS,
45 MK_TRANSFORMATIONS,
46 MK_BINARY_VERSION,
47 MK_SERVICES,
48 MK_MANIFEST_REMOVE_DURING_CLEAN,
49)
50from .architecture_support import DpkgArchitectureBuildProcessValuesTable
51from .filesystem_scan import OSFSROOverlay
52from .installations import InstallRule, PPFInstallRule
53from .manifest_parser.base_types import (
54 BuildEnvironments,
55 BuildEnvironmentDefinition,
56 FileSystemMatchRule,
57)
58from .manifest_parser.exceptions import ManifestParseException
59from .manifest_parser.parser_data import ParserContextData
60from .manifest_parser.util import AttributePath
61from .packager_provided_files import detect_all_packager_provided_files
62from .plugin.api import VirtualPath
63from .plugin.api.feature_set import PluginProvidedFeatureSet
64from .plugin.api.impl_types import (
65 TP,
66 TTP,
67 DispatchingTableParser,
68 PackageContextData,
69)
70from .plugin.api.spec import DebputyIntegrationMode
71from .plugin.plugin_state import with_binary_pkg_parsing_context, begin_parsing_context
72from .yaml import YAMLError, MANIFEST_YAML
75def _detect_possible_typo(
76 d: collections.abc.Iterable[str],
77 key: str,
78 attribute_parent_path: AttributePath,
79 required: bool,
80) -> None:
81 if debputy.util.CAN_DETECT_TYPOS:
82 k_len = len(key)
83 for actual_key in d:
84 if abs(k_len - len(actual_key)) > 2:
85 continue
86 if debputy.util.distance(key, actual_key) > 2:
87 continue
88 path = attribute_parent_path.path
89 ref = f'at "{path}"' if path else "at the manifest root level"
90 _warn(
91 f'Possible typo: The key "{actual_key}" should probably have been "{key}" {ref}'
92 )
93 elif required:
94 _info(
95 "Install python3-levenshtein to have debputy try to detect typos in the manifest."
96 )
99def _per_package_subst_variables(
100 p: BinaryPackage,
101 *,
102 name: str | None = None,
103) -> dict[str, str]:
104 return {
105 "PACKAGE": name if name is not None else p.name,
106 }
109class HighLevelManifestParser(ParserContextData):
110 def __init__(
111 self,
112 manifest_path: str,
113 source_package: SourcePackage,
114 binary_packages: Mapping[str, BinaryPackage],
115 substitution: Substitution,
116 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
117 dpkg_arch_query_table: DpkgArchTable,
118 build_env: DebBuildOptionsAndProfiles,
119 plugin_provided_feature_set: PluginProvidedFeatureSet,
120 debputy_integration_mode: DebputyIntegrationMode,
121 *,
122 # Available for testing purposes only
123 debian_dir: str | VirtualPath = "./debian",
124 ):
125 self.manifest_path = manifest_path
126 self._source_package = source_package
127 self._binary_packages = binary_packages
128 self._mutable_yaml_manifest: MutableYAMLManifest | None = None
129 # In source context, some variables are known to be unresolvable. Record this, so
130 # we can give better error messages.
131 self._substitution = substitution
132 self._dpkg_architecture_variables = dpkg_architecture_variables
133 self._dpkg_arch_query_table = dpkg_arch_query_table
134 self._deb_options_and_profiles = build_env
135 self._package_state_stack: list[PackageTransformationDefinition] = []
136 self._plugin_provided_feature_set = plugin_provided_feature_set
137 self._debputy_integration_mode = debputy_integration_mode
138 self._declared_variables = dict[str, AttributePath]()
139 self._used_named_envs = set[str]()
140 self._build_environments: BuildEnvironments | None = BuildEnvironments(
141 {},
142 None,
143 )
144 self._has_set_default_build_environment = False
145 self._read_build_environment = False
146 self._build_rules: list[BuildRule] | None = None
147 self._value_table: dict[
148 tuple[SourcePackage | BinaryPackage, type[Any]],
149 Any,
150 ] = {}
152 if isinstance(debian_dir, str): 152 ↛ 153line 152 didn't jump to line 153 because the condition on line 152 was never true
153 debian_dir = OSFSROOverlay.create_root_dir("debian", debian_dir)
155 self._debian_dir = debian_dir
157 # Delayed initialized; we rely on this delay to parse the variables.
158 self._all_package_states: dict[str, PackageTransformationDefinition] | None = (
159 None
160 )
162 self._install_rules: list[InstallRule] | None = None
163 self._remove_during_clean_rules: list[FileSystemMatchRule] = []
164 self._ownership_caches_loaded = False
165 self._used = False
167 def _ensure_package_states_is_initialized(self) -> None:
168 if self._all_package_states is not None:
169 return
170 substitution = self._substitution
171 binary_packages = self._binary_packages
173 self._all_package_states = {
174 n: PackageTransformationDefinition(
175 binary_package=p,
176 substitution=substitution.with_extra_substitutions(
177 **_per_package_subst_variables(p)
178 ),
179 is_auto_generated_package=False,
180 maintscript_snippets=collections.defaultdict(
181 MaintscriptSnippetContainer
182 ),
183 )
184 for n, p in binary_packages.items()
185 }
186 for n, p in binary_packages.items():
187 dbgsym_name = f"{n}-dbgsym"
188 if dbgsym_name in self._all_package_states: 188 ↛ 189line 188 didn't jump to line 189 because the condition on line 188 was never true
189 continue
190 self._all_package_states[dbgsym_name] = PackageTransformationDefinition(
191 binary_package=p,
192 substitution=substitution.with_extra_substitutions(
193 **_per_package_subst_variables(p, name=dbgsym_name)
194 ),
195 is_auto_generated_package=True,
196 maintscript_snippets=collections.defaultdict(
197 MaintscriptSnippetContainer
198 ),
199 )
201 @property
202 def source_package(self) -> SourcePackage:
203 return self._source_package
205 @property
206 def binary_packages(self) -> Mapping[str, BinaryPackage]:
207 return self._binary_packages
209 @property
210 def _package_states(self) -> Mapping[str, PackageTransformationDefinition]:
211 assert self._all_package_states is not None
212 return self._all_package_states
214 @property
215 def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable:
216 return self._dpkg_architecture_variables
218 @property
219 def dpkg_arch_query_table(self) -> DpkgArchTable:
220 return self._dpkg_arch_query_table
222 @property
223 def deb_options_and_profiles(self) -> DebBuildOptionsAndProfiles:
224 return self._deb_options_and_profiles
226 def _self_check(self) -> None:
227 unused_envs = (
228 self._build_environments.environments.keys() - self._used_named_envs
229 )
230 if unused_envs:
231 unused_env_names = ", ".join(unused_envs)
232 raise ManifestParseException(
233 f"The following named environments were never referenced: {unused_env_names}"
234 )
236 def build_manifest(self) -> HighLevelManifest:
237 self._self_check()
238 if self._used: 238 ↛ 239line 238 didn't jump to line 239 because the condition on line 238 was never true
239 raise TypeError("build_manifest can only be called once!")
240 self._used = True
241 return begin_parsing_context(
242 self._value_table,
243 self._source_package,
244 self._build_manifest,
245 )
247 def _build_manifest(self) -> HighLevelManifest:
248 self._ensure_package_states_is_initialized()
249 for var, attribute_path in self._declared_variables.items():
250 if not self.substitution.is_used(var):
251 raise ManifestParseException(
252 f'The variable "{var}" is unused. Either use it or remove it.'
253 f" The variable was declared at {attribute_path.path_key_lc}."
254 )
255 if isinstance(self, YAMLManifestParser) and self._mutable_yaml_manifest is None:
256 self._mutable_yaml_manifest = MutableYAMLManifest.empty_manifest()
257 all_packager_provided_files = detect_all_packager_provided_files(
258 self._plugin_provided_feature_set,
259 self._debian_dir,
260 self.binary_packages,
261 )
263 for package in self._package_states:
264 with self.binary_package_context(package) as context:
265 if not context.is_auto_generated_package:
266 ppf_result = all_packager_provided_files[package]
267 if ppf_result.auto_installable: 267 ↛ 268line 267 didn't jump to line 268 because the condition on line 267 was never true
268 context.install_rules.append(
269 PPFInstallRule(
270 context.binary_package,
271 context.substitution,
272 ppf_result.auto_installable,
273 )
274 )
275 context.reserved_packager_provided_files.update(
276 ppf_result.reserved_only
277 )
278 self._transform_dpkg_maintscript_helpers_to_snippets()
279 build_environments = self.build_environments()
280 assert build_environments is not None
282 return HighLevelManifest(
283 self.manifest_path,
284 self._mutable_yaml_manifest,
285 self._remove_during_clean_rules,
286 self._install_rules,
287 self._source_package,
288 self.binary_packages,
289 self.substitution,
290 self._package_states,
291 self._dpkg_architecture_variables,
292 self._dpkg_arch_query_table,
293 self._deb_options_and_profiles,
294 build_environments,
295 self._build_rules,
296 self._value_table,
297 self._plugin_provided_feature_set,
298 self._debian_dir,
299 )
301 @contextlib.contextmanager
302 def binary_package_context(
303 self,
304 package_name: str,
305 ) -> Iterator[PackageTransformationDefinition]:
306 if package_name not in self._package_states:
307 self._error(
308 f'The package "{package_name}" is not present in the debian/control file (could not find'
309 f' "Package: {package_name}" in a binary stanza) nor is it a -dbgsym package for one'
310 " for a package in debian/control."
311 )
312 package_state = self._package_states[package_name]
313 self._package_state_stack.append(package_state)
314 ps_len = len(self._package_state_stack)
315 with with_binary_pkg_parsing_context(package_state.binary_package):
316 yield package_state
317 if ps_len != len(self._package_state_stack): 317 ↛ 318line 317 didn't jump to line 318 because the condition on line 317 was never true
318 raise RuntimeError("Internal error: Unbalanced stack manipulation detected")
319 self._package_state_stack.pop()
321 def dispatch_parser_table_for(self, rule_type: TTP) -> DispatchingTableParser[TP]:
322 t = self._plugin_provided_feature_set.manifest_parser_generator.dispatch_parser_table_for(
323 rule_type
324 )
325 if t is None:
326 raise AssertionError(
327 f"Internal error: No dispatching parser for {rule_type.__name__}"
328 )
329 return t
331 @property
332 def substitution(self) -> Substitution:
333 if self._package_state_stack:
334 return self._package_state_stack[-1].substitution
335 return self._substitution
337 def add_extra_substitution_variables(
338 self,
339 **extra_substitutions: tuple[str, AttributePath],
340 ) -> Substitution:
341 if self._package_state_stack or self._all_package_states is not None: 341 ↛ 346line 341 didn't jump to line 346 because the condition on line 341 was never true
342 # For one, it would not "bubble up" correctly when added to the lowest stack.
343 # And if it is not added to the lowest stack, then you get errors about it being
344 # unknown as soon as you leave the stack (which is weird for the user when
345 # the variable is something known, sometimes not)
346 raise RuntimeError("Cannot use add_extra_substitution from this state")
347 for key, (_, path) in extra_substitutions.items():
348 self._declared_variables[key] = path
349 self._substitution = self._substitution.with_extra_substitutions(
350 **{k: v[0] for k, v in extra_substitutions.items()}
351 )
352 return self._substitution
354 @property
355 def current_binary_package_state(self) -> PackageTransformationDefinition:
356 if not self._package_state_stack: 356 ↛ 357line 356 didn't jump to line 357 because the condition on line 356 was never true
357 raise RuntimeError("Invalid state: Not in a binary package context")
358 return self._package_state_stack[-1]
360 @property
361 def is_in_binary_package_state(self) -> bool:
362 return bool(self._package_state_stack)
364 @property
365 def debputy_integration_mode(self) -> DebputyIntegrationMode:
366 return self._debputy_integration_mode
368 @debputy_integration_mode.setter
369 def debputy_integration_mode(self, new_value: DebputyIntegrationMode) -> None:
370 self._debputy_integration_mode = new_value
372 def _register_build_environment(
373 self,
374 name: str | None,
375 build_environment: BuildEnvironmentDefinition,
376 attribute_path: AttributePath,
377 is_default: bool = False,
378 ) -> None:
379 assert not self._read_build_environment
381 # TODO: Reference the paths of the original environments for the error messages where that is relevant.
382 if is_default:
383 if self._has_set_default_build_environment: 383 ↛ 384line 383 didn't jump to line 384 because the condition on line 383 was never true
384 raise ManifestParseException(
385 f"There cannot be multiple default environments and"
386 f" therefore {attribute_path.path} cannot be a default environment"
387 )
388 self._has_set_default_build_environment = True
389 self._build_environments.default_environment = build_environment
390 if name is None: 390 ↛ 399line 390 didn't jump to line 399 because the condition on line 390 was always true
391 return
392 elif name is None: 392 ↛ 393line 392 didn't jump to line 393 because the condition on line 392 was never true
393 raise ManifestParseException(
394 f"Useless environment defined at {attribute_path.path}. It is neither the"
395 " default environment nor does it have a name (so no rules can reference it"
396 " explicitly)"
397 )
399 if name in self._build_environments.environments: 399 ↛ 400line 399 didn't jump to line 400 because the condition on line 399 was never true
400 raise ManifestParseException(
401 f'The environment defined at {attribute_path.path} reuse the name "{name}".'
402 " The environment name must be unique."
403 )
404 self._build_environments.environments[name] = build_environment
406 def resolve_build_environment(
407 self,
408 name: str | None,
409 attribute_path: AttributePath,
410 ) -> BuildEnvironmentDefinition:
411 if name is None:
412 return self.build_environments().default_environment
413 try:
414 env = self.build_environments().environments[name]
415 except KeyError:
416 raise ManifestParseException(
417 f'The environment "{name}" requested at {attribute_path.path} was not'
418 f" defined in the `build-environments`"
419 )
420 self._used_named_envs.add(name)
421 return env
423 def build_environments(self) -> BuildEnvironments:
424 v = self._build_environments
425 if (
426 not self._read_build_environment
427 and not self._build_environments.environments
428 and self._build_environments.default_environment is None
429 ):
430 self._build_environments.default_environment = BuildEnvironmentDefinition()
431 self._read_build_environment = True
432 return v
434 def _transform_dpkg_maintscript_helpers_to_snippets(self) -> None:
435 package_state = self.current_binary_package_state
436 for dmh in package_state.dpkg_maintscript_helper_snippets:
437 snippet = MaintscriptSnippet(
438 definition_source=dmh.definition_source,
439 snippet=f'dpkg-maintscript-helper {escape_shell(*dmh.cmdline)} -- "$@"\n',
440 )
441 for script in STD_CONTROL_SCRIPTS:
442 package_state.maintscript_snippets[script].append(snippet)
444 def normalize_path(
445 self,
446 path: str,
447 definition_source: AttributePath,
448 *,
449 allow_root_dir_match: bool = False,
450 ) -> ExactFileSystemPath:
451 try:
452 normalized = _normalize_path(path)
453 except ValueError:
454 self._error(
455 f'The path "{path}" provided in {definition_source.path} should be relative to the root of the'
456 ' package and not use any ".." or "." segments.'
457 )
458 if normalized == "." and not allow_root_dir_match:
459 self._error(
460 "Manifests must not change the root directory of the deb file. Please correct"
461 f' "{definition_source.path}" (path: "{path}) in {self.manifest_path}'
462 )
463 return ExactFileSystemPath(
464 self.substitution.substitute(normalized, definition_source.path)
465 )
467 def parse_path_or_glob(
468 self,
469 path_or_glob: str,
470 definition_source: AttributePath,
471 ) -> MatchRule:
472 match_rule = MatchRule.from_path_or_glob(
473 path_or_glob, definition_source.path, substitution=self.substitution
474 )
475 # NB: "." and "/" will be translated to MATCH_ANYTHING by MatchRule.from_path_or_glob,
476 # so there is no need to check for an exact match on "." like in normalize_path.
477 if match_rule.rule_type == MatchRuleType.MATCH_ANYTHING:
478 self._error(
479 f'The chosen match rule "{path_or_glob}" matches everything (including the deb root directory).'
480 f' Please correct "{definition_source.path}" (path: "{path_or_glob}) in {self.manifest_path} to'
481 f' something that matches "less" than everything.'
482 )
483 return match_rule
485 def parse_manifest(self) -> HighLevelManifest:
486 raise NotImplementedError
489class YAMLManifestParser(HighLevelManifestParser):
490 def _optional_key(
491 self,
492 d: Mapping[str, Any],
493 key: str,
494 attribute_parent_path: AttributePath,
495 expected_type=None,
496 default_value=None,
497 ):
498 v = d.get(key)
499 if v is None:
500 _detect_possible_typo(d, key, attribute_parent_path, False)
501 return default_value
502 if expected_type is not None:
503 return self._ensure_value_is_type(
504 v, expected_type, key, attribute_parent_path
505 )
506 return v
508 def _required_key(
509 self,
510 d: Mapping[str, Any],
511 key: str,
512 attribute_parent_path: AttributePath,
513 expected_type=None,
514 extra: str | Callable[[], str] | None = None,
515 ):
516 v = d.get(key)
517 if v is None:
518 _detect_possible_typo(d, key, attribute_parent_path, True)
519 if extra is not None:
520 msg = extra if isinstance(extra, str) else extra()
521 extra_info = " " + msg
522 else:
523 extra_info = ""
524 self._error(
525 f'Missing required key {key} at {attribute_parent_path.path} in manifest "{self.manifest_path}.'
526 f"{extra_info}"
527 )
529 if expected_type is not None:
530 return self._ensure_value_is_type(
531 v, expected_type, key, attribute_parent_path
532 )
533 return v
535 def _ensure_value_is_type(
536 self,
537 v,
538 t,
539 key: str | int | AttributePath,
540 attribute_parent_path: AttributePath | None,
541 ):
542 if v is None:
543 return None
544 if not isinstance(v, t):
545 if isinstance(t, tuple):
546 t_msg = "one of: " + ", ".join(x.__name__ for x in t)
547 else:
548 t_msg = f"a {t.__name__}"
549 key_path = (
550 key.path
551 if isinstance(key, AttributePath)
552 else assume_not_none(attribute_parent_path)[key].path
553 )
554 self._error(
555 f'The key {key_path} must be {t_msg} in manifest "{self.manifest_path}"'
556 )
557 return v
559 def _from_yaml_dict(self, yaml_data: object) -> "HighLevelManifest":
560 attribute_path = AttributePath.root_path(yaml_data)
561 parser_generator = self._plugin_provided_feature_set.manifest_parser_generator
562 dispatchable_object_parsers = parser_generator.dispatchable_object_parsers
563 manifest_root_parser = dispatchable_object_parsers[OPARSER_MANIFEST_ROOT]
564 parsed_data = manifest_root_parser.parse_input(
565 yaml_data,
566 attribute_path,
567 parser_context=self,
568 )
570 packages_dict: Mapping[str, PackageContextData[Mapping[str, Any]]] = cast(
571 "Mapping[str, PackageContextData[Mapping[str, Any]]]",
572 parsed_data.get("packages", {}),
573 )
574 self._remove_during_clean_rules = parsed_data.get(
575 MK_MANIFEST_REMOVE_DURING_CLEAN, []
576 )
577 install_rules = parsed_data.get(MK_INSTALLATIONS)
578 if install_rules:
579 self._install_rules = install_rules
580 packages_parent_path = attribute_path[MK_PACKAGES]
581 for package_name_raw, pcd in packages_dict.items():
582 definition_source = packages_parent_path[package_name_raw]
583 package_name = pcd.resolved_package_name
584 parsed = pcd.value
586 package_state: PackageTransformationDefinition
587 with self.binary_package_context(package_name) as package_state:
588 if package_state.is_auto_generated_package: 588 ↛ 590line 588 didn't jump to line 590 because the condition on line 588 was never true
589 # Maybe lift (part) of this restriction.
590 self._error(
591 f'Cannot define rules for package "{package_name}" (at {definition_source.path}). It is an'
592 " auto-generated package."
593 )
594 binary_version: str | None = parsed.get(MK_BINARY_VERSION)
595 if binary_version is not None:
596 package_state.binary_version = (
597 package_state.substitution.substitute(
598 binary_version,
599 definition_source[MK_BINARY_VERSION].path,
600 )
601 )
602 search_dirs = parsed.get(MK_INSTALLATION_SEARCH_DIRS)
603 if search_dirs is not None: 603 ↛ 604line 603 didn't jump to line 604 because the condition on line 603 was never true
604 package_state.search_dirs = search_dirs
605 transformations = parsed.get(MK_TRANSFORMATIONS)
606 conffile_management = parsed.get(MK_CONFFILE_MANAGEMENT)
607 service_rules = parsed.get(MK_SERVICES)
608 if transformations:
609 package_state.transformations.extend(transformations)
610 if conffile_management:
611 package_state.dpkg_maintscript_helper_snippets.extend(
612 conffile_management
613 )
614 if service_rules: 614 ↛ 615line 614 didn't jump to line 615 because the condition on line 614 was never true
615 package_state.requested_service_rules.extend(service_rules)
616 self._build_rules = parsed_data.get("builds")
618 return self.build_manifest()
620 def _parse_manifest(self, fd: IO[bytes] | str) -> HighLevelManifest:
621 try:
622 data = MANIFEST_YAML.load(fd)
623 except YAMLError as e:
624 msg = str(e)
625 lines = msg.splitlines(keepends=True)
626 i = -1
627 for i, line in enumerate(lines):
628 # Avoid an irrelevant "how do configure the YAML parser" message, which the
629 # user cannot use.
630 if line.startswith("To suppress this check"):
631 break
632 if i > -1 and len(lines) > i + 1:
633 lines = lines[:i]
634 msg = "".join(lines)
635 msg = msg.rstrip()
636 msg += (
637 f"\n\nYou can use `yamllint -d relaxed {escape_shell(self.manifest_path)}` to validate"
638 " the YAML syntax. The yamllint tool also supports style rules for YAML documents"
639 " (such as indentation rules) in case that is of interest."
640 )
641 raise ManifestParseException(
642 f"Could not parse {self.manifest_path} as a YAML document: {msg}"
643 ) from e
644 self._mutable_yaml_manifest = MutableYAMLManifest(data)
646 return begin_parsing_context(
647 self._value_table,
648 self._source_package,
649 self._from_yaml_dict,
650 data,
651 )
653 def parse_manifest(
654 self,
655 *,
656 fd: IO[bytes] | str | None = None,
657 ) -> HighLevelManifest:
658 if fd is None: 658 ↛ 659line 658 didn't jump to line 659 because the condition on line 658 was never true
659 with open(self.manifest_path, "rb") as fd:
660 return self._parse_manifest(fd)
661 else:
662 return self._parse_manifest(fd)