Coverage for src/debputy/highlevel_manifest_parser.py: 73%
315 statements
« prev ^ index » next coverage.py v7.16.1, created at 2026-10-28 20:14 +0000
1import collections.abc
2import contextlib
3from collections.abc import Callable, Mapping, Iterator
4from typing import (
5 Any,
6 IO,
7 cast,
8 TYPE_CHECKING,
9)
11from debian.debian_support import DpkgArchTable
13import debputy.util
14from debputy.highlevel_manifest import (
15 HighLevelManifest,
16 PackageTransformationDefinition,
17 MutableYAMLManifest,
18)
19from debputy.maintscript_snippet import (
20 UnboundMaintscriptSnippet,
21 DPKG_DEB_CONTROL_SCRIPTS,
22 SnippetResolver,
23 PackageMaintscriptSnippetContainer,
24 SUPPORTED_UDEB_SCRIPTS,
25)
26from debputy.packages import BinaryPackage, SourcePackage
27from debputy.path_matcher import (
28 MatchRuleType,
29 ExactFileSystemPath,
30 MatchRule,
31)
32from debputy.plugin.api.parser_tables import OPARSER_MANIFEST_ROOT
33from debputy.plugins.debputy.build_system_rules import BuildRule
34from debputy.substitution import Substitution
35from debputy.util import (
36 _normalize_path,
37 escape_shell,
38 assume_not_none,
39)
40from debputy.util import _warn, _info
41from ._deb_options_profiles import DebBuildOptionsAndProfiles
42from ._manifest_constants import (
43 MK_CONFFILE_MANAGEMENT,
44 MK_INSTALLATIONS,
45 MK_PACKAGES,
46 MK_INSTALLATION_SEARCH_DIRS,
47 MK_TRANSFORMATIONS,
48 MK_BINARY_VERSION,
49 MK_SERVICES,
50 MK_MANIFEST_REMOVE_DURING_CLEAN,
51)
52from .architecture_support import DpkgArchitectureBuildProcessValuesTable
53from .filesystem_scan import OSFSROOverlay
54from .installations import InstallRule, PPFInstallRule
55from .manifest_parser.base_types import (
56 BuildEnvironments,
57 BuildEnvironmentDefinition,
58 FileSystemMatchRule,
59)
60from .manifest_parser.exceptions import ManifestParseException
61from .manifest_parser.parser_data import ParserContextData
62from .manifest_parser.util import AttributePath
63from .packager_provided_files import detect_all_packager_provided_files
64from .plugin.api import VirtualPath
65from .plugin.api.feature_set import PluginProvidedFeatureSet
66from .plugin.api.impl_types import (
67 TP,
68 TTP,
69 DispatchingTableParser,
70 PackageContextData,
71)
72from .plugin.api.spec import DebputyIntegrationMode
73from .plugin.plugin_state import with_binary_pkg_parsing_context, begin_parsing_context
74from .yaml import YAMLError, MANIFEST_YAML
77def _detect_possible_typo(
78 d: collections.abc.Iterable[str],
79 key: str,
80 attribute_parent_path: AttributePath,
81 required: bool,
82) -> None:
83 if debputy.util.CAN_DETECT_TYPOS:
84 k_len = len(key)
85 for actual_key in d:
86 if abs(k_len - len(actual_key)) > 2:
87 continue
88 if debputy.util.distance(key, actual_key) > 2:
89 continue
90 path = attribute_parent_path.path
91 ref = f'at "{path}"' if path else "at the manifest root level"
92 _warn(
93 f'Possible typo: The key "{actual_key}" should probably have been "{key}" {ref}'
94 )
95 elif required:
96 _info(
97 "Install python3-levenshtein to have debputy try to detect typos in the manifest."
98 )
101def _per_package_subst_variables(
102 p: BinaryPackage,
103 *,
104 name: str | None = None,
105) -> dict[str, str]:
106 return {
107 "PACKAGE": name if name is not None else p.name,
108 }
111class HighLevelManifestParser(ParserContextData):
112 def __init__(
113 self,
114 manifest_path: str,
115 source_package: SourcePackage,
116 binary_packages: Mapping[str, BinaryPackage],
117 substitution: Substitution,
118 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
119 dpkg_arch_query_table: DpkgArchTable,
120 build_env: DebBuildOptionsAndProfiles,
121 plugin_provided_feature_set: PluginProvidedFeatureSet,
122 debputy_integration_mode: DebputyIntegrationMode,
123 *,
124 # Available for testing purposes only
125 debian_dir: str | VirtualPath = "./debian",
126 ):
127 self.manifest_path = manifest_path
128 self._source_package = source_package
129 self._binary_packages = binary_packages
130 self._mutable_yaml_manifest: MutableYAMLManifest | None = None
131 # In source context, some variables are known to be unresolvable. Record this, so
132 # we can give better error messages.
133 self._substitution = substitution
134 self._dpkg_architecture_variables = dpkg_architecture_variables
135 self._dpkg_arch_query_table = dpkg_arch_query_table
136 self._deb_options_and_profiles = build_env
137 self._package_state_stack: list[PackageTransformationDefinition] = []
138 self._plugin_provided_feature_set = plugin_provided_feature_set
139 self._debputy_integration_mode = debputy_integration_mode
140 self._declared_variables = dict[str, AttributePath]()
141 self._used_named_envs = set[str]()
142 self._build_environments: BuildEnvironments | None = BuildEnvironments(
143 {},
144 None,
145 )
146 self._has_set_default_build_environment = False
147 self._read_build_environment = False
148 self._build_rules: list[BuildRule] | None = None
149 self._value_table: dict[
150 tuple[SourcePackage | BinaryPackage, type[Any]],
151 Any,
152 ] = {}
154 if isinstance(debian_dir, str): 154 ↛ 155line 154 didn't jump to line 155 because the condition on line 154 was never true
155 self._debian_dir: VirtualPath = OSFSROOverlay.create_root_dir(
156 "debian", debian_dir
157 )
158 else:
159 self._debian_dir = debian_dir
161 # Delayed initialized; we rely on this delay to parse the variables.
162 self._all_package_states: dict[str, PackageTransformationDefinition] | None = (
163 None
164 )
166 self._install_rules: list[InstallRule] | None = None
167 self._remove_during_clean_rules: list[FileSystemMatchRule] = []
168 self._ownership_caches_loaded = False
169 self._used = False
171 def _ensure_package_states_is_initialized(self) -> None:
172 if self._all_package_states is not None:
173 return
174 substitution = self._substitution
175 binary_packages = self._binary_packages
177 self._all_package_states = {
178 n: PackageTransformationDefinition(
179 binary_package=p,
180 substitution=substitution.with_extra_substitutions(
181 **_per_package_subst_variables(p)
182 ),
183 is_auto_generated_package=False,
184 maintscript_snippets=PackageMaintscriptSnippetContainer(
185 SUPPORTED_UDEB_SCRIPTS if p.is_udeb else DPKG_DEB_CONTROL_SCRIPTS
186 ),
187 )
188 for n, p in binary_packages.items()
189 }
190 for n, p in binary_packages.items():
191 dbgsym_name = f"{n}-dbgsym"
192 if dbgsym_name in self._all_package_states: 192 ↛ 193line 192 didn't jump to line 193 because the condition on line 192 was never true
193 continue
194 self._all_package_states[dbgsym_name] = PackageTransformationDefinition(
195 binary_package=p,
196 substitution=substitution.with_extra_substitutions(
197 **_per_package_subst_variables(p, name=dbgsym_name)
198 ),
199 is_auto_generated_package=True,
200 maintscript_snippets=PackageMaintscriptSnippetContainer(frozenset()),
201 )
203 @property
204 def source_package(self) -> SourcePackage:
205 return self._source_package
207 @property
208 def binary_packages(self) -> Mapping[str, BinaryPackage]:
209 return self._binary_packages
211 @property
212 def _package_states(self) -> Mapping[str, PackageTransformationDefinition]:
213 assert self._all_package_states is not None
214 return self._all_package_states
216 @property
217 def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable:
218 return self._dpkg_architecture_variables
220 @property
221 def dpkg_arch_query_table(self) -> DpkgArchTable:
222 return self._dpkg_arch_query_table
224 @property
225 def deb_options_and_profiles(self) -> DebBuildOptionsAndProfiles:
226 return self._deb_options_and_profiles
228 def _self_check(self) -> None:
229 unused_envs = (
230 self._build_environments.environments.keys() - self._used_named_envs
231 )
232 if unused_envs:
233 unused_env_names = ", ".join(unused_envs)
234 raise ManifestParseException(
235 f"The following named environments were never referenced: {unused_env_names}"
236 )
238 def build_manifest(self) -> HighLevelManifest:
239 self._self_check()
240 if self._used: 240 ↛ 241line 240 didn't jump to line 241 because the condition on line 240 was never true
241 raise TypeError("build_manifest can only be called once!")
242 self._used = True
243 return begin_parsing_context(
244 self._value_table,
245 self._source_package,
246 self._build_manifest,
247 )
249 def _build_manifest(self) -> HighLevelManifest:
250 self._ensure_package_states_is_initialized()
251 for var, attribute_path in self._declared_variables.items():
252 if not self.substitution.is_used(var):
253 raise ManifestParseException(
254 f'The variable "{var}" is unused. Either use it or remove it.'
255 f" The variable was declared at {attribute_path.path_key_lc}."
256 )
257 if isinstance(self, YAMLManifestParser) and self._mutable_yaml_manifest is None:
258 self._mutable_yaml_manifest = MutableYAMLManifest.empty_manifest()
259 all_packager_provided_files = detect_all_packager_provided_files(
260 self._plugin_provided_feature_set,
261 self._debian_dir,
262 self.binary_packages,
263 )
265 for package in self._package_states:
266 with self.binary_package_context(package) as context:
267 if not context.is_auto_generated_package:
268 ppf_result = all_packager_provided_files[package]
269 if ppf_result.auto_installable: 269 ↛ 270line 269 didn't jump to line 270 because the condition on line 269 was never true
270 context.install_rules.append(
271 PPFInstallRule(
272 context.binary_package,
273 context.substitution,
274 ppf_result.auto_installable,
275 )
276 )
277 context.reserved_packager_provided_files.update(
278 ppf_result.reserved_only
279 )
280 self._transform_dpkg_maintscript_helpers_to_snippets()
281 build_environments = self.build_environments()
282 assert build_environments is not None
284 return HighLevelManifest(
285 self.manifest_path,
286 self._mutable_yaml_manifest,
287 self._remove_during_clean_rules,
288 self._install_rules,
289 self._source_package,
290 self.binary_packages,
291 self.substitution,
292 self._package_states,
293 self._dpkg_architecture_variables,
294 self._dpkg_arch_query_table,
295 self._deb_options_and_profiles,
296 build_environments,
297 self._build_rules,
298 self._value_table,
299 self._plugin_provided_feature_set,
300 self._debian_dir,
301 )
303 @contextlib.contextmanager
304 def binary_package_context(
305 self,
306 package_name: str,
307 ) -> Iterator[PackageTransformationDefinition]:
308 if package_name not in self._package_states:
309 self._error(
310 f'The package "{package_name}" is not present in the debian/control file (could not find'
311 f' "Package: {package_name}" in a binary stanza) nor is it a -dbgsym package for one'
312 " for a package in debian/control."
313 )
314 package_state = self._package_states[package_name]
315 self._package_state_stack.append(package_state)
316 ps_len = len(self._package_state_stack)
317 with with_binary_pkg_parsing_context(package_state.binary_package):
318 yield package_state
319 if ps_len != len(self._package_state_stack): 319 ↛ 320line 319 didn't jump to line 320 because the condition on line 319 was never true
320 raise RuntimeError("Internal error: Unbalanced stack manipulation detected")
321 self._package_state_stack.pop()
323 def dispatch_parser_table_for(self, rule_type: TTP) -> DispatchingTableParser[TP]:
324 t = self._plugin_provided_feature_set.manifest_parser_generator.dispatch_parser_table_for(
325 rule_type
326 )
327 if t is None:
328 raise AssertionError(
329 f"Internal error: No dispatching parser for {rule_type.__name__}"
330 )
331 return t
333 @property
334 def substitution(self) -> Substitution:
335 if self._package_state_stack:
336 return self._package_state_stack[-1].substitution
337 return self._substitution
339 def add_extra_substitution_variables(
340 self,
341 **extra_substitutions: tuple[str, AttributePath],
342 ) -> Substitution:
343 if self._package_state_stack or self._all_package_states is not None: 343 ↛ 348line 343 didn't jump to line 348 because the condition on line 343 was never true
344 # For one, it would not "bubble up" correctly when added to the lowest stack.
345 # And if it is not added to the lowest stack, then you get errors about it being
346 # unknown as soon as you leave the stack (which is weird for the user when
347 # the variable is something known, sometimes not)
348 raise RuntimeError("Cannot use add_extra_substitution from this state")
349 for key, (_, path) in extra_substitutions.items():
350 self._declared_variables[key] = path
351 self._substitution = self._substitution.with_extra_substitutions(
352 **{k: v[0] for k, v in extra_substitutions.items()}
353 )
354 return self._substitution
356 @property
357 def current_binary_package_state(self) -> PackageTransformationDefinition:
358 if not self._package_state_stack: 358 ↛ 359line 358 didn't jump to line 359 because the condition on line 358 was never true
359 raise RuntimeError("Invalid state: Not in a binary package context")
360 return self._package_state_stack[-1]
362 @property
363 def is_in_binary_package_state(self) -> bool:
364 return bool(self._package_state_stack)
366 @property
367 def debputy_integration_mode(self) -> DebputyIntegrationMode:
368 return self._debputy_integration_mode
370 @debputy_integration_mode.setter
371 def debputy_integration_mode(self, new_value: DebputyIntegrationMode) -> None:
372 self._debputy_integration_mode = new_value
374 def _register_build_environment(
375 self,
376 name: str | None,
377 build_environment: BuildEnvironmentDefinition,
378 attribute_path: AttributePath,
379 is_default: bool = False,
380 ) -> None:
381 assert not self._read_build_environment
383 # TODO: Reference the paths of the original environments for the error messages where that is relevant.
384 if is_default:
385 if self._has_set_default_build_environment: 385 ↛ 386line 385 didn't jump to line 386 because the condition on line 385 was never true
386 raise ManifestParseException(
387 f"There cannot be multiple default environments and"
388 f" therefore {attribute_path.path} cannot be a default environment"
389 )
390 self._has_set_default_build_environment = True
391 self._build_environments.default_environment = build_environment
392 if name is None: 392 ↛ 401line 392 didn't jump to line 401 because the condition on line 392 was always true
393 return
394 elif name is None: 394 ↛ 395line 394 didn't jump to line 395 because the condition on line 394 was never true
395 raise ManifestParseException(
396 f"Useless environment defined at {attribute_path.path}. It is neither the"
397 " default environment nor does it have a name (so no rules can reference it"
398 " explicitly)"
399 )
401 if name in self._build_environments.environments: 401 ↛ 402line 401 didn't jump to line 402 because the condition on line 401 was never true
402 raise ManifestParseException(
403 f'The environment defined at {attribute_path.path} reuse the name "{name}".'
404 " The environment name must be unique."
405 )
406 self._build_environments.environments[name] = build_environment
408 def resolve_build_environment(
409 self,
410 name: str | None,
411 attribute_path: AttributePath,
412 ) -> BuildEnvironmentDefinition:
413 if name is None:
414 return self.build_environments().default_environment
415 try:
416 env = self.build_environments().environments[name]
417 except KeyError:
418 raise ManifestParseException(
419 f'The environment "{name}" requested at {attribute_path.path} was not'
420 f" defined in the `build-environments`"
421 )
422 self._used_named_envs.add(name)
423 return env
425 def build_environments(self) -> BuildEnvironments:
426 v = self._build_environments
427 if (
428 not self._read_build_environment
429 and not self._build_environments.environments
430 and self._build_environments.default_environment is None
431 ):
432 self._build_environments.default_environment = BuildEnvironmentDefinition()
433 self._read_build_environment = True
434 return v
436 def _transform_dpkg_maintscript_helpers_to_snippets(self) -> None:
437 package_state = self.current_binary_package_state
438 for dmh in package_state.dpkg_maintscript_helper_snippets:
439 snippet = UnboundMaintscriptSnippet(
440 definition_source=dmh.definition_source,
441 snippet=SnippetResolver.snippet(
442 f'dpkg-maintscript-helper {escape_shell(*dmh.cmdline)} -- "$@"\n'
443 ),
444 )
445 for script in DPKG_DEB_CONTROL_SCRIPTS:
446 package_state.maintscript_snippets[script].append(snippet)
448 def normalize_path(
449 self,
450 path: str,
451 definition_source: AttributePath,
452 *,
453 allow_root_dir_match: bool = False,
454 ) -> ExactFileSystemPath:
455 try:
456 normalized = _normalize_path(path)
457 except ValueError:
458 self._error(
459 f'The path "{path}" provided in {definition_source.path} should be relative to the root of the'
460 ' package and not use any ".." or "." segments.'
461 )
462 if normalized == "." and not allow_root_dir_match:
463 self._error(
464 "Manifests must not change the root directory of the deb file. Please correct"
465 f' "{definition_source.path}" (path: "{path}) in {self.manifest_path}'
466 )
467 return ExactFileSystemPath(
468 self.substitution.substitute(normalized, definition_source.path)
469 )
471 def parse_path_or_glob(
472 self,
473 path_or_glob: str,
474 definition_source: AttributePath,
475 ) -> MatchRule:
476 match_rule = MatchRule.from_path_or_glob(
477 path_or_glob, definition_source.path, substitution=self.substitution
478 )
479 # NB: "." and "/" will be translated to MATCH_ANYTHING by MatchRule.from_path_or_glob,
480 # so there is no need to check for an exact match on "." like in normalize_path.
481 if match_rule.rule_type == MatchRuleType.MATCH_ANYTHING:
482 self._error(
483 f'The chosen match rule "{path_or_glob}" matches everything (including the deb root directory).'
484 f' Please correct "{definition_source.path}" (path: "{path_or_glob}) in {self.manifest_path} to'
485 f' something that matches "less" than everything.'
486 )
487 return match_rule
489 def parse_manifest(self) -> HighLevelManifest:
490 raise NotImplementedError
493class YAMLManifestParser(HighLevelManifestParser):
494 def _optional_key(
495 self,
496 d: Mapping[str, Any],
497 key: str,
498 attribute_parent_path: AttributePath,
499 expected_type=None,
500 default_value=None,
501 ):
502 v = d.get(key)
503 if v is None:
504 _detect_possible_typo(d, key, attribute_parent_path, False)
505 return default_value
506 if expected_type is not None:
507 return self._ensure_value_is_type(
508 v, expected_type, key, attribute_parent_path
509 )
510 return v
512 def _required_key(
513 self,
514 d: Mapping[str, Any],
515 key: str,
516 attribute_parent_path: AttributePath,
517 expected_type=None,
518 extra: str | Callable[[], str] | None = None,
519 ):
520 v = d.get(key)
521 if v is None:
522 _detect_possible_typo(d, key, attribute_parent_path, True)
523 if extra is not None:
524 msg = extra if isinstance(extra, str) else extra()
525 extra_info = " " + msg
526 else:
527 extra_info = ""
528 self._error(
529 f'Missing required key {key} at {attribute_parent_path.path} in manifest "{self.manifest_path}.'
530 f"{extra_info}"
531 )
533 if expected_type is not None:
534 return self._ensure_value_is_type(
535 v, expected_type, key, attribute_parent_path
536 )
537 return v
539 def _ensure_value_is_type(
540 self,
541 v,
542 t,
543 key: str | int | AttributePath,
544 attribute_parent_path: AttributePath | None,
545 ):
546 if v is None:
547 return None
548 if not isinstance(v, t):
549 if isinstance(t, tuple):
550 t_msg = "one of: " + ", ".join(x.__name__ for x in t)
551 else:
552 t_msg = f"a {t.__name__}"
553 key_path = (
554 key.path
555 if isinstance(key, AttributePath)
556 else assume_not_none(attribute_parent_path)[key].path
557 )
558 self._error(
559 f'The key {key_path} must be {t_msg} in manifest "{self.manifest_path}"'
560 )
561 return v
563 def _from_yaml_dict(self, yaml_data: object) -> "HighLevelManifest":
564 attribute_path = AttributePath.root_path(yaml_data)
565 parser_generator = self._plugin_provided_feature_set.manifest_parser_generator
566 dispatchable_object_parsers = parser_generator.dispatchable_object_parsers
567 manifest_root_parser = dispatchable_object_parsers[OPARSER_MANIFEST_ROOT]
568 parsed_data = manifest_root_parser.parse_input(
569 yaml_data,
570 attribute_path,
571 parser_context=self,
572 )
574 packages_dict: Mapping[str, PackageContextData[Mapping[str, Any]]] = cast(
575 "Mapping[str, PackageContextData[Mapping[str, Any]]]",
576 parsed_data.get("packages", {}),
577 )
578 self._remove_during_clean_rules = parsed_data.get(
579 MK_MANIFEST_REMOVE_DURING_CLEAN, []
580 )
581 install_rules = parsed_data.get(MK_INSTALLATIONS)
582 if install_rules:
583 self._install_rules = install_rules
584 packages_parent_path = attribute_path[MK_PACKAGES]
585 for package_name_raw, pcd in packages_dict.items():
586 definition_source = packages_parent_path[package_name_raw]
587 package_name = pcd.resolved_package_name
588 parsed = pcd.value
590 package_state: PackageTransformationDefinition
591 with self.binary_package_context(package_name) as package_state:
592 if package_state.is_auto_generated_package: 592 ↛ 594line 592 didn't jump to line 594 because the condition on line 592 was never true
593 # Maybe lift (part) of this restriction.
594 self._error(
595 f'Cannot define rules for package "{package_name}" (at {definition_source.path}). It is an'
596 " auto-generated package."
597 )
598 binary_version: str | None = parsed.get(MK_BINARY_VERSION)
599 if binary_version is not None:
600 package_state.binary_version = (
601 package_state.substitution.substitute(
602 binary_version,
603 definition_source[MK_BINARY_VERSION].path,
604 )
605 )
606 search_dirs = parsed.get(MK_INSTALLATION_SEARCH_DIRS)
607 if search_dirs is not None: 607 ↛ 608line 607 didn't jump to line 608 because the condition on line 607 was never true
608 package_state.search_dirs = search_dirs
609 transformations = parsed.get(MK_TRANSFORMATIONS)
610 conffile_management = parsed.get(MK_CONFFILE_MANAGEMENT)
611 service_rules = parsed.get(MK_SERVICES)
612 if transformations:
613 package_state.transformations.extend(transformations)
614 if conffile_management:
615 package_state.dpkg_maintscript_helper_snippets.extend(
616 conffile_management
617 )
618 if service_rules: 618 ↛ 619line 618 didn't jump to line 619 because the condition on line 618 was never true
619 package_state.requested_service_rules.extend(service_rules)
620 self._build_rules = parsed_data.get("builds")
622 return self.build_manifest()
624 def _parse_manifest(self, fd: IO[bytes] | str) -> HighLevelManifest:
625 try:
626 data = MANIFEST_YAML.load(fd)
627 except YAMLError as e:
628 msg = str(e)
629 lines = msg.splitlines(keepends=True)
630 i = -1
631 for i, line in enumerate(lines):
632 # Avoid an irrelevant "how do configure the YAML parser" message, which the
633 # user cannot use.
634 if line.startswith("To suppress this check"):
635 break
636 if i > -1 and len(lines) > i + 1:
637 lines = lines[:i]
638 msg = "".join(lines)
639 msg = msg.rstrip()
640 msg += (
641 f"\n\nYou can use `yamllint -d relaxed {escape_shell(self.manifest_path)}` to validate"
642 " the YAML syntax. The yamllint tool also supports style rules for YAML documents"
643 " (such as indentation rules) in case that is of interest."
644 )
645 raise ManifestParseException(
646 f"Could not parse {self.manifest_path} as a YAML document: {msg}"
647 ) from e
648 self._mutable_yaml_manifest = MutableYAMLManifest(data)
650 return begin_parsing_context(
651 self._value_table,
652 self._source_package,
653 self._from_yaml_dict,
654 data,
655 )
657 def parse_manifest(
658 self,
659 *,
660 fd: IO[bytes] | str | None = None,
661 ) -> HighLevelManifest:
662 if fd is None: 662 ↛ 663line 662 didn't jump to line 663 because the condition on line 662 was never true
663 with open(self.manifest_path, "rb") as fd:
664 return self._parse_manifest(fd)
665 else:
666 return self._parse_manifest(fd)