Coverage for src/debputy/highlevel_manifest_parser.py: 73%
319 statements
« prev ^ index » next coverage.py v7.8.2, created at 2025-12-28 16:12 +0000
1import collections
2import contextlib
3from collections.abc import Callable, Mapping, Iterator
4from typing import (
5 Any,
6 IO,
7 cast,
8 TYPE_CHECKING,
9)
11from debian.debian_support import DpkgArchTable
13from debputy.highlevel_manifest import (
14 HighLevelManifest,
15 PackageTransformationDefinition,
16 MutableYAMLManifest,
17)
18from debputy.maintscript_snippet import (
19 MaintscriptSnippet,
20 STD_CONTROL_SCRIPTS,
21 MaintscriptSnippetContainer,
22)
23from debputy.packages import BinaryPackage, SourcePackage
24from debputy.path_matcher import (
25 MatchRuleType,
26 ExactFileSystemPath,
27 MatchRule,
28)
29from debputy.plugin.api.parser_tables import OPARSER_MANIFEST_ROOT
30from debputy.plugins.debputy.build_system_rules import BuildRule
31from debputy.substitution import Substitution
32from debputy.util import (
33 _normalize_path,
34 escape_shell,
35 assume_not_none,
36)
37from debputy.util import _warn, _info
38from ._deb_options_profiles import DebBuildOptionsAndProfiles
39from ._manifest_constants import (
40 MK_CONFFILE_MANAGEMENT,
41 MK_INSTALLATIONS,
42 MK_PACKAGES,
43 MK_INSTALLATION_SEARCH_DIRS,
44 MK_TRANSFORMATIONS,
45 MK_BINARY_VERSION,
46 MK_SERVICES,
47 MK_MANIFEST_REMOVE_DURING_CLEAN,
48)
49from .architecture_support import DpkgArchitectureBuildProcessValuesTable
50from .filesystem_scan import OSFSROOverlay
51from .installations import InstallRule, PPFInstallRule
52from .manifest_parser.base_types import (
53 BuildEnvironments,
54 BuildEnvironmentDefinition,
55 FileSystemMatchRule,
56)
57from .manifest_parser.exceptions import ManifestParseException
58from .manifest_parser.parser_data import ParserContextData
59from .manifest_parser.util import AttributePath
60from .packager_provided_files import detect_all_packager_provided_files
61from .plugin.api import VirtualPath
62from .plugin.api.feature_set import PluginProvidedFeatureSet
63from .plugin.api.impl_types import (
64 TP,
65 TTP,
66 DispatchingTableParser,
67 PackageContextData,
68)
69from .plugin.api.spec import DebputyIntegrationMode
70from .plugin.plugin_state import with_binary_pkg_parsing_context, begin_parsing_context
71from .yaml import YAMLError, MANIFEST_YAML
74if TYPE_CHECKING:
75 from .plugins.debputy.binary_package_rules import BinaryVersion
78try:
79 from Levenshtein import distance
80except ImportError:
82 def _detect_possible_typo(
83 _d,
84 _key,
85 _attribute_parent_path: AttributePath,
86 required: bool,
87 ) -> None:
88 if required:
89 _info(
90 "Install python3-levenshtein to have debputy try to detect typos in the manifest."
91 )
93else:
95 def _detect_possible_typo(
96 d,
97 key,
98 _attribute_parent_path: AttributePath,
99 _required: bool,
100 ) -> None:
101 k_len = len(key)
102 for actual_key in d:
103 if abs(k_len - len(actual_key)) > 2:
104 continue
105 d = distance(key, actual_key)
106 if d > 2:
107 continue
108 path = _attribute_parent_path.path
109 ref = f'at "{path}"' if path else "at the manifest root level"
110 _warn(
111 f'Possible typo: The key "{actual_key}" should probably have been "{key}" {ref}'
112 )
115def _per_package_subst_variables(
116 p: BinaryPackage,
117 *,
118 name: str | None = None,
119) -> dict[str, str]:
120 return {
121 "PACKAGE": name if name is not None else p.name,
122 }
125class HighLevelManifestParser(ParserContextData):
126 def __init__(
127 self,
128 manifest_path: str,
129 source_package: SourcePackage,
130 binary_packages: Mapping[str, BinaryPackage],
131 substitution: Substitution,
132 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
133 dpkg_arch_query_table: DpkgArchTable,
134 build_env: DebBuildOptionsAndProfiles,
135 plugin_provided_feature_set: PluginProvidedFeatureSet,
136 debputy_integration_mode: DebputyIntegrationMode,
137 *,
138 # Available for testing purposes only
139 debian_dir: str | VirtualPath = "./debian",
140 ):
141 self.manifest_path = manifest_path
142 self._source_package = source_package
143 self._binary_packages = binary_packages
144 self._mutable_yaml_manifest: MutableYAMLManifest | None = None
145 # In source context, some variables are known to be unresolvable. Record this, so
146 # we can give better error messages.
147 self._substitution = substitution
148 self._dpkg_architecture_variables = dpkg_architecture_variables
149 self._dpkg_arch_query_table = dpkg_arch_query_table
150 self._deb_options_and_profiles = build_env
151 self._package_state_stack: list[PackageTransformationDefinition] = []
152 self._plugin_provided_feature_set = plugin_provided_feature_set
153 self._debputy_integration_mode = debputy_integration_mode
154 self._declared_variables = {}
155 self._used_named_envs = set()
156 self._build_environments: BuildEnvironments | None = BuildEnvironments(
157 {},
158 None,
159 )
160 self._has_set_default_build_environment = False
161 self._read_build_environment = False
162 self._build_rules: list[BuildRule] | None = None
163 self._value_table: dict[
164 tuple[SourcePackage | BinaryPackage, type[Any]],
165 Any,
166 ] = {}
168 if isinstance(debian_dir, str): 168 ↛ 169line 168 didn't jump to line 169 because the condition on line 168 was never true
169 debian_dir = OSFSROOverlay.create_root_dir("debian", debian_dir)
171 self._debian_dir = debian_dir
173 # Delayed initialized; we rely on this delay to parse the variables.
174 self._all_package_states = None
176 self._install_rules: list[InstallRule] | None = None
177 self._remove_during_clean_rules: list[FileSystemMatchRule] = []
178 self._ownership_caches_loaded = False
179 self._used = False
181 def _ensure_package_states_is_initialized(self) -> None:
182 if self._all_package_states is not None:
183 return
184 substitution = self._substitution
185 binary_packages = self._binary_packages
186 assert self._all_package_states is None
188 self._all_package_states = {
189 n: PackageTransformationDefinition(
190 binary_package=p,
191 substitution=substitution.with_extra_substitutions(
192 **_per_package_subst_variables(p)
193 ),
194 is_auto_generated_package=False,
195 maintscript_snippets=collections.defaultdict(
196 MaintscriptSnippetContainer
197 ),
198 )
199 for n, p in binary_packages.items()
200 }
201 for n, p in binary_packages.items():
202 dbgsym_name = f"{n}-dbgsym"
203 if dbgsym_name in self._all_package_states: 203 ↛ 204line 203 didn't jump to line 204 because the condition on line 203 was never true
204 continue
205 self._all_package_states[dbgsym_name] = PackageTransformationDefinition(
206 binary_package=p,
207 substitution=substitution.with_extra_substitutions(
208 **_per_package_subst_variables(p, name=dbgsym_name)
209 ),
210 is_auto_generated_package=True,
211 maintscript_snippets=collections.defaultdict(
212 MaintscriptSnippetContainer
213 ),
214 )
216 @property
217 def source_package(self) -> SourcePackage:
218 return self._source_package
220 @property
221 def binary_packages(self) -> Mapping[str, BinaryPackage]:
222 return self._binary_packages
224 @property
225 def _package_states(self) -> Mapping[str, PackageTransformationDefinition]:
226 assert self._all_package_states is not None
227 return self._all_package_states
229 @property
230 def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable:
231 return self._dpkg_architecture_variables
233 @property
234 def dpkg_arch_query_table(self) -> DpkgArchTable:
235 return self._dpkg_arch_query_table
237 @property
238 def deb_options_and_profiles(self) -> DebBuildOptionsAndProfiles:
239 return self._deb_options_and_profiles
241 def _self_check(self) -> None:
242 unused_envs = (
243 self._build_environments.environments.keys() - self._used_named_envs
244 )
245 if unused_envs:
246 unused_env_names = ", ".join(unused_envs)
247 raise ManifestParseException(
248 f"The following named environments were never referenced: {unused_env_names}"
249 )
251 def build_manifest(self) -> HighLevelManifest:
252 self._self_check()
253 if self._used: 253 ↛ 254line 253 didn't jump to line 254 because the condition on line 253 was never true
254 raise TypeError("build_manifest can only be called once!")
255 self._used = True
256 return begin_parsing_context(
257 self._value_table,
258 self._source_package,
259 self._build_manifest,
260 )
262 def _build_manifest(self) -> HighLevelManifest:
263 self._ensure_package_states_is_initialized()
264 for var, attribute_path in self._declared_variables.items():
265 if not self.substitution.is_used(var):
266 raise ManifestParseException(
267 f'The variable "{var}" is unused. Either use it or remove it.'
268 f" The variable was declared at {attribute_path.path_key_lc}."
269 )
270 if isinstance(self, YAMLManifestParser) and self._mutable_yaml_manifest is None:
271 self._mutable_yaml_manifest = MutableYAMLManifest.empty_manifest()
272 all_packager_provided_files = detect_all_packager_provided_files(
273 self._plugin_provided_feature_set,
274 self._debian_dir,
275 self.binary_packages,
276 )
278 for package in self._package_states:
279 with self.binary_package_context(package) as context:
280 if not context.is_auto_generated_package:
281 ppf_result = all_packager_provided_files[package]
282 if ppf_result.auto_installable: 282 ↛ 283line 282 didn't jump to line 283 because the condition on line 282 was never true
283 context.install_rules.append(
284 PPFInstallRule(
285 context.binary_package,
286 context.substitution,
287 ppf_result.auto_installable,
288 )
289 )
290 context.reserved_packager_provided_files.update(
291 ppf_result.reserved_only
292 )
293 self._transform_dpkg_maintscript_helpers_to_snippets()
294 build_environments = self.build_environments()
295 assert build_environments is not None
297 return HighLevelManifest(
298 self.manifest_path,
299 self._mutable_yaml_manifest,
300 self._remove_during_clean_rules,
301 self._install_rules,
302 self._source_package,
303 self.binary_packages,
304 self.substitution,
305 self._package_states,
306 self._dpkg_architecture_variables,
307 self._dpkg_arch_query_table,
308 self._deb_options_and_profiles,
309 build_environments,
310 self._build_rules,
311 self._value_table,
312 self._plugin_provided_feature_set,
313 self._debian_dir,
314 )
316 @contextlib.contextmanager
317 def binary_package_context(
318 self,
319 package_name: str,
320 ) -> Iterator[PackageTransformationDefinition]:
321 if package_name not in self._package_states:
322 self._error(
323 f'The package "{package_name}" is not present in the debian/control file (could not find'
324 f' "Package: {package_name}" in a binary stanza) nor is it a -dbgsym package for one'
325 " for a package in debian/control."
326 )
327 package_state = self._package_states[package_name]
328 self._package_state_stack.append(package_state)
329 ps_len = len(self._package_state_stack)
330 with with_binary_pkg_parsing_context(package_state.binary_package):
331 yield package_state
332 if ps_len != len(self._package_state_stack): 332 ↛ 333line 332 didn't jump to line 333 because the condition on line 332 was never true
333 raise RuntimeError("Internal error: Unbalanced stack manipulation detected")
334 self._package_state_stack.pop()
336 def dispatch_parser_table_for(self, rule_type: TTP) -> DispatchingTableParser[TP]:
337 t = self._plugin_provided_feature_set.manifest_parser_generator.dispatch_parser_table_for(
338 rule_type
339 )
340 if t is None:
341 raise AssertionError(
342 f"Internal error: No dispatching parser for {rule_type.__name__}"
343 )
344 return t
346 @property
347 def substitution(self) -> Substitution:
348 if self._package_state_stack:
349 return self._package_state_stack[-1].substitution
350 return self._substitution
352 def add_extra_substitution_variables(
353 self,
354 **extra_substitutions: tuple[str, AttributePath],
355 ) -> Substitution:
356 if self._package_state_stack or self._all_package_states is not None: 356 ↛ 361line 356 didn't jump to line 361 because the condition on line 356 was never true
357 # For one, it would not "bubble up" correctly when added to the lowest stack.
358 # And if it is not added to the lowest stack, then you get errors about it being
359 # unknown as soon as you leave the stack (which is weird for the user when
360 # the variable is something known, sometimes not)
361 raise RuntimeError("Cannot use add_extra_substitution from this state")
362 for key, (_, path) in extra_substitutions.items():
363 self._declared_variables[key] = path
364 self._substitution = self._substitution.with_extra_substitutions(
365 **{k: v[0] for k, v in extra_substitutions.items()}
366 )
367 return self._substitution
369 @property
370 def current_binary_package_state(self) -> PackageTransformationDefinition:
371 if not self._package_state_stack: 371 ↛ 372line 371 didn't jump to line 372 because the condition on line 371 was never true
372 raise RuntimeError("Invalid state: Not in a binary package context")
373 return self._package_state_stack[-1]
375 @property
376 def is_in_binary_package_state(self) -> bool:
377 return bool(self._package_state_stack)
379 @property
380 def debputy_integration_mode(self) -> DebputyIntegrationMode:
381 return self._debputy_integration_mode
383 @debputy_integration_mode.setter
384 def debputy_integration_mode(self, new_value: DebputyIntegrationMode) -> None:
385 self._debputy_integration_mode = new_value
387 def _register_build_environment(
388 self,
389 name: str | None,
390 build_environment: BuildEnvironmentDefinition,
391 attribute_path: AttributePath,
392 is_default: bool = False,
393 ) -> None:
394 assert not self._read_build_environment
396 # TODO: Reference the paths of the original environments for the error messages where that is relevant.
397 if is_default:
398 if self._has_set_default_build_environment: 398 ↛ 399line 398 didn't jump to line 399 because the condition on line 398 was never true
399 raise ManifestParseException(
400 f"There cannot be multiple default environments and"
401 f" therefore {attribute_path.path} cannot be a default environment"
402 )
403 self._has_set_default_build_environment = True
404 self._build_environments.default_environment = build_environment
405 if name is None: 405 ↛ 414line 405 didn't jump to line 414 because the condition on line 405 was always true
406 return
407 elif name is None: 407 ↛ 408line 407 didn't jump to line 408 because the condition on line 407 was never true
408 raise ManifestParseException(
409 f"Useless environment defined at {attribute_path.path}. It is neither the"
410 " default environment nor does it have a name (so no rules can reference it"
411 " explicitly)"
412 )
414 if name in self._build_environments.environments: 414 ↛ 415line 414 didn't jump to line 415 because the condition on line 414 was never true
415 raise ManifestParseException(
416 f'The environment defined at {attribute_path.path} reuse the name "{name}".'
417 " The environment name must be unique."
418 )
419 self._build_environments.environments[name] = build_environment
421 def resolve_build_environment(
422 self,
423 name: str | None,
424 attribute_path: AttributePath,
425 ) -> BuildEnvironmentDefinition:
426 if name is None:
427 return self.build_environments().default_environment
428 try:
429 env = self.build_environments().environments[name]
430 except KeyError:
431 raise ManifestParseException(
432 f'The environment "{name}" requested at {attribute_path.path} was not'
433 f" defined in the `build-environments`"
434 )
435 else:
436 self._used_named_envs.add(name)
437 return env
439 def build_environments(self) -> BuildEnvironments:
440 v = self._build_environments
441 if (
442 not self._read_build_environment
443 and not self._build_environments.environments
444 and self._build_environments.default_environment is None
445 ):
446 self._build_environments.default_environment = BuildEnvironmentDefinition()
447 self._read_build_environment = True
448 return v
450 def _transform_dpkg_maintscript_helpers_to_snippets(self) -> None:
451 package_state = self.current_binary_package_state
452 for dmh in package_state.dpkg_maintscript_helper_snippets:
453 snippet = MaintscriptSnippet(
454 definition_source=dmh.definition_source,
455 snippet=f'dpkg-maintscript-helper {escape_shell(*dmh.cmdline)} -- "$@"\n',
456 )
457 for script in STD_CONTROL_SCRIPTS:
458 package_state.maintscript_snippets[script].append(snippet)
460 def normalize_path(
461 self,
462 path: str,
463 definition_source: AttributePath,
464 *,
465 allow_root_dir_match: bool = False,
466 ) -> ExactFileSystemPath:
467 try:
468 normalized = _normalize_path(path)
469 except ValueError:
470 self._error(
471 f'The path "{path}" provided in {definition_source.path} should be relative to the root of the'
472 ' package and not use any ".." or "." segments.'
473 )
474 if normalized == "." and not allow_root_dir_match:
475 self._error(
476 "Manifests must not change the root directory of the deb file. Please correct"
477 f' "{definition_source.path}" (path: "{path}) in {self.manifest_path}'
478 )
479 return ExactFileSystemPath(
480 self.substitution.substitute(normalized, definition_source.path)
481 )
483 def parse_path_or_glob(
484 self,
485 path_or_glob: str,
486 definition_source: AttributePath,
487 ) -> MatchRule:
488 match_rule = MatchRule.from_path_or_glob(
489 path_or_glob, definition_source.path, substitution=self.substitution
490 )
491 # NB: "." and "/" will be translated to MATCH_ANYTHING by MatchRule.from_path_or_glob,
492 # so there is no need to check for an exact match on "." like in normalize_path.
493 if match_rule.rule_type == MatchRuleType.MATCH_ANYTHING:
494 self._error(
495 f'The chosen match rule "{path_or_glob}" matches everything (including the deb root directory).'
496 f' Please correct "{definition_source.path}" (path: "{path_or_glob}) in {self.manifest_path} to'
497 f' something that matches "less" than everything.'
498 )
499 return match_rule
501 def parse_manifest(self) -> HighLevelManifest:
502 raise NotImplementedError
505class YAMLManifestParser(HighLevelManifestParser):
506 def _optional_key(
507 self,
508 d: Mapping[str, Any],
509 key: str,
510 attribute_parent_path: AttributePath,
511 expected_type=None,
512 default_value=None,
513 ):
514 v = d.get(key)
515 if v is None:
516 _detect_possible_typo(d, key, attribute_parent_path, False)
517 return default_value
518 if expected_type is not None:
519 return self._ensure_value_is_type(
520 v, expected_type, key, attribute_parent_path
521 )
522 return v
524 def _required_key(
525 self,
526 d: Mapping[str, Any],
527 key: str,
528 attribute_parent_path: AttributePath,
529 expected_type=None,
530 extra: str | Callable[[], str] | None = None,
531 ):
532 v = d.get(key)
533 if v is None:
534 _detect_possible_typo(d, key, attribute_parent_path, True)
535 if extra is not None:
536 msg = extra if isinstance(extra, str) else extra()
537 extra_info = " " + msg
538 else:
539 extra_info = ""
540 self._error(
541 f'Missing required key {key} at {attribute_parent_path.path} in manifest "{self.manifest_path}.'
542 f"{extra_info}"
543 )
545 if expected_type is not None:
546 return self._ensure_value_is_type(
547 v, expected_type, key, attribute_parent_path
548 )
549 return v
551 def _ensure_value_is_type(
552 self,
553 v,
554 t,
555 key: str | int | AttributePath,
556 attribute_parent_path: AttributePath | None,
557 ):
558 if v is None:
559 return None
560 if not isinstance(v, t):
561 if isinstance(t, tuple):
562 t_msg = "one of: " + ", ".join(x.__name__ for x in t)
563 else:
564 t_msg = f"a {t.__name__}"
565 key_path = (
566 key.path
567 if isinstance(key, AttributePath)
568 else assume_not_none(attribute_parent_path)[key].path
569 )
570 self._error(
571 f'The key {key_path} must be {t_msg} in manifest "{self.manifest_path}"'
572 )
573 return v
575 def _from_yaml_dict(self, yaml_data: object) -> "HighLevelManifest":
576 attribute_path = AttributePath.root_path(yaml_data)
577 parser_generator = self._plugin_provided_feature_set.manifest_parser_generator
578 dispatchable_object_parsers = parser_generator.dispatchable_object_parsers
579 manifest_root_parser = dispatchable_object_parsers[OPARSER_MANIFEST_ROOT]
580 parsed_data = manifest_root_parser.parse_input(
581 yaml_data,
582 attribute_path,
583 parser_context=self,
584 )
586 packages_dict: Mapping[str, PackageContextData[Mapping[str, Any]]] = cast(
587 "Mapping[str, PackageContextData[Mapping[str, Any]]]",
588 parsed_data.get("packages", {}),
589 )
590 self._remove_during_clean_rules = parsed_data.get(
591 MK_MANIFEST_REMOVE_DURING_CLEAN, []
592 )
593 install_rules = parsed_data.get(MK_INSTALLATIONS)
594 if install_rules:
595 self._install_rules = install_rules
596 packages_parent_path = attribute_path[MK_PACKAGES]
597 for package_name_raw, pcd in packages_dict.items():
598 definition_source = packages_parent_path[package_name_raw]
599 package_name = pcd.resolved_package_name
600 parsed = pcd.value
602 package_state: PackageTransformationDefinition
603 with self.binary_package_context(package_name) as package_state:
604 if package_state.is_auto_generated_package: 604 ↛ 606line 604 didn't jump to line 606 because the condition on line 604 was never true
605 # Maybe lift (part) of this restriction.
606 self._error(
607 f'Cannot define rules for package "{package_name}" (at {definition_source.path}). It is an'
608 " auto-generated package."
609 )
610 binary_version: str | None = parsed.get(MK_BINARY_VERSION)
611 if binary_version is not None:
612 package_state.binary_version = (
613 package_state.substitution.substitute(
614 binary_version,
615 definition_source[MK_BINARY_VERSION].path,
616 )
617 )
618 search_dirs = parsed.get(MK_INSTALLATION_SEARCH_DIRS)
619 if search_dirs is not None: 619 ↛ 620line 619 didn't jump to line 620 because the condition on line 619 was never true
620 package_state.search_dirs = search_dirs
621 transformations = parsed.get(MK_TRANSFORMATIONS)
622 conffile_management = parsed.get(MK_CONFFILE_MANAGEMENT)
623 service_rules = parsed.get(MK_SERVICES)
624 if transformations:
625 package_state.transformations.extend(transformations)
626 if conffile_management:
627 package_state.dpkg_maintscript_helper_snippets.extend(
628 conffile_management
629 )
630 if service_rules: 630 ↛ 631line 630 didn't jump to line 631 because the condition on line 630 was never true
631 package_state.requested_service_rules.extend(service_rules)
632 self._build_rules = parsed_data.get("builds")
634 return self.build_manifest()
636 def _parse_manifest(self, fd: IO[bytes] | str) -> HighLevelManifest:
637 try:
638 data = MANIFEST_YAML.load(fd)
639 except YAMLError as e:
640 msg = str(e)
641 lines = msg.splitlines(keepends=True)
642 i = -1
643 for i, line in enumerate(lines):
644 # Avoid an irrelevant "how do configure the YAML parser" message, which the
645 # user cannot use.
646 if line.startswith("To suppress this check"):
647 break
648 if i > -1 and len(lines) > i + 1:
649 lines = lines[:i]
650 msg = "".join(lines)
651 msg = msg.rstrip()
652 msg += (
653 f"\n\nYou can use `yamllint -d relaxed {escape_shell(self.manifest_path)}` to validate"
654 " the YAML syntax. The yamllint tool also supports style rules for YAML documents"
655 " (such as indentation rules) in case that is of interest."
656 )
657 raise ManifestParseException(
658 f"Could not parse {self.manifest_path} as a YAML document: {msg}"
659 ) from e
660 self._mutable_yaml_manifest = MutableYAMLManifest(data)
662 return begin_parsing_context(
663 self._value_table,
664 self._source_package,
665 self._from_yaml_dict,
666 data,
667 )
669 def parse_manifest(
670 self,
671 *,
672 fd: IO[bytes] | str | None = None,
673 ) -> HighLevelManifest:
674 if fd is None: 674 ↛ 675line 674 didn't jump to line 675 because the condition on line 674 was never true
675 with open(self.manifest_path, "rb") as fd:
676 return self._parse_manifest(fd)
677 else:
678 return self._parse_manifest(fd)