Coverage for src/debputy/highlevel_manifest_parser.py: 73%
311 statements
« prev ^ index » next coverage.py v7.8.2, created at 2025-10-18 10:33 +0000
1import collections
2import contextlib
3from typing import (
4 Optional,
5 Dict,
6 List,
7 Any,
8 Union,
9 IO,
10 cast,
11 Tuple,
12)
13from collections.abc import Callable, Mapping, Iterator
15from debian.debian_support import DpkgArchTable
17from debputy.highlevel_manifest import (
18 HighLevelManifest,
19 PackageTransformationDefinition,
20 MutableYAMLManifest,
21)
22from debputy.maintscript_snippet import (
23 MaintscriptSnippet,
24 STD_CONTROL_SCRIPTS,
25 MaintscriptSnippetContainer,
26)
27from debputy.packages import BinaryPackage, SourcePackage
28from debputy.path_matcher import (
29 MatchRuleType,
30 ExactFileSystemPath,
31 MatchRule,
32)
33from debputy.substitution import Substitution
34from debputy.util import (
35 _normalize_path,
36 escape_shell,
37 assume_not_none,
38)
39from debputy.util import _warn, _info
40from ._deb_options_profiles import DebBuildOptionsAndProfiles
41from ._manifest_constants import (
42 MK_CONFFILE_MANAGEMENT,
43 MK_INSTALLATIONS,
44 MK_PACKAGES,
45 MK_INSTALLATION_SEARCH_DIRS,
46 MK_TRANSFORMATIONS,
47 MK_BINARY_VERSION,
48 MK_SERVICES,
49 MK_MANIFEST_REMOVE_DURING_CLEAN,
50)
51from .architecture_support import DpkgArchitectureBuildProcessValuesTable
52from .filesystem_scan import FSROOverlay
53from .installations import InstallRule, PPFInstallRule
54from .manifest_parser.base_types import (
55 BuildEnvironments,
56 BuildEnvironmentDefinition,
57 FileSystemMatchRule,
58)
59from .manifest_parser.exceptions import ManifestParseException
60from .manifest_parser.parser_data import ParserContextData
61from .manifest_parser.util import AttributePath
62from .packager_provided_files import detect_all_packager_provided_files
63from .plugin.api import VirtualPath
64from .plugin.api.feature_set import PluginProvidedFeatureSet
65from .plugin.api.impl_types import (
66 TP,
67 TTP,
68 DispatchingTableParser,
69 PackageContextData,
70)
71from debputy.plugin.api.parser_tables import OPARSER_MANIFEST_ROOT
72from .plugin.api.spec import DebputyIntegrationMode
73from debputy.plugins.debputy.build_system_rules import BuildRule
74from .yaml import YAMLError, MANIFEST_YAML
76try:
77 from Levenshtein import distance
78except ImportError:
80 def _detect_possible_typo(
81 _d,
82 _key,
83 _attribute_parent_path: AttributePath,
84 required: bool,
85 ) -> None:
86 if required:
87 _info(
88 "Install python3-levenshtein to have debputy try to detect typos in the manifest."
89 )
91else:
93 def _detect_possible_typo(
94 d,
95 key,
96 _attribute_parent_path: AttributePath,
97 _required: bool,
98 ) -> None:
99 k_len = len(key)
100 for actual_key in d:
101 if abs(k_len - len(actual_key)) > 2:
102 continue
103 d = distance(key, actual_key)
104 if d > 2:
105 continue
106 path = _attribute_parent_path.path
107 ref = f'at "{path}"' if path else "at the manifest root level"
108 _warn(
109 f'Possible typo: The key "{actual_key}" should probably have been "{key}" {ref}'
110 )
113def _per_package_subst_variables(
114 p: BinaryPackage,
115 *,
116 name: str | None = None,
117) -> dict[str, str]:
118 return {
119 "PACKAGE": name if name is not None else p.name,
120 }
123class HighLevelManifestParser(ParserContextData):
124 def __init__(
125 self,
126 manifest_path: str,
127 source_package: SourcePackage,
128 binary_packages: Mapping[str, BinaryPackage],
129 substitution: Substitution,
130 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
131 dpkg_arch_query_table: DpkgArchTable,
132 build_env: DebBuildOptionsAndProfiles,
133 plugin_provided_feature_set: PluginProvidedFeatureSet,
134 debputy_integration_mode: DebputyIntegrationMode,
135 *,
136 # Available for testing purposes only
137 debian_dir: str | VirtualPath = "./debian",
138 ):
139 self.manifest_path = manifest_path
140 self._source_package = source_package
141 self._binary_packages = binary_packages
142 self._mutable_yaml_manifest: MutableYAMLManifest | None = None
143 # In source context, some variables are known to be unresolvable. Record this, so
144 # we can give better error messages.
145 self._substitution = substitution
146 self._dpkg_architecture_variables = dpkg_architecture_variables
147 self._dpkg_arch_query_table = dpkg_arch_query_table
148 self._deb_options_and_profiles = build_env
149 self._package_state_stack: list[PackageTransformationDefinition] = []
150 self._plugin_provided_feature_set = plugin_provided_feature_set
151 self._debputy_integration_mode = debputy_integration_mode
152 self._declared_variables = {}
153 self._used_named_envs = set()
154 self._build_environments: BuildEnvironments | None = BuildEnvironments(
155 {},
156 None,
157 )
158 self._has_set_default_build_environment = False
159 self._read_build_environment = False
160 self._build_rules: list[BuildRule] | None = None
162 if isinstance(debian_dir, str): 162 ↛ 163line 162 didn't jump to line 163 because the condition on line 162 was never true
163 debian_dir = FSROOverlay.create_root_dir("debian", debian_dir)
165 self._debian_dir = debian_dir
167 # Delayed initialized; we rely on this delay to parse the variables.
168 self._all_package_states = None
170 self._install_rules: list[InstallRule] | None = None
171 self._remove_during_clean_rules: list[FileSystemMatchRule] = []
172 self._ownership_caches_loaded = False
173 self._used = False
175 def _ensure_package_states_is_initialized(self) -> None:
176 if self._all_package_states is not None:
177 return
178 substitution = self._substitution
179 binary_packages = self._binary_packages
180 assert self._all_package_states is None
182 self._all_package_states = {
183 n: PackageTransformationDefinition(
184 binary_package=p,
185 substitution=substitution.with_extra_substitutions(
186 **_per_package_subst_variables(p)
187 ),
188 is_auto_generated_package=False,
189 maintscript_snippets=collections.defaultdict(
190 MaintscriptSnippetContainer
191 ),
192 )
193 for n, p in binary_packages.items()
194 }
195 for n, p in binary_packages.items():
196 dbgsym_name = f"{n}-dbgsym"
197 if dbgsym_name in self._all_package_states: 197 ↛ 198line 197 didn't jump to line 198 because the condition on line 197 was never true
198 continue
199 self._all_package_states[dbgsym_name] = PackageTransformationDefinition(
200 binary_package=p,
201 substitution=substitution.with_extra_substitutions(
202 **_per_package_subst_variables(p, name=dbgsym_name)
203 ),
204 is_auto_generated_package=True,
205 maintscript_snippets=collections.defaultdict(
206 MaintscriptSnippetContainer
207 ),
208 )
210 @property
211 def binary_packages(self) -> Mapping[str, BinaryPackage]:
212 return self._binary_packages
214 @property
215 def _package_states(self) -> Mapping[str, PackageTransformationDefinition]:
216 assert self._all_package_states is not None
217 return self._all_package_states
219 @property
220 def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable:
221 return self._dpkg_architecture_variables
223 @property
224 def dpkg_arch_query_table(self) -> DpkgArchTable:
225 return self._dpkg_arch_query_table
227 @property
228 def deb_options_and_profiles(self) -> DebBuildOptionsAndProfiles:
229 return self._deb_options_and_profiles
231 def _self_check(self) -> None:
232 unused_envs = (
233 self._build_environments.environments.keys() - self._used_named_envs
234 )
235 if unused_envs:
236 unused_env_names = ", ".join(unused_envs)
237 raise ManifestParseException(
238 f"The following named environments were never referenced: {unused_env_names}"
239 )
241 def build_manifest(self) -> HighLevelManifest:
242 self._self_check()
243 if self._used: 243 ↛ 244line 243 didn't jump to line 244 because the condition on line 243 was never true
244 raise TypeError("build_manifest can only be called once!")
245 self._used = True
246 self._ensure_package_states_is_initialized()
247 for var, attribute_path in self._declared_variables.items():
248 if not self.substitution.is_used(var):
249 raise ManifestParseException(
250 f'The variable "{var}" is unused. Either use it or remove it.'
251 f" The variable was declared at {attribute_path.path_key_lc}."
252 )
253 if isinstance(self, YAMLManifestParser) and self._mutable_yaml_manifest is None:
254 self._mutable_yaml_manifest = MutableYAMLManifest.empty_manifest()
255 all_packager_provided_files = detect_all_packager_provided_files(
256 self._plugin_provided_feature_set,
257 self._debian_dir,
258 self.binary_packages,
259 )
261 for package in self._package_states:
262 with self.binary_package_context(package) as context:
263 if not context.is_auto_generated_package:
264 ppf_result = all_packager_provided_files[package]
265 if ppf_result.auto_installable: 265 ↛ 266line 265 didn't jump to line 266 because the condition on line 265 was never true
266 context.install_rules.append(
267 PPFInstallRule(
268 context.binary_package,
269 context.substitution,
270 ppf_result.auto_installable,
271 )
272 )
273 context.reserved_packager_provided_files.update(
274 ppf_result.reserved_only
275 )
276 self._transform_dpkg_maintscript_helpers_to_snippets()
277 build_environments = self.build_environments()
278 assert build_environments is not None
280 return HighLevelManifest(
281 self.manifest_path,
282 self._mutable_yaml_manifest,
283 self._remove_during_clean_rules,
284 self._install_rules,
285 self._source_package,
286 self.binary_packages,
287 self.substitution,
288 self._package_states,
289 self._dpkg_architecture_variables,
290 self._dpkg_arch_query_table,
291 self._deb_options_and_profiles,
292 build_environments,
293 self._build_rules,
294 self._plugin_provided_feature_set,
295 self._debian_dir,
296 )
298 @contextlib.contextmanager
299 def binary_package_context(
300 self, package_name: str
301 ) -> Iterator[PackageTransformationDefinition]:
302 if package_name not in self._package_states:
303 self._error(
304 f'The package "{package_name}" is not present in the debian/control file (could not find'
305 f' "Package: {package_name}" in a binary stanza) nor is it a -dbgsym package for one'
306 " for a package in debian/control."
307 )
308 package_state = self._package_states[package_name]
309 self._package_state_stack.append(package_state)
310 ps_len = len(self._package_state_stack)
311 yield package_state
312 if ps_len != len(self._package_state_stack): 312 ↛ 313line 312 didn't jump to line 313 because the condition on line 312 was never true
313 raise RuntimeError("Internal error: Unbalanced stack manipulation detected")
314 self._package_state_stack.pop()
316 def dispatch_parser_table_for(self, rule_type: TTP) -> DispatchingTableParser[TP]:
317 t = self._plugin_provided_feature_set.manifest_parser_generator.dispatch_parser_table_for(
318 rule_type
319 )
320 if t is None:
321 raise AssertionError(
322 f"Internal error: No dispatching parser for {rule_type.__name__}"
323 )
324 return t
326 @property
327 def substitution(self) -> Substitution:
328 if self._package_state_stack:
329 return self._package_state_stack[-1].substitution
330 return self._substitution
332 def add_extra_substitution_variables(
333 self,
334 **extra_substitutions: tuple[str, AttributePath],
335 ) -> Substitution:
336 if self._package_state_stack or self._all_package_states is not None: 336 ↛ 341line 336 didn't jump to line 341 because the condition on line 336 was never true
337 # For one, it would not "bubble up" correctly when added to the lowest stack.
338 # And if it is not added to the lowest stack, then you get errors about it being
339 # unknown as soon as you leave the stack (which is weird for the user when
340 # the variable is something known, sometimes not)
341 raise RuntimeError("Cannot use add_extra_substitution from this state")
342 for key, (_, path) in extra_substitutions.items():
343 self._declared_variables[key] = path
344 self._substitution = self._substitution.with_extra_substitutions(
345 **{k: v[0] for k, v in extra_substitutions.items()}
346 )
347 return self._substitution
349 @property
350 def current_binary_package_state(self) -> PackageTransformationDefinition:
351 if not self._package_state_stack: 351 ↛ 352line 351 didn't jump to line 352 because the condition on line 351 was never true
352 raise RuntimeError("Invalid state: Not in a binary package context")
353 return self._package_state_stack[-1]
355 @property
356 def is_in_binary_package_state(self) -> bool:
357 return bool(self._package_state_stack)
359 @property
360 def debputy_integration_mode(self) -> DebputyIntegrationMode:
361 return self._debputy_integration_mode
363 @debputy_integration_mode.setter
364 def debputy_integration_mode(self, new_value: DebputyIntegrationMode) -> None:
365 self._debputy_integration_mode = new_value
367 def _register_build_environment(
368 self,
369 name: str | None,
370 build_environment: BuildEnvironmentDefinition,
371 attribute_path: AttributePath,
372 is_default: bool = False,
373 ) -> None:
374 assert not self._read_build_environment
376 # TODO: Reference the paths of the original environments for the error messages where that is relevant.
377 if is_default:
378 if self._has_set_default_build_environment: 378 ↛ 379line 378 didn't jump to line 379 because the condition on line 378 was never true
379 raise ManifestParseException(
380 f"There cannot be multiple default environments and"
381 f" therefore {attribute_path.path} cannot be a default environment"
382 )
383 self._has_set_default_build_environment = True
384 self._build_environments.default_environment = build_environment
385 if name is None: 385 ↛ 394line 385 didn't jump to line 394 because the condition on line 385 was always true
386 return
387 elif name is None: 387 ↛ 388line 387 didn't jump to line 388 because the condition on line 387 was never true
388 raise ManifestParseException(
389 f"Useless environment defined at {attribute_path.path}. It is neither the"
390 " default environment nor does it have a name (so no rules can reference it"
391 " explicitly)"
392 )
394 if name in self._build_environments.environments: 394 ↛ 395line 394 didn't jump to line 395 because the condition on line 394 was never true
395 raise ManifestParseException(
396 f'The environment defined at {attribute_path.path} reuse the name "{name}".'
397 " The environment name must be unique."
398 )
399 self._build_environments.environments[name] = build_environment
401 def resolve_build_environment(
402 self,
403 name: str | None,
404 attribute_path: AttributePath,
405 ) -> BuildEnvironmentDefinition:
406 if name is None:
407 return self.build_environments().default_environment
408 try:
409 env = self.build_environments().environments[name]
410 except KeyError:
411 raise ManifestParseException(
412 f'The environment "{name}" requested at {attribute_path.path} was not'
413 f" defined in the `build-environments`"
414 )
415 else:
416 self._used_named_envs.add(name)
417 return env
419 def build_environments(self) -> BuildEnvironments:
420 v = self._build_environments
421 if (
422 not self._read_build_environment
423 and not self._build_environments.environments
424 and self._build_environments.default_environment is None
425 ):
426 self._build_environments.default_environment = BuildEnvironmentDefinition()
427 self._read_build_environment = True
428 return v
430 def _transform_dpkg_maintscript_helpers_to_snippets(self) -> None:
431 package_state = self.current_binary_package_state
432 for dmh in package_state.dpkg_maintscript_helper_snippets:
433 snippet = MaintscriptSnippet(
434 definition_source=dmh.definition_source,
435 snippet=f'dpkg-maintscript-helper {escape_shell(*dmh.cmdline)} -- "$@"\n',
436 )
437 for script in STD_CONTROL_SCRIPTS:
438 package_state.maintscript_snippets[script].append(snippet)
440 def normalize_path(
441 self,
442 path: str,
443 definition_source: AttributePath,
444 *,
445 allow_root_dir_match: bool = False,
446 ) -> ExactFileSystemPath:
447 try:
448 normalized = _normalize_path(path)
449 except ValueError:
450 self._error(
451 f'The path "{path}" provided in {definition_source.path} should be relative to the root of the'
452 ' package and not use any ".." or "." segments.'
453 )
454 if normalized == "." and not allow_root_dir_match:
455 self._error(
456 "Manifests must not change the root directory of the deb file. Please correct"
457 f' "{definition_source.path}" (path: "{path}) in {self.manifest_path}'
458 )
459 return ExactFileSystemPath(
460 self.substitution.substitute(normalized, definition_source.path)
461 )
463 def parse_path_or_glob(
464 self,
465 path_or_glob: str,
466 definition_source: AttributePath,
467 ) -> MatchRule:
468 match_rule = MatchRule.from_path_or_glob(
469 path_or_glob, definition_source.path, substitution=self.substitution
470 )
471 # NB: "." and "/" will be translated to MATCH_ANYTHING by MatchRule.from_path_or_glob,
472 # so there is no need to check for an exact match on "." like in normalize_path.
473 if match_rule.rule_type == MatchRuleType.MATCH_ANYTHING:
474 self._error(
475 f'The chosen match rule "{path_or_glob}" matches everything (including the deb root directory).'
476 f' Please correct "{definition_source.path}" (path: "{path_or_glob}) in {self.manifest_path} to'
477 f' something that matches "less" than everything.'
478 )
479 return match_rule
481 def parse_manifest(self) -> HighLevelManifest:
482 raise NotImplementedError
485class YAMLManifestParser(HighLevelManifestParser):
486 def _optional_key(
487 self,
488 d: Mapping[str, Any],
489 key: str,
490 attribute_parent_path: AttributePath,
491 expected_type=None,
492 default_value=None,
493 ):
494 v = d.get(key)
495 if v is None:
496 _detect_possible_typo(d, key, attribute_parent_path, False)
497 return default_value
498 if expected_type is not None:
499 return self._ensure_value_is_type(
500 v, expected_type, key, attribute_parent_path
501 )
502 return v
504 def _required_key(
505 self,
506 d: Mapping[str, Any],
507 key: str,
508 attribute_parent_path: AttributePath,
509 expected_type=None,
510 extra: str | Callable[[], str] | None = None,
511 ):
512 v = d.get(key)
513 if v is None:
514 _detect_possible_typo(d, key, attribute_parent_path, True)
515 if extra is not None:
516 msg = extra if isinstance(extra, str) else extra()
517 extra_info = " " + msg
518 else:
519 extra_info = ""
520 self._error(
521 f'Missing required key {key} at {attribute_parent_path.path} in manifest "{self.manifest_path}.'
522 f"{extra_info}"
523 )
525 if expected_type is not None:
526 return self._ensure_value_is_type(
527 v, expected_type, key, attribute_parent_path
528 )
529 return v
531 def _ensure_value_is_type(
532 self,
533 v,
534 t,
535 key: str | int | AttributePath,
536 attribute_parent_path: AttributePath | None,
537 ):
538 if v is None:
539 return None
540 if not isinstance(v, t):
541 if isinstance(t, tuple):
542 t_msg = "one of: " + ", ".join(x.__name__ for x in t)
543 else:
544 t_msg = f"a {t.__name__}"
545 key_path = (
546 key.path
547 if isinstance(key, AttributePath)
548 else assume_not_none(attribute_parent_path)[key].path
549 )
550 self._error(
551 f'The key {key_path} must be {t_msg} in manifest "{self.manifest_path}"'
552 )
553 return v
555 def from_yaml_dict(self, yaml_data: object) -> "HighLevelManifest":
556 attribute_path = AttributePath.root_path(yaml_data)
557 parser_generator = self._plugin_provided_feature_set.manifest_parser_generator
558 dispatchable_object_parsers = parser_generator.dispatchable_object_parsers
559 manifest_root_parser = dispatchable_object_parsers[OPARSER_MANIFEST_ROOT]
560 parsed_data = manifest_root_parser.parse_input(
561 yaml_data,
562 attribute_path,
563 parser_context=self,
564 )
566 packages_dict: Mapping[str, PackageContextData[Mapping[str, Any]]] = cast(
567 "Mapping[str, PackageContextData[Mapping[str, Any]]]",
568 parsed_data.get("packages", {}),
569 )
570 self._remove_during_clean_rules = parsed_data.get(
571 MK_MANIFEST_REMOVE_DURING_CLEAN, []
572 )
573 install_rules = parsed_data.get(MK_INSTALLATIONS)
574 if install_rules:
575 self._install_rules = install_rules
576 packages_parent_path = attribute_path[MK_PACKAGES]
577 for package_name_raw, pcd in packages_dict.items():
578 definition_source = packages_parent_path[package_name_raw]
579 package_name = pcd.resolved_package_name
580 parsed = pcd.value
582 package_state: PackageTransformationDefinition
583 with self.binary_package_context(package_name) as package_state:
584 if package_state.is_auto_generated_package: 584 ↛ 586line 584 didn't jump to line 586 because the condition on line 584 was never true
585 # Maybe lift (part) of this restriction.
586 self._error(
587 f'Cannot define rules for package "{package_name}" (at {definition_source.path}). It is an'
588 " auto-generated package."
589 )
590 binary_version = parsed.get(MK_BINARY_VERSION)
591 if binary_version is not None:
592 package_state.binary_version = (
593 package_state.substitution.substitute(
594 binary_version,
595 definition_source[MK_BINARY_VERSION].path,
596 )
597 )
598 search_dirs = parsed.get(MK_INSTALLATION_SEARCH_DIRS)
599 if search_dirs is not None: 599 ↛ 600line 599 didn't jump to line 600 because the condition on line 599 was never true
600 package_state.search_dirs = search_dirs
601 transformations = parsed.get(MK_TRANSFORMATIONS)
602 conffile_management = parsed.get(MK_CONFFILE_MANAGEMENT)
603 service_rules = parsed.get(MK_SERVICES)
604 if transformations:
605 package_state.transformations.extend(transformations)
606 if conffile_management:
607 package_state.dpkg_maintscript_helper_snippets.extend(
608 conffile_management
609 )
610 if service_rules: 610 ↛ 611line 610 didn't jump to line 611 because the condition on line 610 was never true
611 package_state.requested_service_rules.extend(service_rules)
612 self._build_rules = parsed_data.get("builds")
614 return self.build_manifest()
616 def _parse_manifest(self, fd: IO[bytes] | str) -> HighLevelManifest:
617 try:
618 data = MANIFEST_YAML.load(fd)
619 except YAMLError as e:
620 msg = str(e)
621 lines = msg.splitlines(keepends=True)
622 i = -1
623 for i, line in enumerate(lines):
624 # Avoid an irrelevant "how do configure the YAML parser" message, which the
625 # user cannot use.
626 if line.startswith("To suppress this check"):
627 break
628 if i > -1 and len(lines) > i + 1:
629 lines = lines[:i]
630 msg = "".join(lines)
631 msg = msg.rstrip()
632 msg += (
633 f"\n\nYou can use `yamllint -d relaxed {escape_shell(self.manifest_path)}` to validate"
634 " the YAML syntax. The yamllint tool also supports style rules for YAML documents"
635 " (such as indentation rules) in case that is of interest."
636 )
637 raise ManifestParseException(
638 f"Could not parse {self.manifest_path} as a YAML document: {msg}"
639 ) from e
640 self._mutable_yaml_manifest = MutableYAMLManifest(data)
641 return self.from_yaml_dict(data)
643 def parse_manifest(
644 self,
645 *,
646 fd: IO[bytes] | str | None = None,
647 ) -> HighLevelManifest:
648 if fd is None: 648 ↛ 649line 648 didn't jump to line 649 because the condition on line 648 was never true
649 with open(self.manifest_path, "rb") as fd:
650 return self._parse_manifest(fd)
651 else:
652 return self._parse_manifest(fd)