Coverage for src/debputy/highlevel_manifest_parser.py: 73%
310 statements
« prev ^ index » next coverage.py v7.8.2, created at 2025-09-07 09:27 +0000
1import collections
2import contextlib
3from typing import (
4 Optional,
5 Dict,
6 Callable,
7 List,
8 Any,
9 Union,
10 Mapping,
11 IO,
12 Iterator,
13 cast,
14 Tuple,
15)
17from debian.debian_support import DpkgArchTable
19from debputy.highlevel_manifest import (
20 HighLevelManifest,
21 PackageTransformationDefinition,
22 MutableYAMLManifest,
23)
24from debputy.maintscript_snippet import (
25 MaintscriptSnippet,
26 STD_CONTROL_SCRIPTS,
27 MaintscriptSnippetContainer,
28)
29from debputy.packages import BinaryPackage, SourcePackage
30from debputy.path_matcher import (
31 MatchRuleType,
32 ExactFileSystemPath,
33 MatchRule,
34)
35from debputy.substitution import Substitution
36from debputy.util import (
37 _normalize_path,
38 escape_shell,
39 assume_not_none,
40)
41from debputy.util import _warn, _info
42from ._deb_options_profiles import DebBuildOptionsAndProfiles
43from ._manifest_constants import (
44 MK_CONFFILE_MANAGEMENT,
45 MK_INSTALLATIONS,
46 MK_PACKAGES,
47 MK_INSTALLATION_SEARCH_DIRS,
48 MK_TRANSFORMATIONS,
49 MK_BINARY_VERSION,
50 MK_SERVICES,
51 MK_MANIFEST_REMOVE_DURING_CLEAN,
52)
53from .architecture_support import DpkgArchitectureBuildProcessValuesTable
54from .filesystem_scan import FSROOverlay
55from .installations import InstallRule, PPFInstallRule
56from .manifest_parser.base_types import (
57 BuildEnvironments,
58 BuildEnvironmentDefinition,
59 FileSystemMatchRule,
60)
61from .manifest_parser.exceptions import ManifestParseException
62from .manifest_parser.parser_data import ParserContextData
63from .manifest_parser.util import AttributePath
64from .packager_provided_files import detect_all_packager_provided_files
65from .plugin.api import VirtualPath
66from .plugin.api.feature_set import PluginProvidedFeatureSet
67from .plugin.api.impl_types import (
68 TP,
69 TTP,
70 DispatchingTableParser,
71 PackageContextData,
72)
73from debputy.plugin.api.parser_tables import OPARSER_MANIFEST_ROOT
74from .plugin.api.spec import DebputyIntegrationMode
75from debputy.plugins.debputy.build_system_rules import BuildRule
76from .yaml import YAMLError, MANIFEST_YAML
78try:
79 from Levenshtein import distance
80except ImportError:
82 def _detect_possible_typo(
83 _d,
84 _key,
85 _attribute_parent_path: AttributePath,
86 required: bool,
87 ) -> None:
88 if required:
89 _info(
90 "Install python3-levenshtein to have debputy try to detect typos in the manifest."
91 )
93else:
95 def _detect_possible_typo(
96 d,
97 key,
98 _attribute_parent_path: AttributePath,
99 _required: bool,
100 ) -> None:
101 k_len = len(key)
102 for actual_key in d:
103 if abs(k_len - len(actual_key)) > 2:
104 continue
105 d = distance(key, actual_key)
106 if d > 2:
107 continue
108 path = _attribute_parent_path.path
109 ref = f'at "{path}"' if path else "at the manifest root level"
110 _warn(
111 f'Possible typo: The key "{actual_key}" should probably have been "{key}" {ref}'
112 )
115def _per_package_subst_variables(
116 p: BinaryPackage,
117 *,
118 name: Optional[str] = None,
119) -> Dict[str, str]:
120 return {
121 "PACKAGE": name if name is not None else p.name,
122 }
125class HighLevelManifestParser(ParserContextData):
126 def __init__(
127 self,
128 manifest_path: str,
129 source_package: SourcePackage,
130 binary_packages: Mapping[str, BinaryPackage],
131 substitution: Substitution,
132 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
133 dpkg_arch_query_table: DpkgArchTable,
134 build_env: DebBuildOptionsAndProfiles,
135 plugin_provided_feature_set: PluginProvidedFeatureSet,
136 debputy_integration_mode: DebputyIntegrationMode,
137 *,
138 # Available for testing purposes only
139 debian_dir: Union[str, VirtualPath] = "./debian",
140 ):
141 self.manifest_path = manifest_path
142 self._source_package = source_package
143 self._binary_packages = binary_packages
144 self._mutable_yaml_manifest: Optional[MutableYAMLManifest] = None
145 # In source context, some variables are known to be unresolvable. Record this, so
146 # we can give better error messages.
147 self._substitution = substitution
148 self._dpkg_architecture_variables = dpkg_architecture_variables
149 self._dpkg_arch_query_table = dpkg_arch_query_table
150 self._deb_options_and_profiles = build_env
151 self._package_state_stack: List[PackageTransformationDefinition] = []
152 self._plugin_provided_feature_set = plugin_provided_feature_set
153 self._debputy_integration_mode = debputy_integration_mode
154 self._declared_variables = {}
155 self._used_named_envs = set()
156 self._build_environments: Optional[BuildEnvironments] = BuildEnvironments(
157 {},
158 None,
159 )
160 self._has_set_default_build_environment = False
161 self._read_build_environment = False
162 self._build_rules: Optional[List[BuildRule]] = None
164 if isinstance(debian_dir, str): 164 ↛ 165line 164 didn't jump to line 165 because the condition on line 164 was never true
165 debian_dir = FSROOverlay.create_root_dir("debian", debian_dir)
167 self._debian_dir = debian_dir
169 # Delayed initialized; we rely on this delay to parse the variables.
170 self._all_package_states = None
172 self._install_rules: Optional[List[InstallRule]] = None
173 self._remove_during_clean_rules: List[FileSystemMatchRule] = []
174 self._ownership_caches_loaded = False
175 self._used = False
177 def _ensure_package_states_is_initialized(self) -> None:
178 if self._all_package_states is not None:
179 return
180 substitution = self._substitution
181 binary_packages = self._binary_packages
182 assert self._all_package_states is None
184 self._all_package_states = {
185 n: PackageTransformationDefinition(
186 binary_package=p,
187 substitution=substitution.with_extra_substitutions(
188 **_per_package_subst_variables(p)
189 ),
190 is_auto_generated_package=False,
191 maintscript_snippets=collections.defaultdict(
192 MaintscriptSnippetContainer
193 ),
194 )
195 for n, p in binary_packages.items()
196 }
197 for n, p in binary_packages.items():
198 dbgsym_name = f"{n}-dbgsym"
199 if dbgsym_name in self._all_package_states: 199 ↛ 200line 199 didn't jump to line 200 because the condition on line 199 was never true
200 continue
201 self._all_package_states[dbgsym_name] = PackageTransformationDefinition(
202 binary_package=p,
203 substitution=substitution.with_extra_substitutions(
204 **_per_package_subst_variables(p, name=dbgsym_name)
205 ),
206 is_auto_generated_package=True,
207 maintscript_snippets=collections.defaultdict(
208 MaintscriptSnippetContainer
209 ),
210 )
212 @property
213 def binary_packages(self) -> Mapping[str, BinaryPackage]:
214 return self._binary_packages
216 @property
217 def _package_states(self) -> Mapping[str, PackageTransformationDefinition]:
218 assert self._all_package_states is not None
219 return self._all_package_states
221 @property
222 def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable:
223 return self._dpkg_architecture_variables
225 @property
226 def dpkg_arch_query_table(self) -> DpkgArchTable:
227 return self._dpkg_arch_query_table
229 @property
230 def deb_options_and_profiles(self) -> DebBuildOptionsAndProfiles:
231 return self._deb_options_and_profiles
233 def _self_check(self) -> None:
234 unused_envs = (
235 self._build_environments.environments.keys() - self._used_named_envs
236 )
237 if unused_envs:
238 unused_env_names = ", ".join(unused_envs)
239 raise ManifestParseException(
240 f"The following named environments were never referenced: {unused_env_names}"
241 )
243 def build_manifest(self) -> HighLevelManifest:
244 self._self_check()
245 if self._used: 245 ↛ 246line 245 didn't jump to line 246 because the condition on line 245 was never true
246 raise TypeError("build_manifest can only be called once!")
247 self._used = True
248 self._ensure_package_states_is_initialized()
249 for var, attribute_path in self._declared_variables.items():
250 if not self.substitution.is_used(var):
251 raise ManifestParseException(
252 f'The variable "{var}" is unused. Either use it or remove it.'
253 f" The variable was declared at {attribute_path.path_key_lc}."
254 )
255 if isinstance(self, YAMLManifestParser) and self._mutable_yaml_manifest is None:
256 self._mutable_yaml_manifest = MutableYAMLManifest.empty_manifest()
257 all_packager_provided_files = detect_all_packager_provided_files(
258 self._plugin_provided_feature_set,
259 self._debian_dir,
260 self.binary_packages,
261 )
263 for package in self._package_states:
264 with self.binary_package_context(package) as context:
265 if not context.is_auto_generated_package:
266 ppf_result = all_packager_provided_files[package]
267 if ppf_result.auto_installable: 267 ↛ 268line 267 didn't jump to line 268 because the condition on line 267 was never true
268 context.install_rules.append(
269 PPFInstallRule(
270 context.binary_package,
271 context.substitution,
272 ppf_result.auto_installable,
273 )
274 )
275 context.reserved_packager_provided_files.update(
276 ppf_result.reserved_only
277 )
278 self._transform_dpkg_maintscript_helpers_to_snippets()
279 build_environments = self.build_environments()
280 assert build_environments is not None
282 return HighLevelManifest(
283 self.manifest_path,
284 self._mutable_yaml_manifest,
285 self._remove_during_clean_rules,
286 self._install_rules,
287 self._source_package,
288 self.binary_packages,
289 self.substitution,
290 self._package_states,
291 self._dpkg_architecture_variables,
292 self._dpkg_arch_query_table,
293 self._deb_options_and_profiles,
294 build_environments,
295 self._build_rules,
296 self._plugin_provided_feature_set,
297 self._debian_dir,
298 )
300 @contextlib.contextmanager
301 def binary_package_context(
302 self, package_name: str
303 ) -> Iterator[PackageTransformationDefinition]:
304 if package_name not in self._package_states:
305 self._error(
306 f'The package "{package_name}" is not present in the debian/control file (could not find'
307 f' "Package: {package_name}" in a binary stanza) nor is it a -dbgsym package for one'
308 " for a package in debian/control."
309 )
310 package_state = self._package_states[package_name]
311 self._package_state_stack.append(package_state)
312 ps_len = len(self._package_state_stack)
313 yield package_state
314 if ps_len != len(self._package_state_stack): 314 ↛ 315line 314 didn't jump to line 315 because the condition on line 314 was never true
315 raise RuntimeError("Internal error: Unbalanced stack manipulation detected")
316 self._package_state_stack.pop()
318 def dispatch_parser_table_for(self, rule_type: TTP) -> DispatchingTableParser[TP]:
319 t = self._plugin_provided_feature_set.manifest_parser_generator.dispatch_parser_table_for(
320 rule_type
321 )
322 if t is None:
323 raise AssertionError(
324 f"Internal error: No dispatching parser for {rule_type.__name__}"
325 )
326 return t
328 @property
329 def substitution(self) -> Substitution:
330 if self._package_state_stack:
331 return self._package_state_stack[-1].substitution
332 return self._substitution
334 def add_extra_substitution_variables(
335 self,
336 **extra_substitutions: Tuple[str, AttributePath],
337 ) -> Substitution:
338 if self._package_state_stack or self._all_package_states is not None: 338 ↛ 343line 338 didn't jump to line 343 because the condition on line 338 was never true
339 # For one, it would not "bubble up" correctly when added to the lowest stack.
340 # And if it is not added to the lowest stack, then you get errors about it being
341 # unknown as soon as you leave the stack (which is weird for the user when
342 # the variable is something known, sometimes not)
343 raise RuntimeError("Cannot use add_extra_substitution from this state")
344 for key, (_, path) in extra_substitutions.items():
345 self._declared_variables[key] = path
346 self._substitution = self._substitution.with_extra_substitutions(
347 **{k: v[0] for k, v in extra_substitutions.items()}
348 )
349 return self._substitution
351 @property
352 def current_binary_package_state(self) -> PackageTransformationDefinition:
353 if not self._package_state_stack: 353 ↛ 354line 353 didn't jump to line 354 because the condition on line 353 was never true
354 raise RuntimeError("Invalid state: Not in a binary package context")
355 return self._package_state_stack[-1]
357 @property
358 def is_in_binary_package_state(self) -> bool:
359 return bool(self._package_state_stack)
361 @property
362 def debputy_integration_mode(self) -> DebputyIntegrationMode:
363 return self._debputy_integration_mode
365 @debputy_integration_mode.setter
366 def debputy_integration_mode(self, new_value: DebputyIntegrationMode) -> None:
367 self._debputy_integration_mode = new_value
369 def _register_build_environment(
370 self,
371 name: Optional[str],
372 build_environment: BuildEnvironmentDefinition,
373 attribute_path: AttributePath,
374 is_default: bool = False,
375 ) -> None:
376 assert not self._read_build_environment
378 # TODO: Reference the paths of the original environments for the error messages where that is relevant.
379 if is_default:
380 if self._has_set_default_build_environment: 380 ↛ 381line 380 didn't jump to line 381 because the condition on line 380 was never true
381 raise ManifestParseException(
382 f"There cannot be multiple default environments and"
383 f" therefore {attribute_path.path} cannot be a default environment"
384 )
385 self._has_set_default_build_environment = True
386 self._build_environments.default_environment = build_environment
387 if name is None: 387 ↛ 396line 387 didn't jump to line 396 because the condition on line 387 was always true
388 return
389 elif name is None: 389 ↛ 390line 389 didn't jump to line 390 because the condition on line 389 was never true
390 raise ManifestParseException(
391 f"Useless environment defined at {attribute_path.path}. It is neither the"
392 " default environment nor does it have a name (so no rules can reference it"
393 " explicitly)"
394 )
396 if name in self._build_environments.environments: 396 ↛ 397line 396 didn't jump to line 397 because the condition on line 396 was never true
397 raise ManifestParseException(
398 f'The environment defined at {attribute_path.path} reuse the name "{name}".'
399 " The environment name must be unique."
400 )
401 self._build_environments.environments[name] = build_environment
403 def resolve_build_environment(
404 self,
405 name: Optional[str],
406 attribute_path: AttributePath,
407 ) -> BuildEnvironmentDefinition:
408 if name is None:
409 return self.build_environments().default_environment
410 try:
411 env = self.build_environments().environments[name]
412 except KeyError:
413 raise ManifestParseException(
414 f'The environment "{name}" requested at {attribute_path.path} was not'
415 f" defined in the `build-environments`"
416 )
417 else:
418 self._used_named_envs.add(name)
419 return env
421 def build_environments(self) -> BuildEnvironments:
422 v = self._build_environments
423 if (
424 not self._read_build_environment
425 and not self._build_environments.environments
426 and self._build_environments.default_environment is None
427 ):
428 self._build_environments.default_environment = BuildEnvironmentDefinition()
429 self._read_build_environment = True
430 return v
432 def _transform_dpkg_maintscript_helpers_to_snippets(self) -> None:
433 package_state = self.current_binary_package_state
434 for dmh in package_state.dpkg_maintscript_helper_snippets:
435 snippet = MaintscriptSnippet(
436 definition_source=dmh.definition_source,
437 snippet=f'dpkg-maintscript-helper {escape_shell(*dmh.cmdline)} -- "$@"\n',
438 )
439 for script in STD_CONTROL_SCRIPTS:
440 package_state.maintscript_snippets[script].append(snippet)
442 def normalize_path(
443 self,
444 path: str,
445 definition_source: AttributePath,
446 *,
447 allow_root_dir_match: bool = False,
448 ) -> ExactFileSystemPath:
449 try:
450 normalized = _normalize_path(path)
451 except ValueError:
452 self._error(
453 f'The path "{path}" provided in {definition_source.path} should be relative to the root of the'
454 ' package and not use any ".." or "." segments.'
455 )
456 if normalized == "." and not allow_root_dir_match:
457 self._error(
458 "Manifests must not change the root directory of the deb file. Please correct"
459 f' "{definition_source.path}" (path: "{path}) in {self.manifest_path}'
460 )
461 return ExactFileSystemPath(
462 self.substitution.substitute(normalized, definition_source.path)
463 )
465 def parse_path_or_glob(
466 self,
467 path_or_glob: str,
468 definition_source: AttributePath,
469 ) -> MatchRule:
470 match_rule = MatchRule.from_path_or_glob(
471 path_or_glob, definition_source.path, substitution=self.substitution
472 )
473 # NB: "." and "/" will be translated to MATCH_ANYTHING by MatchRule.from_path_or_glob,
474 # so there is no need to check for an exact match on "." like in normalize_path.
475 if match_rule.rule_type == MatchRuleType.MATCH_ANYTHING:
476 self._error(
477 f'The chosen match rule "{path_or_glob}" matches everything (including the deb root directory).'
478 f' Please correct "{definition_source.path}" (path: "{path_or_glob}) in {self.manifest_path} to'
479 f' something that matches "less" than everything.'
480 )
481 return match_rule
483 def parse_manifest(self) -> HighLevelManifest:
484 raise NotImplementedError
487class YAMLManifestParser(HighLevelManifestParser):
488 def _optional_key(
489 self,
490 d: Mapping[str, Any],
491 key: str,
492 attribute_parent_path: AttributePath,
493 expected_type=None,
494 default_value=None,
495 ):
496 v = d.get(key)
497 if v is None:
498 _detect_possible_typo(d, key, attribute_parent_path, False)
499 return default_value
500 if expected_type is not None:
501 return self._ensure_value_is_type(
502 v, expected_type, key, attribute_parent_path
503 )
504 return v
506 def _required_key(
507 self,
508 d: Mapping[str, Any],
509 key: str,
510 attribute_parent_path: AttributePath,
511 expected_type=None,
512 extra: Optional[Union[str, Callable[[], str]]] = None,
513 ):
514 v = d.get(key)
515 if v is None:
516 _detect_possible_typo(d, key, attribute_parent_path, True)
517 if extra is not None:
518 msg = extra if isinstance(extra, str) else extra()
519 extra_info = " " + msg
520 else:
521 extra_info = ""
522 self._error(
523 f'Missing required key {key} at {attribute_parent_path.path} in manifest "{self.manifest_path}.'
524 f"{extra_info}"
525 )
527 if expected_type is not None:
528 return self._ensure_value_is_type(
529 v, expected_type, key, attribute_parent_path
530 )
531 return v
533 def _ensure_value_is_type(
534 self,
535 v,
536 t,
537 key: Union[str, int, AttributePath],
538 attribute_parent_path: Optional[AttributePath],
539 ):
540 if v is None:
541 return None
542 if not isinstance(v, t):
543 if isinstance(t, tuple):
544 t_msg = "one of: " + ", ".join(x.__name__ for x in t)
545 else:
546 t_msg = f"a {t.__name__}"
547 key_path = (
548 key.path
549 if isinstance(key, AttributePath)
550 else assume_not_none(attribute_parent_path)[key].path
551 )
552 self._error(
553 f'The key {key_path} must be {t_msg} in manifest "{self.manifest_path}"'
554 )
555 return v
557 def from_yaml_dict(self, yaml_data: object) -> "HighLevelManifest":
558 attribute_path = AttributePath.root_path(yaml_data)
559 parser_generator = self._plugin_provided_feature_set.manifest_parser_generator
560 dispatchable_object_parsers = parser_generator.dispatchable_object_parsers
561 manifest_root_parser = dispatchable_object_parsers[OPARSER_MANIFEST_ROOT]
562 parsed_data = manifest_root_parser.parse_input(
563 yaml_data,
564 attribute_path,
565 parser_context=self,
566 )
568 packages_dict: Mapping[str, PackageContextData[Mapping[str, Any]]] = cast(
569 "Mapping[str, PackageContextData[Mapping[str, Any]]]",
570 parsed_data.get("packages", {}),
571 )
572 self._remove_during_clean_rules = parsed_data.get(
573 MK_MANIFEST_REMOVE_DURING_CLEAN, []
574 )
575 install_rules = parsed_data.get(MK_INSTALLATIONS)
576 if install_rules:
577 self._install_rules = install_rules
578 packages_parent_path = attribute_path[MK_PACKAGES]
579 for package_name_raw, pcd in packages_dict.items():
580 definition_source = packages_parent_path[package_name_raw]
581 package_name = pcd.resolved_package_name
582 parsed = pcd.value
584 package_state: PackageTransformationDefinition
585 with self.binary_package_context(package_name) as package_state:
586 if package_state.is_auto_generated_package: 586 ↛ 588line 586 didn't jump to line 588 because the condition on line 586 was never true
587 # Maybe lift (part) of this restriction.
588 self._error(
589 f'Cannot define rules for package "{package_name}" (at {definition_source.path}). It is an'
590 " auto-generated package."
591 )
592 binary_version = parsed.get(MK_BINARY_VERSION)
593 if binary_version is not None:
594 package_state.binary_version = (
595 package_state.substitution.substitute(
596 binary_version,
597 definition_source[MK_BINARY_VERSION].path,
598 )
599 )
600 search_dirs = parsed.get(MK_INSTALLATION_SEARCH_DIRS)
601 if search_dirs is not None: 601 ↛ 602line 601 didn't jump to line 602 because the condition on line 601 was never true
602 package_state.search_dirs = search_dirs
603 transformations = parsed.get(MK_TRANSFORMATIONS)
604 conffile_management = parsed.get(MK_CONFFILE_MANAGEMENT)
605 service_rules = parsed.get(MK_SERVICES)
606 if transformations:
607 package_state.transformations.extend(transformations)
608 if conffile_management:
609 package_state.dpkg_maintscript_helper_snippets.extend(
610 conffile_management
611 )
612 if service_rules: 612 ↛ 613line 612 didn't jump to line 613 because the condition on line 612 was never true
613 package_state.requested_service_rules.extend(service_rules)
614 self._build_rules = parsed_data.get("builds")
616 return self.build_manifest()
618 def _parse_manifest(self, fd: Union[IO[bytes], str]) -> HighLevelManifest:
619 try:
620 data = MANIFEST_YAML.load(fd)
621 except YAMLError as e:
622 msg = str(e)
623 lines = msg.splitlines(keepends=True)
624 i = -1
625 for i, line in enumerate(lines):
626 # Avoid an irrelevant "how do configure the YAML parser" message, which the
627 # user cannot use.
628 if line.startswith("To suppress this check"):
629 break
630 if i > -1 and len(lines) > i + 1:
631 lines = lines[:i]
632 msg = "".join(lines)
633 msg = msg.rstrip()
634 msg += (
635 f"\n\nYou can use `yamllint -d relaxed {escape_shell(self.manifest_path)}` to validate"
636 " the YAML syntax. The yamllint tool also supports style rules for YAML documents"
637 " (such as indentation rules) in case that is of interest."
638 )
639 raise ManifestParseException(
640 f"Could not parse {self.manifest_path} as a YAML document: {msg}"
641 ) from e
642 self._mutable_yaml_manifest = MutableYAMLManifest(data)
643 return self.from_yaml_dict(data)
645 def parse_manifest(
646 self,
647 *,
648 fd: Optional[Union[IO[bytes], str]] = None,
649 ) -> HighLevelManifest:
650 if fd is None: 650 ↛ 651line 650 didn't jump to line 651 because the condition on line 650 was never true
651 with open(self.manifest_path, "rb") as fd:
652 return self._parse_manifest(fd)
653 else:
654 return self._parse_manifest(fd)