Coverage for src/debputy/manifest_parser/base_types.py: 73%
259 statements
« prev ^ index » next coverage.py v7.8.2, created at 2025-10-19 09:24 +0000
1import dataclasses
2import os
3import subprocess
4from functools import lru_cache
5from typing import (
6 Optional,
7 Union,
8 Literal,
9 Tuple,
10 TYPE_CHECKING,
11 Dict,
12 NotRequired,
13 FrozenSet,
14)
15from collections.abc import Sequence, Mapping, Iterable, MutableMapping
17from debputy.manifest_conditions import ManifestCondition
18from debputy.manifest_parser.exceptions import ManifestParseException
19from debputy.manifest_parser.tagging_types import DebputyParsedContent
20from debputy.manifest_parser.util import (
21 AttributePath,
22 _SymbolicModeSegment,
23 parse_symbolic_mode,
24)
25from debputy.path_matcher import MatchRule, ExactFileSystemPath
26from debputy.substitution import Substitution
27from debputy.util import (
28 _normalize_path,
29 _error,
30 _warn,
31 _debug_log,
32 _is_debug_log_enabled,
33)
35if TYPE_CHECKING:
36 from debputy.manifest_parser.parser_data import ParserContextData
39@dataclasses.dataclass(slots=True, frozen=True)
40class OwnershipDefinition:
41 entity_name: str
42 entity_id: int
45class DebputyParsedContentStandardConditional(DebputyParsedContent):
46 when: NotRequired[ManifestCondition]
49ROOT_DEFINITION = OwnershipDefinition("root", 0)
52BAD_OWNER_NAMES = {
53 "_apt", # All things owned by _apt are generated by apt after installation
54 "nogroup", # It is not supposed to own anything as it is an entity used for dropping permissions
55 "nobody", # It is not supposed to own anything as it is an entity used for dropping permissions
56}
57BAD_OWNER_IDS = {
58 65534, # ID of nobody / nogroup
59}
62def _parse_ownership(
63 v: str | int,
64 attribute_path: AttributePath,
65) -> tuple[str | None, int | None]:
66 if isinstance(v, str) and ":" in v: 66 ↛ 67line 66 didn't jump to line 67 because the condition on line 66 was never true
67 if v == ":":
68 raise ManifestParseException(
69 f'Invalid ownership value "{v}" at {attribute_path.path}: Ownership is redundant if it is ":"'
70 f" (blank name and blank id). Please provide non-default values or remove the definition."
71 )
72 entity_name: str | None
73 entity_id: int | None
74 entity_name, entity_id_str = v.split(":")
75 if entity_name == "":
76 entity_name = None
77 if entity_id_str != "":
78 entity_id = int(entity_id_str)
79 else:
80 entity_id = None
81 return entity_name, entity_id
83 if isinstance(v, int):
84 return None, v
85 if v.isdigit(): 85 ↛ 86line 85 didn't jump to line 86 because the condition on line 85 was never true
86 raise ManifestParseException(
87 f'Invalid ownership value "{v}" at {attribute_path.path}: The provided value "{v}" is a string (implying'
88 " name lookup), but it contains an integer (implying id lookup). Please use a regular int for id lookup"
89 f' (removing the quotes) or add a ":" in the end ("{v}:") as a disambiguation if you are *really* looking'
90 " for an entity with that name."
91 )
92 return v, None
95@lru_cache
96def _load_ownership_table_from_file(
97 name: Literal["passwd.master", "group.master"],
98) -> tuple[Mapping[str, OwnershipDefinition], Mapping[int, OwnershipDefinition]]:
99 filename = os.path.join("/usr/share/base-passwd", name)
100 name_table = {}
101 uid_table = {}
102 for owner_def in _read_ownership_def_from_base_password_template(filename):
103 # Could happen if base-passwd template has two users with the same ID. We assume this will not occur.
104 assert owner_def.entity_name not in name_table
105 assert owner_def.entity_id not in uid_table
106 name_table[owner_def.entity_name] = owner_def
107 uid_table[owner_def.entity_id] = owner_def
109 return name_table, uid_table
112def _read_ownership_def_from_base_password_template(
113 template_file: str,
114) -> Iterable[OwnershipDefinition]:
115 with open(template_file) as fd:
116 for line in fd:
117 entity_name, _star, entity_id, _remainder = line.split(":", 3)
118 if entity_id == "0" and entity_name == "root":
119 yield ROOT_DEFINITION
120 else:
121 yield OwnershipDefinition(entity_name, int(entity_id))
124class FileSystemMode:
125 @classmethod
126 def parse_filesystem_mode(
127 cls,
128 mode_raw: str,
129 attribute_path: AttributePath,
130 ) -> "FileSystemMode":
131 if mode_raw and mode_raw[0].isdigit():
132 return OctalMode.parse_filesystem_mode(mode_raw, attribute_path)
133 return SymbolicMode.parse_filesystem_mode(mode_raw, attribute_path)
135 def compute_mode(self, current_mode: int, is_dir: bool) -> int:
136 raise NotImplementedError
139@dataclasses.dataclass(slots=True, frozen=True)
140class SymbolicMode(FileSystemMode):
141 provided_mode: str
142 segments: Sequence[_SymbolicModeSegment]
144 @classmethod
145 def parse_filesystem_mode(
146 cls,
147 mode_raw: str,
148 attribute_path: AttributePath,
149 ) -> "SymbolicMode":
150 segments = list(parse_symbolic_mode(mode_raw, attribute_path))
151 return SymbolicMode(mode_raw, segments)
153 def __str__(self) -> str:
154 return self.symbolic_mode()
156 @property
157 def is_symbolic_mode(self) -> bool:
158 return False
160 def symbolic_mode(self) -> str:
161 return self.provided_mode
163 def compute_mode(self, current_mode: int, is_dir: bool) -> int:
164 final_mode = current_mode
165 for segment in self.segments:
166 final_mode = segment.apply(final_mode, is_dir)
167 return final_mode
170@dataclasses.dataclass(slots=True, frozen=True)
171class OctalMode(FileSystemMode):
172 octal_mode: int
174 @classmethod
175 def parse_filesystem_mode(
176 cls,
177 mode_raw: str,
178 attribute_path: AttributePath,
179 ) -> "FileSystemMode":
180 try:
181 mode = int(mode_raw, base=8)
182 except ValueError as e:
183 error_msg = 'An octal mode must be all digits between 0-7 (such as "644")'
184 raise ManifestParseException(
185 f"Cannot parse {attribute_path.path} as an octal mode: {error_msg}"
186 ) from e
187 return OctalMode(mode)
189 @property
190 def is_octal_mode(self) -> bool:
191 return True
193 def compute_mode(self, _current_mode: int, _is_dir: bool) -> int:
194 return self.octal_mode
196 def __str__(self) -> str:
197 return f"0{oct(self.octal_mode)[2:]}"
200@dataclasses.dataclass(slots=True, frozen=True)
201class _StaticFileSystemOwnerGroup:
202 ownership_definition: OwnershipDefinition
204 @property
205 def entity_name(self) -> str:
206 return self.ownership_definition.entity_name
208 @property
209 def entity_id(self) -> int:
210 return self.ownership_definition.entity_id
212 @classmethod
213 def from_manifest_value(
214 cls,
215 raw_input: str | int,
216 attribute_path: AttributePath,
217 ) -> "_StaticFileSystemOwnerGroup":
218 provided_name, provided_id = _parse_ownership(raw_input, attribute_path)
219 owner_def = cls._resolve(raw_input, provided_name, provided_id, attribute_path)
220 if ( 220 ↛ 224line 220 didn't jump to line 224 because the condition on line 220 was never true
221 owner_def.entity_name in BAD_OWNER_NAMES
222 or owner_def.entity_id in BAD_OWNER_IDS
223 ):
224 raise ManifestParseException(
225 f'Refusing to use "{raw_input}" as {cls._owner_type()} (defined at {attribute_path.path})'
226 f' as it resolves to "{owner_def.entity_name}:{owner_def.entity_id}" and no path should have this'
227 f" entity as {cls._owner_type()} as it is unsafe."
228 )
229 return cls(owner_def)
231 @classmethod
232 def _resolve(
233 cls,
234 raw_input: str | int,
235 provided_name: str | None,
236 provided_id: int | None,
237 attribute_path: AttributePath,
238 ) -> OwnershipDefinition:
239 table_name = cls._ownership_table_name()
240 name_table, id_table = _load_ownership_table_from_file(table_name)
241 name_match = (
242 name_table.get(provided_name) if provided_name is not None else None
243 )
244 id_match = id_table.get(provided_id) if provided_id is not None else None
245 if id_match is None and name_match is None: 245 ↛ 246line 245 didn't jump to line 246 because the condition on line 245 was never true
246 name_part = provided_name if provided_name is not None else "N/A"
247 id_part = provided_id if provided_id is not None else "N/A"
248 raise ManifestParseException(
249 f'Cannot resolve "{raw_input}" as {cls._owner_type()} (from {attribute_path.path}):'
250 f" It is not known to be a static {cls._owner_type()} from base-passwd."
251 f' The value was interpreted as name: "{name_part}" and id: {id_part}'
252 )
253 if id_match is None:
254 assert name_match is not None
255 return name_match
256 if name_match is None: 256 ↛ 259line 256 didn't jump to line 259 because the condition on line 256 was always true
257 assert id_match is not None
258 return id_match
259 if provided_name != id_match.entity_name:
260 raise ManifestParseException(
261 f"Bad {cls._owner_type()} declaration: The id {provided_id} resolves to {id_match.entity_name}"
262 f" according to base-passwd, but the packager declared to should have been {provided_name}"
263 f" at {attribute_path.path}"
264 )
265 if provided_id != name_match.entity_id:
266 raise ManifestParseException(
267 f"Bad {cls._owner_type} declaration: The name {provided_name} resolves to {name_match.entity_id}"
268 f" according to base-passwd, but the packager declared to should have been {provided_id}"
269 f" at {attribute_path.path}"
270 )
271 return id_match
273 @classmethod
274 def _owner_type(cls) -> Literal["owner", "group"]:
275 raise NotImplementedError
277 @classmethod
278 def _ownership_table_name(cls) -> Literal["passwd.master", "group.master"]:
279 raise NotImplementedError
282class StaticFileSystemOwner(_StaticFileSystemOwnerGroup):
283 @classmethod
284 def _owner_type(cls) -> Literal["owner", "group"]:
285 return "owner"
287 @classmethod
288 def _ownership_table_name(cls) -> Literal["passwd.master", "group.master"]:
289 return "passwd.master"
292class StaticFileSystemGroup(_StaticFileSystemOwnerGroup):
293 @classmethod
294 def _owner_type(cls) -> Literal["owner", "group"]:
295 return "group"
297 @classmethod
298 def _ownership_table_name(cls) -> Literal["passwd.master", "group.master"]:
299 return "group.master"
302@dataclasses.dataclass(slots=True, frozen=True)
303class SymlinkTarget:
304 raw_symlink_target: str
305 attribute_path: AttributePath
306 symlink_target: str
308 @classmethod
309 def parse_symlink_target(
310 cls,
311 raw_symlink_target: str,
312 attribute_path: AttributePath,
313 substitution: Substitution,
314 ) -> "SymlinkTarget":
315 return SymlinkTarget(
316 raw_symlink_target,
317 attribute_path,
318 substitution.substitute(raw_symlink_target, attribute_path.path),
319 )
322class FileSystemMatchRule:
323 @property
324 def raw_match_rule(self) -> str:
325 raise NotImplementedError
327 @property
328 def attribute_path(self) -> AttributePath:
329 raise NotImplementedError
331 @property
332 def match_rule(self) -> MatchRule:
333 raise NotImplementedError
335 @classmethod
336 def parse_path_match(
337 cls,
338 raw_match_rule: str,
339 attribute_path: AttributePath,
340 parser_context: "ParserContextData",
341 ) -> "FileSystemMatchRule":
342 return cls.from_path_match(
343 raw_match_rule, attribute_path, parser_context.substitution
344 )
346 @classmethod
347 def from_path_match(
348 cls,
349 raw_match_rule: str,
350 attribute_path: AttributePath,
351 substitution: "Substitution",
352 ) -> "FileSystemMatchRule":
353 try:
354 mr = MatchRule.from_path_or_glob(
355 raw_match_rule,
356 attribute_path.path,
357 substitution=substitution,
358 )
359 except ValueError as e:
360 raise ManifestParseException(
361 f'Could not parse "{raw_match_rule}" (defined at {attribute_path.path})'
362 f" as a path or a glob: {e.args[0]}"
363 )
365 if isinstance(mr, ExactFileSystemPath):
366 return FileSystemExactMatchRule(
367 raw_match_rule,
368 attribute_path,
369 mr,
370 )
371 return FileSystemGenericMatch(
372 raw_match_rule,
373 attribute_path,
374 mr,
375 )
378@dataclasses.dataclass(slots=True, frozen=True)
379class FileSystemGenericMatch(FileSystemMatchRule):
380 raw_match_rule: str
381 attribute_path: AttributePath
382 match_rule: MatchRule
385@dataclasses.dataclass(slots=True, frozen=True)
386class FileSystemExactMatchRule(FileSystemMatchRule):
387 raw_match_rule: str
388 attribute_path: AttributePath
389 match_rule: ExactFileSystemPath
391 @classmethod
392 def from_path_match(
393 cls,
394 raw_match_rule: str,
395 attribute_path: AttributePath,
396 substitution: "Substitution",
397 ) -> "FileSystemExactMatchRule":
398 try:
399 normalized = _normalize_path(raw_match_rule)
400 except ValueError as e:
401 raise ManifestParseException(
402 f'The path "{raw_match_rule}" provided in {attribute_path.path} should be relative to the'
403 ' root of the package and not use any ".." or "." segments.'
404 ) from e
405 if normalized == ".": 405 ↛ 406line 405 didn't jump to line 406 because the condition on line 405 was never true
406 raise ManifestParseException(
407 f'The path "{raw_match_rule}" matches a file system root and that is not a valid match'
408 f' at "{attribute_path.path}". Please narrow the provided path.'
409 )
410 mr = ExactFileSystemPath(
411 substitution.substitute(normalized, attribute_path.path)
412 )
413 if mr.path.endswith("/") and issubclass(cls, FileSystemExactNonDirMatchRule): 413 ↛ 414line 413 didn't jump to line 414 because the condition on line 413 was never true
414 raise ManifestParseException(
415 f'The path "{raw_match_rule}" at {attribute_path.path} resolved to'
416 f' "{mr.path}". Since the resolved path ends with a slash ("/"), this'
417 " means only a directory can match. However, this attribute should"
418 " match a *non*-directory"
419 )
420 return cls(
421 raw_match_rule,
422 attribute_path,
423 mr,
424 )
427class FileSystemExactNonDirMatchRule(FileSystemExactMatchRule):
428 pass
431class DpkgBuildflagsCache:
433 __slots__ = ("_cache_keys",)
435 def __init__(self) -> None:
436 self._cache_keys: dict[frozenset[tuple[str, str]], Mapping[str, str]] = {}
438 def run_dpkg_buildflags(
439 self,
440 env: Mapping[str, str],
441 definition_source: str | None,
442 ) -> Mapping[str, str]:
443 cache_key = frozenset((k, v) for k, v in env.items() if k.startswith("DEB_"))
444 dpkg_env = self._cache_keys.get(cache_key)
445 if dpkg_env is not None:
446 return dpkg_env
447 dpkg_env = {}
448 try:
449 bf_output = subprocess.check_output(["dpkg-buildflags"], env=env)
450 except FileNotFoundError:
451 if definition_source is None:
452 _error(
453 "The dpkg-buildflags command was not available and is necessary to set the relevant"
454 "env variables by default."
455 )
456 _error(
457 "The dpkg-buildflags command was not available and is necessary to set the relevant"
458 f"env variables for the environment defined at {definition_source}."
459 )
460 except subprocess.CalledProcessError as e:
461 if definition_source is None:
462 _error(
463 f"The dpkg-buildflags command failed with exit code {e.returncode}. Please review the output from"
464 f" dpkg-buildflags above to resolve the issue."
465 )
466 _error(
467 f"The dpkg-buildflags command failed with exit code {e.returncode}. Please review the output from"
468 f" dpkg-buildflags above to resolve the issue. The environment definition that triggered this call"
469 f" was {definition_source}"
470 )
471 else:
472 warned = False
473 for line in bf_output.decode("utf-8").splitlines(keepends=False):
474 if "=" not in line or line.startswith("="): 474 ↛ 475line 474 didn't jump to line 475 because the condition on line 474 was never true
475 if not warned:
476 _warn(
477 f"Unexpected output from dpkg-buildflags (not a K=V line): {line}"
478 )
479 continue
480 k, v = line.split("=", 1)
481 if k.strip() != k: 481 ↛ 482line 481 didn't jump to line 482 because the condition on line 481 was never true
482 if not warned:
483 _warn(
484 f'Unexpected output from dpkg-buildflags (Key had spaces): "{line}"'
485 )
486 continue
487 dpkg_env[k] = v
488 self._cache_keys[cache_key] = dpkg_env
489 return dpkg_env
492_DPKG_BUILDFLAGS_CACHE = DpkgBuildflagsCache()
493del DpkgBuildflagsCache
496class BuildEnvironmentDefinition:
498 def dpkg_buildflags_env(
499 self,
500 env: Mapping[str, str],
501 definition_source: str | None,
502 ) -> Mapping[str, str]:
503 return _DPKG_BUILDFLAGS_CACHE.run_dpkg_buildflags(env, definition_source)
505 def log_computed_env(self, source: str, computed_env: Mapping[str, str]) -> None:
506 _debug_log(f"Computed environment variables from {source}")
507 for k, v in computed_env.items():
508 _debug_log(f" {k}={v}")
510 def update_env(self, env: MutableMapping[str, str]) -> None:
511 dpkg_env = self.dpkg_buildflags_env(env, None)
512 if _is_debug_log_enabled():
513 self.log_computed_env("dpkg-buildflags", dpkg_env)
514 env.update(dpkg_env)
517class BuildEnvironments:
519 def __init__(
520 self,
521 environments: dict[str, BuildEnvironmentDefinition],
522 default_environment: BuildEnvironmentDefinition | None,
523 ) -> None:
524 self.environments = environments
525 self.default_environment = default_environment