Coverage for src/debputy/manifest_parser/base_types.py: 73%
258 statements
« prev ^ index » next coverage.py v7.8.2, created at 2025-09-07 09:27 +0000
1import dataclasses
2import os
3import subprocess
4from functools import lru_cache
5from typing import (
6 Sequence,
7 Optional,
8 Union,
9 Literal,
10 Tuple,
11 Mapping,
12 Iterable,
13 TYPE_CHECKING,
14 Dict,
15 MutableMapping,
16 NotRequired,
17 FrozenSet,
18)
20from debputy.manifest_conditions import ManifestCondition
21from debputy.manifest_parser.exceptions import ManifestParseException
22from debputy.manifest_parser.tagging_types import DebputyParsedContent
23from debputy.manifest_parser.util import (
24 AttributePath,
25 _SymbolicModeSegment,
26 parse_symbolic_mode,
27)
28from debputy.path_matcher import MatchRule, ExactFileSystemPath
29from debputy.substitution import Substitution
30from debputy.util import (
31 _normalize_path,
32 _error,
33 _warn,
34 _debug_log,
35 _is_debug_log_enabled,
36)
38if TYPE_CHECKING:
39 from debputy.manifest_parser.parser_data import ParserContextData
42@dataclasses.dataclass(slots=True, frozen=True)
43class OwnershipDefinition:
44 entity_name: str
45 entity_id: int
48class DebputyParsedContentStandardConditional(DebputyParsedContent):
49 when: NotRequired[ManifestCondition]
52ROOT_DEFINITION = OwnershipDefinition("root", 0)
55BAD_OWNER_NAMES = {
56 "_apt", # All things owned by _apt are generated by apt after installation
57 "nogroup", # It is not supposed to own anything as it is an entity used for dropping permissions
58 "nobody", # It is not supposed to own anything as it is an entity used for dropping permissions
59}
60BAD_OWNER_IDS = {
61 65534, # ID of nobody / nogroup
62}
65def _parse_ownership(
66 v: Union[str, int],
67 attribute_path: AttributePath,
68) -> Tuple[Optional[str], Optional[int]]:
69 if isinstance(v, str) and ":" in v: 69 ↛ 70line 69 didn't jump to line 70 because the condition on line 69 was never true
70 if v == ":":
71 raise ManifestParseException(
72 f'Invalid ownership value "{v}" at {attribute_path.path}: Ownership is redundant if it is ":"'
73 f" (blank name and blank id). Please provide non-default values or remove the definition."
74 )
75 entity_name: Optional[str]
76 entity_id: Optional[int]
77 entity_name, entity_id_str = v.split(":")
78 if entity_name == "":
79 entity_name = None
80 if entity_id_str != "":
81 entity_id = int(entity_id_str)
82 else:
83 entity_id = None
84 return entity_name, entity_id
86 if isinstance(v, int):
87 return None, v
88 if v.isdigit(): 88 ↛ 89line 88 didn't jump to line 89 because the condition on line 88 was never true
89 raise ManifestParseException(
90 f'Invalid ownership value "{v}" at {attribute_path.path}: The provided value "{v}" is a string (implying'
91 " name lookup), but it contains an integer (implying id lookup). Please use a regular int for id lookup"
92 f' (removing the quotes) or add a ":" in the end ("{v}:") as a disambiguation if you are *really* looking'
93 " for an entity with that name."
94 )
95 return v, None
98@lru_cache
99def _load_ownership_table_from_file(
100 name: Literal["passwd.master", "group.master"],
101) -> Tuple[Mapping[str, OwnershipDefinition], Mapping[int, OwnershipDefinition]]:
102 filename = os.path.join("/usr/share/base-passwd", name)
103 name_table = {}
104 uid_table = {}
105 for owner_def in _read_ownership_def_from_base_password_template(filename):
106 # Could happen if base-passwd template has two users with the same ID. We assume this will not occur.
107 assert owner_def.entity_name not in name_table
108 assert owner_def.entity_id not in uid_table
109 name_table[owner_def.entity_name] = owner_def
110 uid_table[owner_def.entity_id] = owner_def
112 return name_table, uid_table
115def _read_ownership_def_from_base_password_template(
116 template_file: str,
117) -> Iterable[OwnershipDefinition]:
118 with open(template_file) as fd:
119 for line in fd:
120 entity_name, _star, entity_id, _remainder = line.split(":", 3)
121 if entity_id == "0" and entity_name == "root":
122 yield ROOT_DEFINITION
123 else:
124 yield OwnershipDefinition(entity_name, int(entity_id))
127class FileSystemMode:
128 @classmethod
129 def parse_filesystem_mode(
130 cls,
131 mode_raw: str,
132 attribute_path: AttributePath,
133 ) -> "FileSystemMode":
134 if mode_raw and mode_raw[0].isdigit():
135 return OctalMode.parse_filesystem_mode(mode_raw, attribute_path)
136 return SymbolicMode.parse_filesystem_mode(mode_raw, attribute_path)
138 def compute_mode(self, current_mode: int, is_dir: bool) -> int:
139 raise NotImplementedError
142@dataclasses.dataclass(slots=True, frozen=True)
143class SymbolicMode(FileSystemMode):
144 provided_mode: str
145 segments: Sequence[_SymbolicModeSegment]
147 @classmethod
148 def parse_filesystem_mode(
149 cls,
150 mode_raw: str,
151 attribute_path: AttributePath,
152 ) -> "SymbolicMode":
153 segments = list(parse_symbolic_mode(mode_raw, attribute_path))
154 return SymbolicMode(mode_raw, segments)
156 def __str__(self) -> str:
157 return self.symbolic_mode()
159 @property
160 def is_symbolic_mode(self) -> bool:
161 return False
163 def symbolic_mode(self) -> str:
164 return self.provided_mode
166 def compute_mode(self, current_mode: int, is_dir: bool) -> int:
167 final_mode = current_mode
168 for segment in self.segments:
169 final_mode = segment.apply(final_mode, is_dir)
170 return final_mode
173@dataclasses.dataclass(slots=True, frozen=True)
174class OctalMode(FileSystemMode):
175 octal_mode: int
177 @classmethod
178 def parse_filesystem_mode(
179 cls,
180 mode_raw: str,
181 attribute_path: AttributePath,
182 ) -> "FileSystemMode":
183 try:
184 mode = int(mode_raw, base=8)
185 except ValueError as e:
186 error_msg = 'An octal mode must be all digits between 0-7 (such as "644")'
187 raise ManifestParseException(
188 f"Cannot parse {attribute_path.path} as an octal mode: {error_msg}"
189 ) from e
190 return OctalMode(mode)
192 @property
193 def is_octal_mode(self) -> bool:
194 return True
196 def compute_mode(self, _current_mode: int, _is_dir: bool) -> int:
197 return self.octal_mode
199 def __str__(self) -> str:
200 return f"0{oct(self.octal_mode)[2:]}"
203@dataclasses.dataclass(slots=True, frozen=True)
204class _StaticFileSystemOwnerGroup:
205 ownership_definition: OwnershipDefinition
207 @property
208 def entity_name(self) -> str:
209 return self.ownership_definition.entity_name
211 @property
212 def entity_id(self) -> int:
213 return self.ownership_definition.entity_id
215 @classmethod
216 def from_manifest_value(
217 cls,
218 raw_input: Union[str, int],
219 attribute_path: AttributePath,
220 ) -> "_StaticFileSystemOwnerGroup":
221 provided_name, provided_id = _parse_ownership(raw_input, attribute_path)
222 owner_def = cls._resolve(raw_input, provided_name, provided_id, attribute_path)
223 if ( 223 ↛ 227line 223 didn't jump to line 227 because the condition on line 223 was never true
224 owner_def.entity_name in BAD_OWNER_NAMES
225 or owner_def.entity_id in BAD_OWNER_IDS
226 ):
227 raise ManifestParseException(
228 f'Refusing to use "{raw_input}" as {cls._owner_type()} (defined at {attribute_path.path})'
229 f' as it resolves to "{owner_def.entity_name}:{owner_def.entity_id}" and no path should have this'
230 f" entity as {cls._owner_type()} as it is unsafe."
231 )
232 return cls(owner_def)
234 @classmethod
235 def _resolve(
236 cls,
237 raw_input: Union[str, int],
238 provided_name: Optional[str],
239 provided_id: Optional[int],
240 attribute_path: AttributePath,
241 ) -> OwnershipDefinition:
242 table_name = cls._ownership_table_name()
243 name_table, id_table = _load_ownership_table_from_file(table_name)
244 name_match = (
245 name_table.get(provided_name) if provided_name is not None else None
246 )
247 id_match = id_table.get(provided_id) if provided_id is not None else None
248 if id_match is None and name_match is None: 248 ↛ 249line 248 didn't jump to line 249 because the condition on line 248 was never true
249 name_part = provided_name if provided_name is not None else "N/A"
250 id_part = provided_id if provided_id is not None else "N/A"
251 raise ManifestParseException(
252 f'Cannot resolve "{raw_input}" as {cls._owner_type()} (from {attribute_path.path}):'
253 f" It is not known to be a static {cls._owner_type()} from base-passwd."
254 f' The value was interpreted as name: "{name_part}" and id: {id_part}'
255 )
256 if id_match is None:
257 assert name_match is not None
258 return name_match
259 if name_match is None: 259 ↛ 262line 259 didn't jump to line 262 because the condition on line 259 was always true
260 assert id_match is not None
261 return id_match
262 if provided_name != id_match.entity_name:
263 raise ManifestParseException(
264 f"Bad {cls._owner_type()} declaration: The id {provided_id} resolves to {id_match.entity_name}"
265 f" according to base-passwd, but the packager declared to should have been {provided_name}"
266 f" at {attribute_path.path}"
267 )
268 if provided_id != name_match.entity_id:
269 raise ManifestParseException(
270 f"Bad {cls._owner_type} declaration: The name {provided_name} resolves to {name_match.entity_id}"
271 f" according to base-passwd, but the packager declared to should have been {provided_id}"
272 f" at {attribute_path.path}"
273 )
274 return id_match
276 @classmethod
277 def _owner_type(cls) -> Literal["owner", "group"]:
278 raise NotImplementedError
280 @classmethod
281 def _ownership_table_name(cls) -> Literal["passwd.master", "group.master"]:
282 raise NotImplementedError
285class StaticFileSystemOwner(_StaticFileSystemOwnerGroup):
286 @classmethod
287 def _owner_type(cls) -> Literal["owner", "group"]:
288 return "owner"
290 @classmethod
291 def _ownership_table_name(cls) -> Literal["passwd.master", "group.master"]:
292 return "passwd.master"
295class StaticFileSystemGroup(_StaticFileSystemOwnerGroup):
296 @classmethod
297 def _owner_type(cls) -> Literal["owner", "group"]:
298 return "group"
300 @classmethod
301 def _ownership_table_name(cls) -> Literal["passwd.master", "group.master"]:
302 return "group.master"
305@dataclasses.dataclass(slots=True, frozen=True)
306class SymlinkTarget:
307 raw_symlink_target: str
308 attribute_path: AttributePath
309 symlink_target: str
311 @classmethod
312 def parse_symlink_target(
313 cls,
314 raw_symlink_target: str,
315 attribute_path: AttributePath,
316 substitution: Substitution,
317 ) -> "SymlinkTarget":
318 return SymlinkTarget(
319 raw_symlink_target,
320 attribute_path,
321 substitution.substitute(raw_symlink_target, attribute_path.path),
322 )
325class FileSystemMatchRule:
326 @property
327 def raw_match_rule(self) -> str:
328 raise NotImplementedError
330 @property
331 def attribute_path(self) -> AttributePath:
332 raise NotImplementedError
334 @property
335 def match_rule(self) -> MatchRule:
336 raise NotImplementedError
338 @classmethod
339 def parse_path_match(
340 cls,
341 raw_match_rule: str,
342 attribute_path: AttributePath,
343 parser_context: "ParserContextData",
344 ) -> "FileSystemMatchRule":
345 return cls.from_path_match(
346 raw_match_rule, attribute_path, parser_context.substitution
347 )
349 @classmethod
350 def from_path_match(
351 cls,
352 raw_match_rule: str,
353 attribute_path: AttributePath,
354 substitution: "Substitution",
355 ) -> "FileSystemMatchRule":
356 try:
357 mr = MatchRule.from_path_or_glob(
358 raw_match_rule,
359 attribute_path.path,
360 substitution=substitution,
361 )
362 except ValueError as e:
363 raise ManifestParseException(
364 f'Could not parse "{raw_match_rule}" (defined at {attribute_path.path})'
365 f" as a path or a glob: {e.args[0]}"
366 )
368 if isinstance(mr, ExactFileSystemPath):
369 return FileSystemExactMatchRule(
370 raw_match_rule,
371 attribute_path,
372 mr,
373 )
374 return FileSystemGenericMatch(
375 raw_match_rule,
376 attribute_path,
377 mr,
378 )
381@dataclasses.dataclass(slots=True, frozen=True)
382class FileSystemGenericMatch(FileSystemMatchRule):
383 raw_match_rule: str
384 attribute_path: AttributePath
385 match_rule: MatchRule
388@dataclasses.dataclass(slots=True, frozen=True)
389class FileSystemExactMatchRule(FileSystemMatchRule):
390 raw_match_rule: str
391 attribute_path: AttributePath
392 match_rule: ExactFileSystemPath
394 @classmethod
395 def from_path_match(
396 cls,
397 raw_match_rule: str,
398 attribute_path: AttributePath,
399 substitution: "Substitution",
400 ) -> "FileSystemExactMatchRule":
401 try:
402 normalized = _normalize_path(raw_match_rule)
403 except ValueError as e:
404 raise ManifestParseException(
405 f'The path "{raw_match_rule}" provided in {attribute_path.path} should be relative to the'
406 ' root of the package and not use any ".." or "." segments.'
407 ) from e
408 if normalized == ".": 408 ↛ 409line 408 didn't jump to line 409 because the condition on line 408 was never true
409 raise ManifestParseException(
410 f'The path "{raw_match_rule}" matches a file system root and that is not a valid match'
411 f' at "{attribute_path.path}". Please narrow the provided path.'
412 )
413 mr = ExactFileSystemPath(
414 substitution.substitute(normalized, attribute_path.path)
415 )
416 if mr.path.endswith("/") and issubclass(cls, FileSystemExactNonDirMatchRule): 416 ↛ 417line 416 didn't jump to line 417 because the condition on line 416 was never true
417 raise ManifestParseException(
418 f'The path "{raw_match_rule}" at {attribute_path.path} resolved to'
419 f' "{mr.path}". Since the resolved path ends with a slash ("/"), this'
420 " means only a directory can match. However, this attribute should"
421 " match a *non*-directory"
422 )
423 return cls(
424 raw_match_rule,
425 attribute_path,
426 mr,
427 )
430class FileSystemExactNonDirMatchRule(FileSystemExactMatchRule):
431 pass
434class DpkgBuildflagsCache:
436 __slots__ = ("_cache_keys",)
438 def __init__(self) -> None:
439 self._cache_keys: Dict[FrozenSet[Tuple[str, str]], Mapping[str, str]] = {}
441 def run_dpkg_buildflags(
442 self,
443 env: Mapping[str, str],
444 definition_source: Optional[str],
445 ) -> Mapping[str, str]:
446 cache_key = frozenset((k, v) for k, v in env.items() if k.startswith("DEB_"))
447 dpkg_env = self._cache_keys.get(cache_key)
448 if dpkg_env is not None:
449 return dpkg_env
450 dpkg_env = {}
451 try:
452 bf_output = subprocess.check_output(["dpkg-buildflags"], env=env)
453 except FileNotFoundError:
454 if definition_source is None:
455 _error(
456 "The dpkg-buildflags command was not available and is necessary to set the relevant"
457 "env variables by default."
458 )
459 _error(
460 "The dpkg-buildflags command was not available and is necessary to set the relevant"
461 f"env variables for the environment defined at {definition_source}."
462 )
463 except subprocess.CalledProcessError as e:
464 if definition_source is None:
465 _error(
466 f"The dpkg-buildflags command failed with exit code {e.returncode}. Please review the output from"
467 f" dpkg-buildflags above to resolve the issue."
468 )
469 _error(
470 f"The dpkg-buildflags command failed with exit code {e.returncode}. Please review the output from"
471 f" dpkg-buildflags above to resolve the issue. The environment definition that triggered this call"
472 f" was {definition_source}"
473 )
474 else:
475 warned = False
476 for line in bf_output.decode("utf-8").splitlines(keepends=False):
477 if "=" not in line or line.startswith("="): 477 ↛ 478line 477 didn't jump to line 478 because the condition on line 477 was never true
478 if not warned:
479 _warn(
480 f"Unexpected output from dpkg-buildflags (not a K=V line): {line}"
481 )
482 continue
483 k, v = line.split("=", 1)
484 if k.strip() != k: 484 ↛ 485line 484 didn't jump to line 485 because the condition on line 484 was never true
485 if not warned:
486 _warn(
487 f'Unexpected output from dpkg-buildflags (Key had spaces): "{line}"'
488 )
489 continue
490 dpkg_env[k] = v
491 self._cache_keys[cache_key] = dpkg_env
492 return dpkg_env
495_DPKG_BUILDFLAGS_CACHE = DpkgBuildflagsCache()
496del DpkgBuildflagsCache
499class BuildEnvironmentDefinition:
501 def dpkg_buildflags_env(
502 self,
503 env: Mapping[str, str],
504 definition_source: Optional[str],
505 ) -> Mapping[str, str]:
506 return _DPKG_BUILDFLAGS_CACHE.run_dpkg_buildflags(env, definition_source)
508 def log_computed_env(self, source: str, computed_env: Mapping[str, str]) -> None:
509 _debug_log(f"Computed environment variables from {source}")
510 for k, v in computed_env.items():
511 _debug_log(f" {k}={v}")
513 def update_env(self, env: MutableMapping[str, str]) -> None:
514 dpkg_env = self.dpkg_buildflags_env(env, None)
515 if _is_debug_log_enabled():
516 self.log_computed_env("dpkg-buildflags", dpkg_env)
517 env.update(dpkg_env)
520class BuildEnvironments:
522 def __init__(
523 self,
524 environments: Dict[str, BuildEnvironmentDefinition],
525 default_environment: Optional[BuildEnvironmentDefinition],
526 ) -> None:
527 self.environments = environments
528 self.default_environment = default_environment