Coverage for src/debputy/manifest_parser/base_types.py: 78%
244 statements
« prev ^ index » next coverage.py v7.6.0, created at 2025-03-24 16:38 +0000
1import dataclasses
2import os
3import subprocess
4from functools import lru_cache
5from typing import (
6 Sequence,
7 Optional,
8 Union,
9 Literal,
10 Tuple,
11 Mapping,
12 Iterable,
13 TYPE_CHECKING,
14 Dict,
15 MutableMapping,
16 NotRequired,
17)
19from debputy.manifest_conditions import ManifestCondition
20from debputy.manifest_parser.exceptions import ManifestParseException
21from debputy.manifest_parser.tagging_types import DebputyParsedContent
22from debputy.manifest_parser.util import (
23 AttributePath,
24 _SymbolicModeSegment,
25 parse_symbolic_mode,
26)
27from debputy.path_matcher import MatchRule, ExactFileSystemPath
28from debputy.substitution import Substitution
29from debputy.util import _normalize_path, _error, _warn, _debug_log
31if TYPE_CHECKING:
32 from debputy.manifest_parser.parser_data import ParserContextData
35@dataclasses.dataclass(slots=True, frozen=True)
36class OwnershipDefinition:
37 entity_name: str
38 entity_id: int
41class DebputyParsedContentStandardConditional(DebputyParsedContent):
42 when: NotRequired[ManifestCondition]
45ROOT_DEFINITION = OwnershipDefinition("root", 0)
48BAD_OWNER_NAMES = {
49 "_apt", # All things owned by _apt are generated by apt after installation
50 "nogroup", # It is not supposed to own anything as it is an entity used for dropping permissions
51 "nobody", # It is not supposed to own anything as it is an entity used for dropping permissions
52}
53BAD_OWNER_IDS = {
54 65534, # ID of nobody / nogroup
55}
58def _parse_ownership(
59 v: Union[str, int],
60 attribute_path: AttributePath,
61) -> Tuple[Optional[str], Optional[int]]:
62 if isinstance(v, str) and ":" in v: 62 ↛ 63line 62 didn't jump to line 63 because the condition on line 62 was never true
63 if v == ":":
64 raise ManifestParseException(
65 f'Invalid ownership value "{v}" at {attribute_path.path}: Ownership is redundant if it is ":"'
66 f" (blank name and blank id). Please provide non-default values or remove the definition."
67 )
68 entity_name: Optional[str]
69 entity_id: Optional[int]
70 entity_name, entity_id_str = v.split(":")
71 if entity_name == "":
72 entity_name = None
73 if entity_id_str != "":
74 entity_id = int(entity_id_str)
75 else:
76 entity_id = None
77 return entity_name, entity_id
79 if isinstance(v, int):
80 return None, v
81 if v.isdigit(): 81 ↛ 82line 81 didn't jump to line 82 because the condition on line 81 was never true
82 raise ManifestParseException(
83 f'Invalid ownership value "{v}" at {attribute_path.path}: The provided value "{v}" is a string (implying'
84 " name lookup), but it contains an integer (implying id lookup). Please use a regular int for id lookup"
85 f' (removing the quotes) or add a ":" in the end ("{v}:") as a disambiguation if you are *really* looking'
86 " for an entity with that name."
87 )
88 return v, None
91@lru_cache
92def _load_ownership_table_from_file(
93 name: Literal["passwd.master", "group.master"],
94) -> Tuple[Mapping[str, OwnershipDefinition], Mapping[int, OwnershipDefinition]]:
95 filename = os.path.join("/usr/share/base-passwd", name)
96 name_table = {}
97 uid_table = {}
98 for owner_def in _read_ownership_def_from_base_password_template(filename):
99 # Could happen if base-passwd template has two users with the same ID. We assume this will not occur.
100 assert owner_def.entity_name not in name_table
101 assert owner_def.entity_id not in uid_table
102 name_table[owner_def.entity_name] = owner_def
103 uid_table[owner_def.entity_id] = owner_def
105 return name_table, uid_table
108def _read_ownership_def_from_base_password_template(
109 template_file: str,
110) -> Iterable[OwnershipDefinition]:
111 with open(template_file) as fd:
112 for line in fd:
113 entity_name, _star, entity_id, _remainder = line.split(":", 3)
114 if entity_id == "0" and entity_name == "root":
115 yield ROOT_DEFINITION
116 else:
117 yield OwnershipDefinition(entity_name, int(entity_id))
120class FileSystemMode:
121 @classmethod
122 def parse_filesystem_mode(
123 cls,
124 mode_raw: str,
125 attribute_path: AttributePath,
126 ) -> "FileSystemMode":
127 if mode_raw and mode_raw[0].isdigit():
128 return OctalMode.parse_filesystem_mode(mode_raw, attribute_path)
129 return SymbolicMode.parse_filesystem_mode(mode_raw, attribute_path)
131 def compute_mode(self, current_mode: int, is_dir: bool) -> int:
132 raise NotImplementedError
135@dataclasses.dataclass(slots=True, frozen=True)
136class SymbolicMode(FileSystemMode):
137 provided_mode: str
138 segments: Sequence[_SymbolicModeSegment]
140 @classmethod
141 def parse_filesystem_mode(
142 cls,
143 mode_raw: str,
144 attribute_path: AttributePath,
145 ) -> "SymbolicMode":
146 segments = list(parse_symbolic_mode(mode_raw, attribute_path))
147 return SymbolicMode(mode_raw, segments)
149 def __str__(self) -> str:
150 return self.symbolic_mode()
152 @property
153 def is_symbolic_mode(self) -> bool:
154 return False
156 def symbolic_mode(self) -> str:
157 return self.provided_mode
159 def compute_mode(self, current_mode: int, is_dir: bool) -> int:
160 final_mode = current_mode
161 for segment in self.segments:
162 final_mode = segment.apply(final_mode, is_dir)
163 return final_mode
166@dataclasses.dataclass(slots=True, frozen=True)
167class OctalMode(FileSystemMode):
168 octal_mode: int
170 @classmethod
171 def parse_filesystem_mode(
172 cls,
173 mode_raw: str,
174 attribute_path: AttributePath,
175 ) -> "FileSystemMode":
176 try:
177 mode = int(mode_raw, base=8)
178 except ValueError as e:
179 error_msg = 'An octal mode must be all digits between 0-7 (such as "644")'
180 raise ManifestParseException(
181 f"Cannot parse {attribute_path.path} as an octal mode: {error_msg}"
182 ) from e
183 return OctalMode(mode)
185 @property
186 def is_octal_mode(self) -> bool:
187 return True
189 def compute_mode(self, _current_mode: int, _is_dir: bool) -> int:
190 return self.octal_mode
192 def __str__(self) -> str:
193 return f"0{oct(self.octal_mode)[2:]}"
196@dataclasses.dataclass(slots=True, frozen=True)
197class _StaticFileSystemOwnerGroup:
198 ownership_definition: OwnershipDefinition
200 @property
201 def entity_name(self) -> str:
202 return self.ownership_definition.entity_name
204 @property
205 def entity_id(self) -> int:
206 return self.ownership_definition.entity_id
208 @classmethod
209 def from_manifest_value(
210 cls,
211 raw_input: Union[str, int],
212 attribute_path: AttributePath,
213 ) -> "_StaticFileSystemOwnerGroup":
214 provided_name, provided_id = _parse_ownership(raw_input, attribute_path)
215 owner_def = cls._resolve(raw_input, provided_name, provided_id, attribute_path)
216 if ( 216 ↛ 220line 216 didn't jump to line 220
217 owner_def.entity_name in BAD_OWNER_NAMES
218 or owner_def.entity_id in BAD_OWNER_IDS
219 ):
220 raise ManifestParseException(
221 f'Refusing to use "{raw_input}" as {cls._owner_type()} (defined at {attribute_path.path})'
222 f' as it resolves to "{owner_def.entity_name}:{owner_def.entity_id}" and no path should have this'
223 f" entity as {cls._owner_type()} as it is unsafe."
224 )
225 return cls(owner_def)
227 @classmethod
228 def _resolve(
229 cls,
230 raw_input: Union[str, int],
231 provided_name: Optional[str],
232 provided_id: Optional[int],
233 attribute_path: AttributePath,
234 ) -> OwnershipDefinition:
235 table_name = cls._ownership_table_name()
236 name_table, id_table = _load_ownership_table_from_file(table_name)
237 name_match = (
238 name_table.get(provided_name) if provided_name is not None else None
239 )
240 id_match = id_table.get(provided_id) if provided_id is not None else None
241 if id_match is None and name_match is None: 241 ↛ 242line 241 didn't jump to line 242 because the condition on line 241 was never true
242 name_part = provided_name if provided_name is not None else "N/A"
243 id_part = provided_id if provided_id is not None else "N/A"
244 raise ManifestParseException(
245 f'Cannot resolve "{raw_input}" as {cls._owner_type()} (from {attribute_path.path}):'
246 f" It is not known to be a static {cls._owner_type()} from base-passwd."
247 f' The value was interpreted as name: "{name_part}" and id: {id_part}'
248 )
249 if id_match is None:
250 assert name_match is not None
251 return name_match
252 if name_match is None: 252 ↛ 255line 252 didn't jump to line 255 because the condition on line 252 was always true
253 assert id_match is not None
254 return id_match
255 if provided_name != id_match.entity_name:
256 raise ManifestParseException(
257 f"Bad {cls._owner_type()} declaration: The id {provided_id} resolves to {id_match.entity_name}"
258 f" according to base-passwd, but the packager declared to should have been {provided_name}"
259 f" at {attribute_path.path}"
260 )
261 if provided_id != name_match.entity_id:
262 raise ManifestParseException(
263 f"Bad {cls._owner_type} declaration: The name {provided_name} resolves to {name_match.entity_id}"
264 f" according to base-passwd, but the packager declared to should have been {provided_id}"
265 f" at {attribute_path.path}"
266 )
267 return id_match
269 @classmethod
270 def _owner_type(cls) -> Literal["owner", "group"]:
271 raise NotImplementedError
273 @classmethod
274 def _ownership_table_name(cls) -> Literal["passwd.master", "group.master"]:
275 raise NotImplementedError
278class StaticFileSystemOwner(_StaticFileSystemOwnerGroup):
279 @classmethod
280 def _owner_type(cls) -> Literal["owner", "group"]:
281 return "owner"
283 @classmethod
284 def _ownership_table_name(cls) -> Literal["passwd.master", "group.master"]:
285 return "passwd.master"
288class StaticFileSystemGroup(_StaticFileSystemOwnerGroup):
289 @classmethod
290 def _owner_type(cls) -> Literal["owner", "group"]:
291 return "group"
293 @classmethod
294 def _ownership_table_name(cls) -> Literal["passwd.master", "group.master"]:
295 return "group.master"
298@dataclasses.dataclass(slots=True, frozen=True)
299class SymlinkTarget:
300 raw_symlink_target: str
301 attribute_path: AttributePath
302 symlink_target: str
304 @classmethod
305 def parse_symlink_target(
306 cls,
307 raw_symlink_target: str,
308 attribute_path: AttributePath,
309 substitution: Substitution,
310 ) -> "SymlinkTarget":
311 return SymlinkTarget(
312 raw_symlink_target,
313 attribute_path,
314 substitution.substitute(raw_symlink_target, attribute_path.path),
315 )
318class FileSystemMatchRule:
319 @property
320 def raw_match_rule(self) -> str:
321 raise NotImplementedError
323 @property
324 def attribute_path(self) -> AttributePath:
325 raise NotImplementedError
327 @property
328 def match_rule(self) -> MatchRule:
329 raise NotImplementedError
331 @classmethod
332 def parse_path_match(
333 cls,
334 raw_match_rule: str,
335 attribute_path: AttributePath,
336 parser_context: "ParserContextData",
337 ) -> "FileSystemMatchRule":
338 return cls.from_path_match(
339 raw_match_rule, attribute_path, parser_context.substitution
340 )
342 @classmethod
343 def from_path_match(
344 cls,
345 raw_match_rule: str,
346 attribute_path: AttributePath,
347 substitution: "Substitution",
348 ) -> "FileSystemMatchRule":
349 try:
350 mr = MatchRule.from_path_or_glob(
351 raw_match_rule,
352 attribute_path.path,
353 substitution=substitution,
354 )
355 except ValueError as e:
356 raise ManifestParseException(
357 f'Could not parse "{raw_match_rule}" (defined at {attribute_path.path})'
358 f" as a path or a glob: {e.args[0]}"
359 )
361 if isinstance(mr, ExactFileSystemPath):
362 return FileSystemExactMatchRule(
363 raw_match_rule,
364 attribute_path,
365 mr,
366 )
367 return FileSystemGenericMatch(
368 raw_match_rule,
369 attribute_path,
370 mr,
371 )
374@dataclasses.dataclass(slots=True, frozen=True)
375class FileSystemGenericMatch(FileSystemMatchRule):
376 raw_match_rule: str
377 attribute_path: AttributePath
378 match_rule: MatchRule
381@dataclasses.dataclass(slots=True, frozen=True)
382class FileSystemExactMatchRule(FileSystemMatchRule):
383 raw_match_rule: str
384 attribute_path: AttributePath
385 match_rule: ExactFileSystemPath
387 @classmethod
388 def from_path_match(
389 cls,
390 raw_match_rule: str,
391 attribute_path: AttributePath,
392 substitution: "Substitution",
393 ) -> "FileSystemExactMatchRule":
394 try:
395 normalized = _normalize_path(raw_match_rule)
396 except ValueError as e:
397 raise ManifestParseException(
398 f'The path "{raw_match_rule}" provided in {attribute_path.path} should be relative to the'
399 ' root of the package and not use any ".." or "." segments.'
400 ) from e
401 if normalized == ".": 401 ↛ 402line 401 didn't jump to line 402 because the condition on line 401 was never true
402 raise ManifestParseException(
403 f'The path "{raw_match_rule}" matches a file system root and that is not a valid match'
404 f' at "{attribute_path.path}". Please narrow the provided path.'
405 )
406 mr = ExactFileSystemPath(
407 substitution.substitute(normalized, attribute_path.path)
408 )
409 if mr.path.endswith("/") and issubclass(cls, FileSystemExactNonDirMatchRule): 409 ↛ 410line 409 didn't jump to line 410 because the condition on line 409 was never true
410 raise ManifestParseException(
411 f'The path "{raw_match_rule}" at {attribute_path.path} resolved to'
412 f' "{mr.path}". Since the resolved path ends with a slash ("/"), this'
413 " means only a directory can match. However, this attribute should"
414 " match a *non*-directory"
415 )
416 return cls(
417 raw_match_rule,
418 attribute_path,
419 mr,
420 )
423class FileSystemExactNonDirMatchRule(FileSystemExactMatchRule):
424 pass
427class BuildEnvironmentDefinition:
429 def dpkg_buildflags_env(
430 self,
431 env: Mapping[str, str],
432 definition_source: Optional[str],
433 ) -> Dict[str, str]:
434 dpkg_env = {}
435 try:
436 bf_output = subprocess.check_output(["dpkg-buildflags"], env=env)
437 except FileNotFoundError:
438 if definition_source is None:
439 _error(
440 "The dpkg-buildflags command was not available and is necessary to set the relevant"
441 "env variables by default."
442 )
443 _error(
444 "The dpkg-buildflags command was not available and is necessary to set the relevant"
445 f"env variables for the environment defined at {definition_source}."
446 )
447 except subprocess.CalledProcessError as e:
448 if definition_source is None:
449 _error(
450 f"The dpkg-buildflags command failed with exit code {e.returncode}. Please review the output from"
451 f" dpkg-buildflags above to resolve the issue."
452 )
453 _error(
454 f"The dpkg-buildflags command failed with exit code {e.returncode}. Please review the output from"
455 f" dpkg-buildflags above to resolve the issue. The environment definition that triggered this call"
456 f" was {definition_source}"
457 )
458 else:
459 warned = False
460 for line in bf_output.decode("utf-8").splitlines(keepends=False):
461 if "=" not in line or line.startswith("="): 461 ↛ 462line 461 didn't jump to line 462 because the condition on line 461 was never true
462 if not warned:
463 _warn(
464 f"Unexpected output from dpkg-buildflags (not a K=V line): {line}"
465 )
466 continue
467 k, v = line.split("=", 1)
468 if k.strip() != k: 468 ↛ 469line 468 didn't jump to line 469 because the condition on line 468 was never true
469 if not warned:
470 _warn(
471 f'Unexpected output from dpkg-buildflags (Key had spaces): "{line}"'
472 )
473 continue
474 dpkg_env[k] = v
476 return dpkg_env
478 def log_computed_env(self, source: str, computed_env: Mapping[str, str]) -> None:
479 _debug_log(f"Computed environment variables from {source}")
480 for k, v in computed_env.items():
481 _debug_log(f" {k}={v}")
483 def update_env(self, env: MutableMapping[str, str]) -> None:
484 dpkg_env = self.dpkg_buildflags_env(env, None)
485 self.log_computed_env("dpkg-buildflags", dpkg_env)
486 env.update(dpkg_env)
489class BuildEnvironments:
491 def __init__(
492 self,
493 environments: Dict[str, BuildEnvironmentDefinition],
494 default_environment: Optional[BuildEnvironmentDefinition],
495 ) -> None:
496 self.environments = environments
497 self.default_environment = default_environment