Coverage for src/debputy/manifest_parser/base_types.py: 73%

259 statements  

« prev     ^ index     » next       coverage.py v7.8.2, created at 2025-10-12 15:06 +0000

1import dataclasses 

2import os 

3import subprocess 

4from functools import lru_cache 

5from typing import ( 

6 Optional, 

7 Union, 

8 Literal, 

9 Tuple, 

10 TYPE_CHECKING, 

11 Dict, 

12 NotRequired, 

13 FrozenSet, 

14) 

15from collections.abc import Sequence, Mapping, Iterable, MutableMapping 

16 

17from debputy.manifest_conditions import ManifestCondition 

18from debputy.manifest_parser.exceptions import ManifestParseException 

19from debputy.manifest_parser.tagging_types import DebputyParsedContent 

20from debputy.manifest_parser.util import ( 

21 AttributePath, 

22 _SymbolicModeSegment, 

23 parse_symbolic_mode, 

24) 

25from debputy.path_matcher import MatchRule, ExactFileSystemPath 

26from debputy.substitution import Substitution 

27from debputy.util import ( 

28 _normalize_path, 

29 _error, 

30 _warn, 

31 _debug_log, 

32 _is_debug_log_enabled, 

33) 

34 

35if TYPE_CHECKING: 

36 from debputy.manifest_parser.parser_data import ParserContextData 

37 

38 

39@dataclasses.dataclass(slots=True, frozen=True) 

40class OwnershipDefinition: 

41 entity_name: str 

42 entity_id: int 

43 

44 

45class DebputyParsedContentStandardConditional(DebputyParsedContent): 

46 when: NotRequired[ManifestCondition] 

47 

48 

49ROOT_DEFINITION = OwnershipDefinition("root", 0) 

50 

51 

52BAD_OWNER_NAMES = { 

53 "_apt", # All things owned by _apt are generated by apt after installation 

54 "nogroup", # It is not supposed to own anything as it is an entity used for dropping permissions 

55 "nobody", # It is not supposed to own anything as it is an entity used for dropping permissions 

56} 

57BAD_OWNER_IDS = { 

58 65534, # ID of nobody / nogroup 

59} 

60 

61 

62def _parse_ownership( 

63 v: str | int, 

64 attribute_path: AttributePath, 

65) -> tuple[str | None, int | None]: 

66 if isinstance(v, str) and ":" in v: 66 ↛ 67line 66 didn't jump to line 67 because the condition on line 66 was never true

67 if v == ":": 

68 raise ManifestParseException( 

69 f'Invalid ownership value "{v}" at {attribute_path.path}: Ownership is redundant if it is ":"' 

70 f" (blank name and blank id). Please provide non-default values or remove the definition." 

71 ) 

72 entity_name: str | None 

73 entity_id: int | None 

74 entity_name, entity_id_str = v.split(":") 

75 if entity_name == "": 

76 entity_name = None 

77 if entity_id_str != "": 

78 entity_id = int(entity_id_str) 

79 else: 

80 entity_id = None 

81 return entity_name, entity_id 

82 

83 if isinstance(v, int): 

84 return None, v 

85 if v.isdigit(): 85 ↛ 86line 85 didn't jump to line 86 because the condition on line 85 was never true

86 raise ManifestParseException( 

87 f'Invalid ownership value "{v}" at {attribute_path.path}: The provided value "{v}" is a string (implying' 

88 " name lookup), but it contains an integer (implying id lookup). Please use a regular int for id lookup" 

89 f' (removing the quotes) or add a ":" in the end ("{v}:") as a disambiguation if you are *really* looking' 

90 " for an entity with that name." 

91 ) 

92 return v, None 

93 

94 

95@lru_cache 

96def _load_ownership_table_from_file( 

97 name: Literal["passwd.master", "group.master"], 

98) -> tuple[Mapping[str, OwnershipDefinition], Mapping[int, OwnershipDefinition]]: 

99 filename = os.path.join("/usr/share/base-passwd", name) 

100 name_table = {} 

101 uid_table = {} 

102 for owner_def in _read_ownership_def_from_base_password_template(filename): 

103 # Could happen if base-passwd template has two users with the same ID. We assume this will not occur. 

104 assert owner_def.entity_name not in name_table 

105 assert owner_def.entity_id not in uid_table 

106 name_table[owner_def.entity_name] = owner_def 

107 uid_table[owner_def.entity_id] = owner_def 

108 

109 return name_table, uid_table 

110 

111 

112def _read_ownership_def_from_base_password_template( 

113 template_file: str, 

114) -> Iterable[OwnershipDefinition]: 

115 with open(template_file) as fd: 

116 for line in fd: 

117 entity_name, _star, entity_id, _remainder = line.split(":", 3) 

118 if entity_id == "0" and entity_name == "root": 

119 yield ROOT_DEFINITION 

120 else: 

121 yield OwnershipDefinition(entity_name, int(entity_id)) 

122 

123 

124class FileSystemMode: 

125 @classmethod 

126 def parse_filesystem_mode( 

127 cls, 

128 mode_raw: str, 

129 attribute_path: AttributePath, 

130 ) -> "FileSystemMode": 

131 if mode_raw and mode_raw[0].isdigit(): 

132 return OctalMode.parse_filesystem_mode(mode_raw, attribute_path) 

133 return SymbolicMode.parse_filesystem_mode(mode_raw, attribute_path) 

134 

135 def compute_mode(self, current_mode: int, is_dir: bool) -> int: 

136 raise NotImplementedError 

137 

138 

139@dataclasses.dataclass(slots=True, frozen=True) 

140class SymbolicMode(FileSystemMode): 

141 provided_mode: str 

142 segments: Sequence[_SymbolicModeSegment] 

143 

144 @classmethod 

145 def parse_filesystem_mode( 

146 cls, 

147 mode_raw: str, 

148 attribute_path: AttributePath, 

149 ) -> "SymbolicMode": 

150 segments = list(parse_symbolic_mode(mode_raw, attribute_path)) 

151 return SymbolicMode(mode_raw, segments) 

152 

153 def __str__(self) -> str: 

154 return self.symbolic_mode() 

155 

156 @property 

157 def is_symbolic_mode(self) -> bool: 

158 return False 

159 

160 def symbolic_mode(self) -> str: 

161 return self.provided_mode 

162 

163 def compute_mode(self, current_mode: int, is_dir: bool) -> int: 

164 final_mode = current_mode 

165 for segment in self.segments: 

166 final_mode = segment.apply(final_mode, is_dir) 

167 return final_mode 

168 

169 

170@dataclasses.dataclass(slots=True, frozen=True) 

171class OctalMode(FileSystemMode): 

172 octal_mode: int 

173 

174 @classmethod 

175 def parse_filesystem_mode( 

176 cls, 

177 mode_raw: str, 

178 attribute_path: AttributePath, 

179 ) -> "FileSystemMode": 

180 try: 

181 mode = int(mode_raw, base=8) 

182 except ValueError as e: 

183 error_msg = 'An octal mode must be all digits between 0-7 (such as "644")' 

184 raise ManifestParseException( 

185 f"Cannot parse {attribute_path.path} as an octal mode: {error_msg}" 

186 ) from e 

187 return OctalMode(mode) 

188 

189 @property 

190 def is_octal_mode(self) -> bool: 

191 return True 

192 

193 def compute_mode(self, _current_mode: int, _is_dir: bool) -> int: 

194 return self.octal_mode 

195 

196 def __str__(self) -> str: 

197 return f"0{oct(self.octal_mode)[2:]}" 

198 

199 

200@dataclasses.dataclass(slots=True, frozen=True) 

201class _StaticFileSystemOwnerGroup: 

202 ownership_definition: OwnershipDefinition 

203 

204 @property 

205 def entity_name(self) -> str: 

206 return self.ownership_definition.entity_name 

207 

208 @property 

209 def entity_id(self) -> int: 

210 return self.ownership_definition.entity_id 

211 

212 @classmethod 

213 def from_manifest_value( 

214 cls, 

215 raw_input: str | int, 

216 attribute_path: AttributePath, 

217 ) -> "_StaticFileSystemOwnerGroup": 

218 provided_name, provided_id = _parse_ownership(raw_input, attribute_path) 

219 owner_def = cls._resolve(raw_input, provided_name, provided_id, attribute_path) 

220 if ( 220 ↛ 224line 220 didn't jump to line 224 because the condition on line 220 was never true

221 owner_def.entity_name in BAD_OWNER_NAMES 

222 or owner_def.entity_id in BAD_OWNER_IDS 

223 ): 

224 raise ManifestParseException( 

225 f'Refusing to use "{raw_input}" as {cls._owner_type()} (defined at {attribute_path.path})' 

226 f' as it resolves to "{owner_def.entity_name}:{owner_def.entity_id}" and no path should have this' 

227 f" entity as {cls._owner_type()} as it is unsafe." 

228 ) 

229 return cls(owner_def) 

230 

231 @classmethod 

232 def _resolve( 

233 cls, 

234 raw_input: str | int, 

235 provided_name: str | None, 

236 provided_id: int | None, 

237 attribute_path: AttributePath, 

238 ) -> OwnershipDefinition: 

239 table_name = cls._ownership_table_name() 

240 name_table, id_table = _load_ownership_table_from_file(table_name) 

241 name_match = ( 

242 name_table.get(provided_name) if provided_name is not None else None 

243 ) 

244 id_match = id_table.get(provided_id) if provided_id is not None else None 

245 if id_match is None and name_match is None: 245 ↛ 246line 245 didn't jump to line 246 because the condition on line 245 was never true

246 name_part = provided_name if provided_name is not None else "N/A" 

247 id_part = provided_id if provided_id is not None else "N/A" 

248 raise ManifestParseException( 

249 f'Cannot resolve "{raw_input}" as {cls._owner_type()} (from {attribute_path.path}):' 

250 f" It is not known to be a static {cls._owner_type()} from base-passwd." 

251 f' The value was interpreted as name: "{name_part}" and id: {id_part}' 

252 ) 

253 if id_match is None: 

254 assert name_match is not None 

255 return name_match 

256 if name_match is None: 256 ↛ 259line 256 didn't jump to line 259 because the condition on line 256 was always true

257 assert id_match is not None 

258 return id_match 

259 if provided_name != id_match.entity_name: 

260 raise ManifestParseException( 

261 f"Bad {cls._owner_type()} declaration: The id {provided_id} resolves to {id_match.entity_name}" 

262 f" according to base-passwd, but the packager declared to should have been {provided_name}" 

263 f" at {attribute_path.path}" 

264 ) 

265 if provided_id != name_match.entity_id: 

266 raise ManifestParseException( 

267 f"Bad {cls._owner_type} declaration: The name {provided_name} resolves to {name_match.entity_id}" 

268 f" according to base-passwd, but the packager declared to should have been {provided_id}" 

269 f" at {attribute_path.path}" 

270 ) 

271 return id_match 

272 

273 @classmethod 

274 def _owner_type(cls) -> Literal["owner", "group"]: 

275 raise NotImplementedError 

276 

277 @classmethod 

278 def _ownership_table_name(cls) -> Literal["passwd.master", "group.master"]: 

279 raise NotImplementedError 

280 

281 

282class StaticFileSystemOwner(_StaticFileSystemOwnerGroup): 

283 @classmethod 

284 def _owner_type(cls) -> Literal["owner", "group"]: 

285 return "owner" 

286 

287 @classmethod 

288 def _ownership_table_name(cls) -> Literal["passwd.master", "group.master"]: 

289 return "passwd.master" 

290 

291 

292class StaticFileSystemGroup(_StaticFileSystemOwnerGroup): 

293 @classmethod 

294 def _owner_type(cls) -> Literal["owner", "group"]: 

295 return "group" 

296 

297 @classmethod 

298 def _ownership_table_name(cls) -> Literal["passwd.master", "group.master"]: 

299 return "group.master" 

300 

301 

302@dataclasses.dataclass(slots=True, frozen=True) 

303class SymlinkTarget: 

304 raw_symlink_target: str 

305 attribute_path: AttributePath 

306 symlink_target: str 

307 

308 @classmethod 

309 def parse_symlink_target( 

310 cls, 

311 raw_symlink_target: str, 

312 attribute_path: AttributePath, 

313 substitution: Substitution, 

314 ) -> "SymlinkTarget": 

315 return SymlinkTarget( 

316 raw_symlink_target, 

317 attribute_path, 

318 substitution.substitute(raw_symlink_target, attribute_path.path), 

319 ) 

320 

321 

322class FileSystemMatchRule: 

323 @property 

324 def raw_match_rule(self) -> str: 

325 raise NotImplementedError 

326 

327 @property 

328 def attribute_path(self) -> AttributePath: 

329 raise NotImplementedError 

330 

331 @property 

332 def match_rule(self) -> MatchRule: 

333 raise NotImplementedError 

334 

335 @classmethod 

336 def parse_path_match( 

337 cls, 

338 raw_match_rule: str, 

339 attribute_path: AttributePath, 

340 parser_context: "ParserContextData", 

341 ) -> "FileSystemMatchRule": 

342 return cls.from_path_match( 

343 raw_match_rule, attribute_path, parser_context.substitution 

344 ) 

345 

346 @classmethod 

347 def from_path_match( 

348 cls, 

349 raw_match_rule: str, 

350 attribute_path: AttributePath, 

351 substitution: "Substitution", 

352 ) -> "FileSystemMatchRule": 

353 try: 

354 mr = MatchRule.from_path_or_glob( 

355 raw_match_rule, 

356 attribute_path.path, 

357 substitution=substitution, 

358 ) 

359 except ValueError as e: 

360 raise ManifestParseException( 

361 f'Could not parse "{raw_match_rule}" (defined at {attribute_path.path})' 

362 f" as a path or a glob: {e.args[0]}" 

363 ) 

364 

365 if isinstance(mr, ExactFileSystemPath): 

366 return FileSystemExactMatchRule( 

367 raw_match_rule, 

368 attribute_path, 

369 mr, 

370 ) 

371 return FileSystemGenericMatch( 

372 raw_match_rule, 

373 attribute_path, 

374 mr, 

375 ) 

376 

377 

378@dataclasses.dataclass(slots=True, frozen=True) 

379class FileSystemGenericMatch(FileSystemMatchRule): 

380 raw_match_rule: str 

381 attribute_path: AttributePath 

382 match_rule: MatchRule 

383 

384 

385@dataclasses.dataclass(slots=True, frozen=True) 

386class FileSystemExactMatchRule(FileSystemMatchRule): 

387 raw_match_rule: str 

388 attribute_path: AttributePath 

389 match_rule: ExactFileSystemPath 

390 

391 @classmethod 

392 def from_path_match( 

393 cls, 

394 raw_match_rule: str, 

395 attribute_path: AttributePath, 

396 substitution: "Substitution", 

397 ) -> "FileSystemExactMatchRule": 

398 try: 

399 normalized = _normalize_path(raw_match_rule) 

400 except ValueError as e: 

401 raise ManifestParseException( 

402 f'The path "{raw_match_rule}" provided in {attribute_path.path} should be relative to the' 

403 ' root of the package and not use any ".." or "." segments.' 

404 ) from e 

405 if normalized == ".": 405 ↛ 406line 405 didn't jump to line 406 because the condition on line 405 was never true

406 raise ManifestParseException( 

407 f'The path "{raw_match_rule}" matches a file system root and that is not a valid match' 

408 f' at "{attribute_path.path}". Please narrow the provided path.' 

409 ) 

410 mr = ExactFileSystemPath( 

411 substitution.substitute(normalized, attribute_path.path) 

412 ) 

413 if mr.path.endswith("/") and issubclass(cls, FileSystemExactNonDirMatchRule): 413 ↛ 414line 413 didn't jump to line 414 because the condition on line 413 was never true

414 raise ManifestParseException( 

415 f'The path "{raw_match_rule}" at {attribute_path.path} resolved to' 

416 f' "{mr.path}". Since the resolved path ends with a slash ("/"), this' 

417 " means only a directory can match. However, this attribute should" 

418 " match a *non*-directory" 

419 ) 

420 return cls( 

421 raw_match_rule, 

422 attribute_path, 

423 mr, 

424 ) 

425 

426 

427class FileSystemExactNonDirMatchRule(FileSystemExactMatchRule): 

428 pass 

429 

430 

431class DpkgBuildflagsCache: 

432 

433 __slots__ = ("_cache_keys",) 

434 

435 def __init__(self) -> None: 

436 self._cache_keys: dict[frozenset[tuple[str, str]], Mapping[str, str]] = {} 

437 

438 def run_dpkg_buildflags( 

439 self, 

440 env: Mapping[str, str], 

441 definition_source: str | None, 

442 ) -> Mapping[str, str]: 

443 cache_key = frozenset((k, v) for k, v in env.items() if k.startswith("DEB_")) 

444 dpkg_env = self._cache_keys.get(cache_key) 

445 if dpkg_env is not None: 

446 return dpkg_env 

447 dpkg_env = {} 

448 try: 

449 bf_output = subprocess.check_output(["dpkg-buildflags"], env=env) 

450 except FileNotFoundError: 

451 if definition_source is None: 

452 _error( 

453 "The dpkg-buildflags command was not available and is necessary to set the relevant" 

454 "env variables by default." 

455 ) 

456 _error( 

457 "The dpkg-buildflags command was not available and is necessary to set the relevant" 

458 f"env variables for the environment defined at {definition_source}." 

459 ) 

460 except subprocess.CalledProcessError as e: 

461 if definition_source is None: 

462 _error( 

463 f"The dpkg-buildflags command failed with exit code {e.returncode}. Please review the output from" 

464 f" dpkg-buildflags above to resolve the issue." 

465 ) 

466 _error( 

467 f"The dpkg-buildflags command failed with exit code {e.returncode}. Please review the output from" 

468 f" dpkg-buildflags above to resolve the issue. The environment definition that triggered this call" 

469 f" was {definition_source}" 

470 ) 

471 else: 

472 warned = False 

473 for line in bf_output.decode("utf-8").splitlines(keepends=False): 

474 if "=" not in line or line.startswith("="): 474 ↛ 475line 474 didn't jump to line 475 because the condition on line 474 was never true

475 if not warned: 

476 _warn( 

477 f"Unexpected output from dpkg-buildflags (not a K=V line): {line}" 

478 ) 

479 continue 

480 k, v = line.split("=", 1) 

481 if k.strip() != k: 481 ↛ 482line 481 didn't jump to line 482 because the condition on line 481 was never true

482 if not warned: 

483 _warn( 

484 f'Unexpected output from dpkg-buildflags (Key had spaces): "{line}"' 

485 ) 

486 continue 

487 dpkg_env[k] = v 

488 self._cache_keys[cache_key] = dpkg_env 

489 return dpkg_env 

490 

491 

492_DPKG_BUILDFLAGS_CACHE = DpkgBuildflagsCache() 

493del DpkgBuildflagsCache 

494 

495 

496class BuildEnvironmentDefinition: 

497 

498 def dpkg_buildflags_env( 

499 self, 

500 env: Mapping[str, str], 

501 definition_source: str | None, 

502 ) -> Mapping[str, str]: 

503 return _DPKG_BUILDFLAGS_CACHE.run_dpkg_buildflags(env, definition_source) 

504 

505 def log_computed_env(self, source: str, computed_env: Mapping[str, str]) -> None: 

506 _debug_log(f"Computed environment variables from {source}") 

507 for k, v in computed_env.items(): 

508 _debug_log(f" {k}={v}") 

509 

510 def update_env(self, env: MutableMapping[str, str]) -> None: 

511 dpkg_env = self.dpkg_buildflags_env(env, None) 

512 if _is_debug_log_enabled(): 

513 self.log_computed_env("dpkg-buildflags", dpkg_env) 

514 env.update(dpkg_env) 

515 

516 

517class BuildEnvironments: 

518 

519 def __init__( 

520 self, 

521 environments: dict[str, BuildEnvironmentDefinition], 

522 default_environment: BuildEnvironmentDefinition | None, 

523 ) -> None: 

524 self.environments = environments 

525 self.default_environment = default_environment