Coverage for src/debputy/packages.py: 69%

199 statements  

« prev     ^ index     » next       coverage.py v7.8.2, created at 2025-10-12 15:06 +0000

1from typing import ( 

2 Dict, 

3 Union, 

4 Tuple, 

5 Optional, 

6 Set, 

7 cast, 

8 FrozenSet, 

9 overload, 

10) 

11from collections.abc import Mapping, Iterable 

12 

13from debian.deb822 import Deb822 

14from debian.debian_support import DpkgArchTable 

15 

16from ._deb_options_profiles import DebBuildOptionsAndProfiles 

17from .architecture_support import ( 

18 DpkgArchitectureBuildProcessValuesTable, 

19 dpkg_architecture_table, 

20) 

21from .lsp.vendoring._deb822_repro import ( 

22 parse_deb822_file, 

23 Deb822ParagraphElement, 

24 Deb822FileElement, 

25) 

26from .util import DEFAULT_PACKAGE_TYPE, UDEB_PACKAGE_TYPE, _error, active_profiles_match 

27 

28_MANDATORY_BINARY_PACKAGE_FIELD = [ 

29 "Package", 

30 "Architecture", 

31] 

32 

33 

34class DctrlParser: 

35 

36 def __init__( 

37 self, 

38 selected_packages: set[str] | frozenset[str], 

39 excluded_packages: set[str] | frozenset[str], 

40 select_arch_all: bool, 

41 select_arch_any: bool, 

42 dpkg_architecture_variables: None | ( 

43 DpkgArchitectureBuildProcessValuesTable 

44 ) = None, 

45 dpkg_arch_query_table: DpkgArchTable | None = None, 

46 deb_options_and_profiles: DebBuildOptionsAndProfiles | None = None, 

47 ignore_errors: bool = False, 

48 ) -> None: 

49 if dpkg_architecture_variables is None: 49 ↛ 50line 49 didn't jump to line 50 because the condition on line 49 was never true

50 dpkg_architecture_variables = dpkg_architecture_table() 

51 if dpkg_arch_query_table is None: 51 ↛ 52line 51 didn't jump to line 52 because the condition on line 51 was never true

52 dpkg_arch_query_table = DpkgArchTable.load_arch_table() 

53 if deb_options_and_profiles is None: 53 ↛ 54line 53 didn't jump to line 54 because the condition on line 53 was never true

54 deb_options_and_profiles = DebBuildOptionsAndProfiles.instance() 

55 

56 # If no selection option is set, then all packages are acted on (except the 

57 # excluded ones) 

58 if not selected_packages and not select_arch_all and not select_arch_any: 58 ↛ 59line 58 didn't jump to line 59 because the condition on line 58 was never true

59 select_arch_all = True 

60 select_arch_any = True 

61 

62 self.selected_packages = selected_packages 

63 self.excluded_packages = excluded_packages 

64 self.select_arch_all = select_arch_all 

65 self.select_arch_any = select_arch_any 

66 self.dpkg_architecture_variables = dpkg_architecture_variables 

67 self.dpkg_arch_query_table = dpkg_arch_query_table 

68 self.deb_options_and_profiles = deb_options_and_profiles 

69 self.ignore_errors = ignore_errors 

70 

71 @overload 

72 def parse_source_debian_control( 72 ↛ exitline 72 didn't return from function 'parse_source_debian_control' because

73 self, 

74 debian_control_lines: Iterable[str], 

75 ) -> tuple[Deb822FileElement, "SourcePackage", dict[str, "BinaryPackage"]]: ... 

76 

77 @overload 

78 def parse_source_debian_control( 78 ↛ exitline 78 didn't return from function 'parse_source_debian_control' because

79 self, 

80 debian_control_lines: Iterable[str], 

81 *, 

82 ignore_errors: bool = False, 

83 ) -> tuple[ 

84 Deb822FileElement, 

85 Optional["SourcePackage"], 

86 dict[str, "BinaryPackage"] | None, 

87 ]: ... 

88 

89 def parse_source_debian_control( 

90 self, 

91 debian_control_lines: Iterable[str], 

92 *, 

93 ignore_errors: bool = False, 

94 ) -> tuple[ 

95 Deb822FileElement | None, 

96 Optional["SourcePackage"], 

97 dict[str, "BinaryPackage"] | None, 

98 ]: 

99 deb822_file = parse_deb822_file( 

100 debian_control_lines, 

101 accept_files_with_error_tokens=ignore_errors, 

102 accept_files_with_duplicated_fields=ignore_errors, 

103 ) 

104 dctrl_paragraphs = list(deb822_file) 

105 if len(dctrl_paragraphs) < 2: 105 ↛ 106line 105 didn't jump to line 106 because the condition on line 105 was never true

106 if not ignore_errors: 

107 _error( 

108 "debian/control must contain at least two stanza (1 Source + 1-N Package stanza)" 

109 ) 

110 source_package = ( 

111 SourcePackage(dctrl_paragraphs[0]) if dctrl_paragraphs else None 

112 ) 

113 return deb822_file, source_package, None 

114 

115 source_package = SourcePackage(dctrl_paragraphs[0]) 

116 bin_pkgs = [] 

117 for i, p in enumerate(dctrl_paragraphs[1:], 1): 

118 if ignore_errors: 118 ↛ 131line 118 didn't jump to line 131 because the condition on line 118 was always true

119 if "Package" not in p: 

120 continue 

121 missing_field = any(f not in p for f in _MANDATORY_BINARY_PACKAGE_FIELD) 

122 if missing_field: 

123 # In the LSP context, it is problematic if we "add" fields as it ranges and provides invalid 

124 # results. However, `debputy` also needs the mandatory fields to be there, so we clone the 

125 # stanzas that `debputy` (build) will see to add missing fields. 

126 copy = Deb822(p) 

127 for f in _MANDATORY_BINARY_PACKAGE_FIELD: 

128 if f not in p: 

129 copy[f] = "unknown" 

130 p = copy 

131 bin_pkgs.append( 

132 _create_binary_package( 

133 p, 

134 self.selected_packages, 

135 self.excluded_packages, 

136 self.select_arch_all, 

137 self.select_arch_any, 

138 self.dpkg_architecture_variables, 

139 self.dpkg_arch_query_table, 

140 self.deb_options_and_profiles, 

141 i, 

142 ) 

143 ) 

144 bin_pkgs_table = {p.name: p for p in bin_pkgs} 

145 

146 if not ignore_errors: 146 ↛ 147line 146 didn't jump to line 147 because the condition on line 146 was never true

147 if not self.selected_packages.issubset(bin_pkgs_table.keys()): 

148 unknown = self.selected_packages - bin_pkgs_table.keys() 

149 _error( 

150 f"The following *selected* packages (-p) are not listed in debian/control: {sorted(unknown)}" 

151 ) 

152 if not self.excluded_packages.issubset(bin_pkgs_table.keys()): 

153 unknown = self.selected_packages - bin_pkgs_table.keys() 

154 _error( 

155 f"The following *excluded* packages (-N) are not listed in debian/control: {sorted(unknown)}" 

156 ) 

157 

158 return deb822_file, source_package, bin_pkgs_table 

159 

160 

161def _check_package_sets( 

162 provided_packages: set[str], 

163 valid_package_names: set[str], 

164 option_name: str, 

165) -> None: 

166 # SonarLint proposes to use `provided_packages > valid_package_names`, which is valid for boolean 

167 # logic, but not for set logic. We want to assert that provided_packages is a proper subset 

168 # of valid_package_names. The rewrite would cause no errors for {'foo'} > {'bar'} - in set logic, 

169 # neither is a superset / subset of the other, but we want an error for this case. 

170 # 

171 # Bug filed: 

172 # https://community.sonarsource.com/t/sonarlint-python-s1940-rule-does-not-seem-to-take-set-logic-into-account/79718 

173 if not (provided_packages <= valid_package_names): 

174 non_existing_packages = sorted(provided_packages - valid_package_names) 

175 invalid_package_list = ", ".join(non_existing_packages) 

176 msg = ( 

177 f"Invalid package names passed to {option_name}: {invalid_package_list}: " 

178 f'Valid package names are: {", ".join(valid_package_names)}' 

179 ) 

180 _error(msg) 

181 

182 

183def _create_binary_package( 

184 paragraph: Deb822ParagraphElement | dict[str, str], 

185 selected_packages: set[str] | frozenset[str], 

186 excluded_packages: set[str] | frozenset[str], 

187 select_arch_all: bool, 

188 select_arch_any: bool, 

189 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable, 

190 dpkg_arch_query_table: DpkgArchTable, 

191 build_env: DebBuildOptionsAndProfiles, 

192 paragraph_index: int, 

193) -> "BinaryPackage": 

194 try: 

195 package_name = paragraph["Package"] 

196 except KeyError: 

197 _error(f'Missing mandatory field "Package" in stanza number {paragraph_index}') 

198 # The raise is there to help PyCharm type-checking (which fails at "NoReturn") 

199 raise 

200 

201 for mandatory_field in _MANDATORY_BINARY_PACKAGE_FIELD: 

202 if mandatory_field not in paragraph: 202 ↛ 203line 202 didn't jump to line 203 because the condition on line 202 was never true

203 _error( 

204 f'Missing mandatory field "{mandatory_field}" for binary package {package_name}' 

205 f" (stanza number {paragraph_index})" 

206 ) 

207 

208 architecture = paragraph["Architecture"] 

209 

210 if paragraph_index < 1: 210 ↛ 211line 210 didn't jump to line 211 because the condition on line 210 was never true

211 raise ValueError("stanza index must be 1-indexed (1, 2, ...)") 

212 is_main_package = paragraph_index == 1 

213 

214 if package_name in excluded_packages: 214 ↛ 215line 214 didn't jump to line 215 because the condition on line 214 was never true

215 should_act_on = False 

216 elif package_name in selected_packages: 216 ↛ 217line 216 didn't jump to line 217 because the condition on line 216 was never true

217 should_act_on = True 

218 elif architecture == "all": 

219 should_act_on = select_arch_all 

220 else: 

221 should_act_on = select_arch_any 

222 

223 profiles_raw = paragraph.get("Build-Profiles", "").strip() 

224 if should_act_on and profiles_raw: 224 ↛ 225line 224 didn't jump to line 225 because the condition on line 224 was never true

225 try: 

226 should_act_on = active_profiles_match( 

227 profiles_raw, build_env.deb_build_profiles 

228 ) 

229 except ValueError as e: 

230 _error(f"Invalid Build-Profiles field for {package_name}: {e.args[0]}") 

231 

232 return BinaryPackage( 

233 paragraph, 

234 dpkg_architecture_variables, 

235 dpkg_arch_query_table, 

236 should_be_acted_on=should_act_on, 

237 is_main_package=is_main_package, 

238 ) 

239 

240 

241def _check_binary_arch( 

242 arch_table: DpkgArchTable, 

243 binary_arch: str, 

244 declared_arch: str, 

245) -> bool: 

246 if binary_arch == "all": 

247 return True 

248 arch_wildcards = declared_arch.split() 

249 for arch_wildcard in arch_wildcards: 

250 if arch_table.matches_architecture(binary_arch, arch_wildcard): 

251 return True 

252 return False 

253 

254 

255class BinaryPackage: 

256 __slots__ = [ 

257 "_package_fields", 

258 "_dbgsym_binary_package", 

259 "_should_be_acted_on", 

260 "_dpkg_architecture_variables", 

261 "_declared_arch_matches_output_arch", 

262 "_is_main_package", 

263 "_substvars", 

264 "_maintscript_snippets", 

265 ] 

266 

267 def __init__( 

268 self, 

269 fields: Mapping[str, str] | Deb822ParagraphElement, 

270 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable, 

271 dpkg_arch_query: DpkgArchTable, 

272 *, 

273 is_main_package: bool = False, 

274 should_be_acted_on: bool = True, 

275 ) -> None: 

276 super().__init__() 

277 # Typing-wise, Deb822ParagraphElement is *not* a Mapping[str, str] but it behaves enough 

278 # like one that we rely on it and just cast it. 

279 self._package_fields = cast("Mapping[str, str]", fields) 

280 self._dbgsym_binary_package = None 

281 self._should_be_acted_on = should_be_acted_on 

282 self._dpkg_architecture_variables = dpkg_architecture_variables 

283 self._is_main_package = is_main_package 

284 self._declared_arch_matches_output_arch = _check_binary_arch( 

285 dpkg_arch_query, self.resolved_architecture, self.declared_architecture 

286 ) 

287 

288 @property 

289 def name(self) -> str: 

290 return self.fields["Package"] 

291 

292 @property 

293 def archive_section(self) -> str: 

294 value = self.fields.get("Section") 

295 if value is None: 295 ↛ 296line 295 didn't jump to line 296 because the condition on line 295 was never true

296 return "Unknown" 

297 return value 

298 

299 @property 

300 def archive_component(self) -> str: 

301 component = "" 

302 section = self.archive_section 

303 if "/" in section: 

304 component = section.rsplit("/", 1)[0] 

305 # The "main" component is always shortened to "" 

306 if component == "main": 

307 component = "" 

308 return component 

309 

310 @property 

311 def is_essential(self) -> bool: 

312 return self._package_fields.get("Essential") == "yes" 

313 

314 @property 

315 def is_udeb(self) -> bool: 

316 return self.package_type == UDEB_PACKAGE_TYPE 

317 

318 @property 

319 def should_be_acted_on(self) -> bool: 

320 return self._should_be_acted_on and self._declared_arch_matches_output_arch 

321 

322 @property 

323 def fields(self) -> Mapping[str, str]: 

324 return self._package_fields 

325 

326 @property 

327 def resolved_architecture(self) -> str: 

328 arch = self.declared_architecture 

329 if arch == "all": 

330 return arch 

331 if self._x_dh_build_for_type == "target": 331 ↛ 332line 331 didn't jump to line 332 because the condition on line 331 was never true

332 return self._dpkg_architecture_variables["DEB_TARGET_ARCH"] 

333 return self._dpkg_architecture_variables.current_host_arch 

334 

335 def package_deb_architecture_variable(self, variable_suffix: str) -> str: 

336 if self._x_dh_build_for_type == "target": 336 ↛ 337line 336 didn't jump to line 337 because the condition on line 336 was never true

337 return self._dpkg_architecture_variables[f"DEB_TARGET_{variable_suffix}"] 

338 return self._dpkg_architecture_variables[f"DEB_HOST_{variable_suffix}"] 

339 

340 @property 

341 def deb_multiarch(self) -> str: 

342 return self.package_deb_architecture_variable("MULTIARCH") 

343 

344 @property 

345 def _x_dh_build_for_type(self) -> str: 

346 v = self._package_fields.get("X-DH-Build-For-Type") 

347 if v is None: 347 ↛ 349line 347 didn't jump to line 349 because the condition on line 347 was always true

348 return "host" 

349 return v.lower() 

350 

351 @property 

352 def package_type(self) -> str: 

353 """Short for Package-Type (with proper default if absent)""" 

354 v = self.fields.get("Package-Type") 

355 if v is None: 

356 return DEFAULT_PACKAGE_TYPE 

357 return v 

358 

359 @property 

360 def is_main_package(self) -> bool: 

361 return self._is_main_package 

362 

363 def cross_command(self, command: str) -> str: 

364 arch_table = self._dpkg_architecture_variables 

365 if self._x_dh_build_for_type == "target": 

366 target_gnu_type = arch_table["DEB_TARGET_GNU_TYPE"] 

367 if arch_table["DEB_HOST_GNU_TYPE"] != target_gnu_type: 

368 return f"{target_gnu_type}-{command}" 

369 if arch_table.is_cross_compiling: 

370 return f"{arch_table['DEB_HOST_GNU_TYPE']}-{command}" 

371 return command 

372 

373 @property 

374 def declared_architecture(self) -> str: 

375 return self.fields["Architecture"] 

376 

377 @property 

378 def is_arch_all(self) -> bool: 

379 return self.declared_architecture == "all" 

380 

381 

382class SourcePackage: 

383 __slots__ = ("_package_fields",) 

384 

385 def __init__(self, fields: Mapping[str, str] | Deb822ParagraphElement): 

386 # Typing-wise, Deb822ParagraphElement is *not* a Mapping[str, str] but it behaves enough 

387 # like one that we rely on it and just cast it. 

388 self._package_fields = cast("Mapping[str, str]", fields) 

389 

390 @property 

391 def fields(self) -> Mapping[str, str]: 

392 return self._package_fields 

393 

394 @property 

395 def name(self) -> str: 

396 return self._package_fields["Source"]