Coverage for src/debputy/packages.py: 69%
199 statements
« prev ^ index » next coverage.py v7.8.2, created at 2025-10-12 15:06 +0000
1from typing import (
2 Dict,
3 Union,
4 Tuple,
5 Optional,
6 Set,
7 cast,
8 FrozenSet,
9 overload,
10)
11from collections.abc import Mapping, Iterable
13from debian.deb822 import Deb822
14from debian.debian_support import DpkgArchTable
16from ._deb_options_profiles import DebBuildOptionsAndProfiles
17from .architecture_support import (
18 DpkgArchitectureBuildProcessValuesTable,
19 dpkg_architecture_table,
20)
21from .lsp.vendoring._deb822_repro import (
22 parse_deb822_file,
23 Deb822ParagraphElement,
24 Deb822FileElement,
25)
26from .util import DEFAULT_PACKAGE_TYPE, UDEB_PACKAGE_TYPE, _error, active_profiles_match
28_MANDATORY_BINARY_PACKAGE_FIELD = [
29 "Package",
30 "Architecture",
31]
34class DctrlParser:
36 def __init__(
37 self,
38 selected_packages: set[str] | frozenset[str],
39 excluded_packages: set[str] | frozenset[str],
40 select_arch_all: bool,
41 select_arch_any: bool,
42 dpkg_architecture_variables: None | (
43 DpkgArchitectureBuildProcessValuesTable
44 ) = None,
45 dpkg_arch_query_table: DpkgArchTable | None = None,
46 deb_options_and_profiles: DebBuildOptionsAndProfiles | None = None,
47 ignore_errors: bool = False,
48 ) -> None:
49 if dpkg_architecture_variables is None: 49 ↛ 50line 49 didn't jump to line 50 because the condition on line 49 was never true
50 dpkg_architecture_variables = dpkg_architecture_table()
51 if dpkg_arch_query_table is None: 51 ↛ 52line 51 didn't jump to line 52 because the condition on line 51 was never true
52 dpkg_arch_query_table = DpkgArchTable.load_arch_table()
53 if deb_options_and_profiles is None: 53 ↛ 54line 53 didn't jump to line 54 because the condition on line 53 was never true
54 deb_options_and_profiles = DebBuildOptionsAndProfiles.instance()
56 # If no selection option is set, then all packages are acted on (except the
57 # excluded ones)
58 if not selected_packages and not select_arch_all and not select_arch_any: 58 ↛ 59line 58 didn't jump to line 59 because the condition on line 58 was never true
59 select_arch_all = True
60 select_arch_any = True
62 self.selected_packages = selected_packages
63 self.excluded_packages = excluded_packages
64 self.select_arch_all = select_arch_all
65 self.select_arch_any = select_arch_any
66 self.dpkg_architecture_variables = dpkg_architecture_variables
67 self.dpkg_arch_query_table = dpkg_arch_query_table
68 self.deb_options_and_profiles = deb_options_and_profiles
69 self.ignore_errors = ignore_errors
71 @overload
72 def parse_source_debian_control( 72 ↛ exitline 72 didn't return from function 'parse_source_debian_control' because
73 self,
74 debian_control_lines: Iterable[str],
75 ) -> tuple[Deb822FileElement, "SourcePackage", dict[str, "BinaryPackage"]]: ...
77 @overload
78 def parse_source_debian_control( 78 ↛ exitline 78 didn't return from function 'parse_source_debian_control' because
79 self,
80 debian_control_lines: Iterable[str],
81 *,
82 ignore_errors: bool = False,
83 ) -> tuple[
84 Deb822FileElement,
85 Optional["SourcePackage"],
86 dict[str, "BinaryPackage"] | None,
87 ]: ...
89 def parse_source_debian_control(
90 self,
91 debian_control_lines: Iterable[str],
92 *,
93 ignore_errors: bool = False,
94 ) -> tuple[
95 Deb822FileElement | None,
96 Optional["SourcePackage"],
97 dict[str, "BinaryPackage"] | None,
98 ]:
99 deb822_file = parse_deb822_file(
100 debian_control_lines,
101 accept_files_with_error_tokens=ignore_errors,
102 accept_files_with_duplicated_fields=ignore_errors,
103 )
104 dctrl_paragraphs = list(deb822_file)
105 if len(dctrl_paragraphs) < 2: 105 ↛ 106line 105 didn't jump to line 106 because the condition on line 105 was never true
106 if not ignore_errors:
107 _error(
108 "debian/control must contain at least two stanza (1 Source + 1-N Package stanza)"
109 )
110 source_package = (
111 SourcePackage(dctrl_paragraphs[0]) if dctrl_paragraphs else None
112 )
113 return deb822_file, source_package, None
115 source_package = SourcePackage(dctrl_paragraphs[0])
116 bin_pkgs = []
117 for i, p in enumerate(dctrl_paragraphs[1:], 1):
118 if ignore_errors: 118 ↛ 131line 118 didn't jump to line 131 because the condition on line 118 was always true
119 if "Package" not in p:
120 continue
121 missing_field = any(f not in p for f in _MANDATORY_BINARY_PACKAGE_FIELD)
122 if missing_field:
123 # In the LSP context, it is problematic if we "add" fields as it ranges and provides invalid
124 # results. However, `debputy` also needs the mandatory fields to be there, so we clone the
125 # stanzas that `debputy` (build) will see to add missing fields.
126 copy = Deb822(p)
127 for f in _MANDATORY_BINARY_PACKAGE_FIELD:
128 if f not in p:
129 copy[f] = "unknown"
130 p = copy
131 bin_pkgs.append(
132 _create_binary_package(
133 p,
134 self.selected_packages,
135 self.excluded_packages,
136 self.select_arch_all,
137 self.select_arch_any,
138 self.dpkg_architecture_variables,
139 self.dpkg_arch_query_table,
140 self.deb_options_and_profiles,
141 i,
142 )
143 )
144 bin_pkgs_table = {p.name: p for p in bin_pkgs}
146 if not ignore_errors: 146 ↛ 147line 146 didn't jump to line 147 because the condition on line 146 was never true
147 if not self.selected_packages.issubset(bin_pkgs_table.keys()):
148 unknown = self.selected_packages - bin_pkgs_table.keys()
149 _error(
150 f"The following *selected* packages (-p) are not listed in debian/control: {sorted(unknown)}"
151 )
152 if not self.excluded_packages.issubset(bin_pkgs_table.keys()):
153 unknown = self.selected_packages - bin_pkgs_table.keys()
154 _error(
155 f"The following *excluded* packages (-N) are not listed in debian/control: {sorted(unknown)}"
156 )
158 return deb822_file, source_package, bin_pkgs_table
161def _check_package_sets(
162 provided_packages: set[str],
163 valid_package_names: set[str],
164 option_name: str,
165) -> None:
166 # SonarLint proposes to use `provided_packages > valid_package_names`, which is valid for boolean
167 # logic, but not for set logic. We want to assert that provided_packages is a proper subset
168 # of valid_package_names. The rewrite would cause no errors for {'foo'} > {'bar'} - in set logic,
169 # neither is a superset / subset of the other, but we want an error for this case.
170 #
171 # Bug filed:
172 # https://community.sonarsource.com/t/sonarlint-python-s1940-rule-does-not-seem-to-take-set-logic-into-account/79718
173 if not (provided_packages <= valid_package_names):
174 non_existing_packages = sorted(provided_packages - valid_package_names)
175 invalid_package_list = ", ".join(non_existing_packages)
176 msg = (
177 f"Invalid package names passed to {option_name}: {invalid_package_list}: "
178 f'Valid package names are: {", ".join(valid_package_names)}'
179 )
180 _error(msg)
183def _create_binary_package(
184 paragraph: Deb822ParagraphElement | dict[str, str],
185 selected_packages: set[str] | frozenset[str],
186 excluded_packages: set[str] | frozenset[str],
187 select_arch_all: bool,
188 select_arch_any: bool,
189 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
190 dpkg_arch_query_table: DpkgArchTable,
191 build_env: DebBuildOptionsAndProfiles,
192 paragraph_index: int,
193) -> "BinaryPackage":
194 try:
195 package_name = paragraph["Package"]
196 except KeyError:
197 _error(f'Missing mandatory field "Package" in stanza number {paragraph_index}')
198 # The raise is there to help PyCharm type-checking (which fails at "NoReturn")
199 raise
201 for mandatory_field in _MANDATORY_BINARY_PACKAGE_FIELD:
202 if mandatory_field not in paragraph: 202 ↛ 203line 202 didn't jump to line 203 because the condition on line 202 was never true
203 _error(
204 f'Missing mandatory field "{mandatory_field}" for binary package {package_name}'
205 f" (stanza number {paragraph_index})"
206 )
208 architecture = paragraph["Architecture"]
210 if paragraph_index < 1: 210 ↛ 211line 210 didn't jump to line 211 because the condition on line 210 was never true
211 raise ValueError("stanza index must be 1-indexed (1, 2, ...)")
212 is_main_package = paragraph_index == 1
214 if package_name in excluded_packages: 214 ↛ 215line 214 didn't jump to line 215 because the condition on line 214 was never true
215 should_act_on = False
216 elif package_name in selected_packages: 216 ↛ 217line 216 didn't jump to line 217 because the condition on line 216 was never true
217 should_act_on = True
218 elif architecture == "all":
219 should_act_on = select_arch_all
220 else:
221 should_act_on = select_arch_any
223 profiles_raw = paragraph.get("Build-Profiles", "").strip()
224 if should_act_on and profiles_raw: 224 ↛ 225line 224 didn't jump to line 225 because the condition on line 224 was never true
225 try:
226 should_act_on = active_profiles_match(
227 profiles_raw, build_env.deb_build_profiles
228 )
229 except ValueError as e:
230 _error(f"Invalid Build-Profiles field for {package_name}: {e.args[0]}")
232 return BinaryPackage(
233 paragraph,
234 dpkg_architecture_variables,
235 dpkg_arch_query_table,
236 should_be_acted_on=should_act_on,
237 is_main_package=is_main_package,
238 )
241def _check_binary_arch(
242 arch_table: DpkgArchTable,
243 binary_arch: str,
244 declared_arch: str,
245) -> bool:
246 if binary_arch == "all":
247 return True
248 arch_wildcards = declared_arch.split()
249 for arch_wildcard in arch_wildcards:
250 if arch_table.matches_architecture(binary_arch, arch_wildcard):
251 return True
252 return False
255class BinaryPackage:
256 __slots__ = [
257 "_package_fields",
258 "_dbgsym_binary_package",
259 "_should_be_acted_on",
260 "_dpkg_architecture_variables",
261 "_declared_arch_matches_output_arch",
262 "_is_main_package",
263 "_substvars",
264 "_maintscript_snippets",
265 ]
267 def __init__(
268 self,
269 fields: Mapping[str, str] | Deb822ParagraphElement,
270 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
271 dpkg_arch_query: DpkgArchTable,
272 *,
273 is_main_package: bool = False,
274 should_be_acted_on: bool = True,
275 ) -> None:
276 super().__init__()
277 # Typing-wise, Deb822ParagraphElement is *not* a Mapping[str, str] but it behaves enough
278 # like one that we rely on it and just cast it.
279 self._package_fields = cast("Mapping[str, str]", fields)
280 self._dbgsym_binary_package = None
281 self._should_be_acted_on = should_be_acted_on
282 self._dpkg_architecture_variables = dpkg_architecture_variables
283 self._is_main_package = is_main_package
284 self._declared_arch_matches_output_arch = _check_binary_arch(
285 dpkg_arch_query, self.resolved_architecture, self.declared_architecture
286 )
288 @property
289 def name(self) -> str:
290 return self.fields["Package"]
292 @property
293 def archive_section(self) -> str:
294 value = self.fields.get("Section")
295 if value is None: 295 ↛ 296line 295 didn't jump to line 296 because the condition on line 295 was never true
296 return "Unknown"
297 return value
299 @property
300 def archive_component(self) -> str:
301 component = ""
302 section = self.archive_section
303 if "/" in section:
304 component = section.rsplit("/", 1)[0]
305 # The "main" component is always shortened to ""
306 if component == "main":
307 component = ""
308 return component
310 @property
311 def is_essential(self) -> bool:
312 return self._package_fields.get("Essential") == "yes"
314 @property
315 def is_udeb(self) -> bool:
316 return self.package_type == UDEB_PACKAGE_TYPE
318 @property
319 def should_be_acted_on(self) -> bool:
320 return self._should_be_acted_on and self._declared_arch_matches_output_arch
322 @property
323 def fields(self) -> Mapping[str, str]:
324 return self._package_fields
326 @property
327 def resolved_architecture(self) -> str:
328 arch = self.declared_architecture
329 if arch == "all":
330 return arch
331 if self._x_dh_build_for_type == "target": 331 ↛ 332line 331 didn't jump to line 332 because the condition on line 331 was never true
332 return self._dpkg_architecture_variables["DEB_TARGET_ARCH"]
333 return self._dpkg_architecture_variables.current_host_arch
335 def package_deb_architecture_variable(self, variable_suffix: str) -> str:
336 if self._x_dh_build_for_type == "target": 336 ↛ 337line 336 didn't jump to line 337 because the condition on line 336 was never true
337 return self._dpkg_architecture_variables[f"DEB_TARGET_{variable_suffix}"]
338 return self._dpkg_architecture_variables[f"DEB_HOST_{variable_suffix}"]
340 @property
341 def deb_multiarch(self) -> str:
342 return self.package_deb_architecture_variable("MULTIARCH")
344 @property
345 def _x_dh_build_for_type(self) -> str:
346 v = self._package_fields.get("X-DH-Build-For-Type")
347 if v is None: 347 ↛ 349line 347 didn't jump to line 349 because the condition on line 347 was always true
348 return "host"
349 return v.lower()
351 @property
352 def package_type(self) -> str:
353 """Short for Package-Type (with proper default if absent)"""
354 v = self.fields.get("Package-Type")
355 if v is None:
356 return DEFAULT_PACKAGE_TYPE
357 return v
359 @property
360 def is_main_package(self) -> bool:
361 return self._is_main_package
363 def cross_command(self, command: str) -> str:
364 arch_table = self._dpkg_architecture_variables
365 if self._x_dh_build_for_type == "target":
366 target_gnu_type = arch_table["DEB_TARGET_GNU_TYPE"]
367 if arch_table["DEB_HOST_GNU_TYPE"] != target_gnu_type:
368 return f"{target_gnu_type}-{command}"
369 if arch_table.is_cross_compiling:
370 return f"{arch_table['DEB_HOST_GNU_TYPE']}-{command}"
371 return command
373 @property
374 def declared_architecture(self) -> str:
375 return self.fields["Architecture"]
377 @property
378 def is_arch_all(self) -> bool:
379 return self.declared_architecture == "all"
382class SourcePackage:
383 __slots__ = ("_package_fields",)
385 def __init__(self, fields: Mapping[str, str] | Deb822ParagraphElement):
386 # Typing-wise, Deb822ParagraphElement is *not* a Mapping[str, str] but it behaves enough
387 # like one that we rely on it and just cast it.
388 self._package_fields = cast("Mapping[str, str]", fields)
390 @property
391 def fields(self) -> Mapping[str, str]:
392 return self._package_fields
394 @property
395 def name(self) -> str:
396 return self._package_fields["Source"]