Coverage for src/debputy/packages.py: 72%
198 statements
« prev ^ index » next coverage.py v7.6.0, created at 2025-01-27 13:59 +0000
1from typing import (
2 Dict,
3 Union,
4 Tuple,
5 Optional,
6 Set,
7 cast,
8 Mapping,
9 FrozenSet,
10 Iterable,
11 overload,
12)
14from debian.deb822 import Deb822
15from debian.debian_support import DpkgArchTable
17from ._deb_options_profiles import DebBuildOptionsAndProfiles
18from .architecture_support import (
19 DpkgArchitectureBuildProcessValuesTable,
20 dpkg_architecture_table,
21)
22from .lsp.vendoring._deb822_repro import (
23 parse_deb822_file,
24 Deb822ParagraphElement,
25 Deb822FileElement,
26)
27from .util import DEFAULT_PACKAGE_TYPE, UDEB_PACKAGE_TYPE, _error, active_profiles_match
29_MANDATORY_BINARY_PACKAGE_FIELD = [
30 "Package",
31 "Architecture",
32]
35class DctrlParser:
37 def __init__(
38 self,
39 selected_packages: Union[Set[str], FrozenSet[str]],
40 excluded_packages: Union[Set[str], FrozenSet[str]],
41 select_arch_all: bool,
42 select_arch_any: bool,
43 dpkg_architecture_variables: Optional[
44 DpkgArchitectureBuildProcessValuesTable
45 ] = None,
46 dpkg_arch_query_table: Optional[DpkgArchTable] = None,
47 deb_options_and_profiles: Optional[DebBuildOptionsAndProfiles] = None,
48 ignore_errors: bool = False,
49 ) -> None:
50 if dpkg_architecture_variables is None: 50 ↛ 51line 50 didn't jump to line 51 because the condition on line 50 was never true
51 dpkg_architecture_variables = dpkg_architecture_table()
52 if dpkg_arch_query_table is None: 52 ↛ 53line 52 didn't jump to line 53 because the condition on line 52 was never true
53 dpkg_arch_query_table = DpkgArchTable.load_arch_table()
54 if deb_options_and_profiles is None: 54 ↛ 55line 54 didn't jump to line 55 because the condition on line 54 was never true
55 deb_options_and_profiles = DebBuildOptionsAndProfiles.instance()
57 # If no selection option is set, then all packages are acted on (except the
58 # excluded ones)
59 if not selected_packages and not select_arch_all and not select_arch_any: 59 ↛ 60line 59 didn't jump to line 60 because the condition on line 59 was never true
60 select_arch_all = True
61 select_arch_any = True
63 self.selected_packages = selected_packages
64 self.excluded_packages = excluded_packages
65 self.select_arch_all = select_arch_all
66 self.select_arch_any = select_arch_any
67 self.dpkg_architecture_variables = dpkg_architecture_variables
68 self.dpkg_arch_query_table = dpkg_arch_query_table
69 self.deb_options_and_profiles = deb_options_and_profiles
70 self.ignore_errors = ignore_errors
72 @overload
73 def parse_source_debian_control( 73 ↛ exitline 73 didn't jump to the function exit
74 self,
75 debian_control_lines: Iterable[str],
76 ) -> Tuple[Deb822FileElement, "SourcePackage", Dict[str, "BinaryPackage"]]: ...
78 @overload
79 def parse_source_debian_control( 79 ↛ exitline 79 didn't jump to the function exit
80 self,
81 debian_control_lines: Iterable[str],
82 *,
83 ignore_errors: bool = False,
84 ) -> Tuple[
85 Deb822FileElement,
86 Optional["SourcePackage"],
87 Optional[Dict[str, "BinaryPackage"]],
88 ]: ...
90 def parse_source_debian_control(
91 self,
92 debian_control_lines: Iterable[str],
93 *,
94 ignore_errors: bool = False,
95 ) -> Tuple[
96 Optional[Deb822FileElement],
97 Optional["SourcePackage"],
98 Optional[Dict[str, "BinaryPackage"]],
99 ]:
100 deb822_file = parse_deb822_file(
101 debian_control_lines,
102 accept_files_with_error_tokens=ignore_errors,
103 accept_files_with_duplicated_fields=ignore_errors,
104 )
105 dctrl_paragraphs = list(deb822_file)
106 if len(dctrl_paragraphs) < 2: 106 ↛ 107line 106 didn't jump to line 107 because the condition on line 106 was never true
107 if not ignore_errors:
108 _error(
109 "debian/control must contain at least two stanza (1 Source + 1-N Package stanza)"
110 )
111 source_package = (
112 SourcePackage(dctrl_paragraphs[0]) if dctrl_paragraphs else None
113 )
114 return deb822_file, source_package, None
116 source_package = SourcePackage(dctrl_paragraphs[0])
117 bin_pkgs = []
118 for i, p in enumerate(dctrl_paragraphs[1:], 1):
119 if ignore_errors: 119 ↛ 132line 119 didn't jump to line 132 because the condition on line 119 was always true
120 if "Package" not in p:
121 continue
122 missing_field = any(f not in p for f in _MANDATORY_BINARY_PACKAGE_FIELD)
123 if missing_field:
124 # In the LSP context, it is problematic if we "add" fields as it ranges and provides invalid
125 # results. However, `debputy` also needs the mandatory fields to be there, so we clone the
126 # stanzas that `debputy` (build) will see to add missing fields.
127 copy = Deb822(p)
128 for f in _MANDATORY_BINARY_PACKAGE_FIELD:
129 if f not in p:
130 copy[f] = "unknown"
131 p = copy
132 bin_pkgs.append(
133 _create_binary_package(
134 p,
135 self.selected_packages,
136 self.excluded_packages,
137 self.select_arch_all,
138 self.select_arch_any,
139 self.dpkg_architecture_variables,
140 self.dpkg_arch_query_table,
141 self.deb_options_and_profiles,
142 i,
143 )
144 )
145 bin_pkgs_table = {p.name: p for p in bin_pkgs}
147 if not ignore_errors: 147 ↛ 148line 147 didn't jump to line 148 because the condition on line 147 was never true
148 if not self.selected_packages.issubset(bin_pkgs_table.keys()):
149 unknown = self.selected_packages - bin_pkgs_table.keys()
150 _error(
151 f"The following *selected* packages (-p) are not listed in debian/control: {sorted(unknown)}"
152 )
153 if not self.excluded_packages.issubset(bin_pkgs_table.keys()):
154 unknown = self.selected_packages - bin_pkgs_table.keys()
155 _error(
156 f"The following *excluded* packages (-N) are not listed in debian/control: {sorted(unknown)}"
157 )
159 return deb822_file, source_package, bin_pkgs_table
162def _check_package_sets(
163 provided_packages: Set[str],
164 valid_package_names: Set[str],
165 option_name: str,
166) -> None:
167 # SonarLint proposes to use `provided_packages > valid_package_names`, which is valid for boolean
168 # logic, but not for set logic. We want to assert that provided_packages is a proper subset
169 # of valid_package_names. The rewrite would cause no errors for {'foo'} > {'bar'} - in set logic,
170 # neither is a superset / subset of the other, but we want an error for this case.
171 #
172 # Bug filed:
173 # https://community.sonarsource.com/t/sonarlint-python-s1940-rule-does-not-seem-to-take-set-logic-into-account/79718
174 if not (provided_packages <= valid_package_names):
175 non_existing_packages = sorted(provided_packages - valid_package_names)
176 invalid_package_list = ", ".join(non_existing_packages)
177 msg = (
178 f"Invalid package names passed to {option_name}: {invalid_package_list}: "
179 f'Valid package names are: {", ".join(valid_package_names)}'
180 )
181 _error(msg)
184def _create_binary_package(
185 paragraph: Union[Deb822ParagraphElement, Dict[str, str]],
186 selected_packages: Union[Set[str], FrozenSet[str]],
187 excluded_packages: Union[Set[str], FrozenSet[str]],
188 select_arch_all: bool,
189 select_arch_any: bool,
190 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
191 dpkg_arch_query_table: DpkgArchTable,
192 build_env: DebBuildOptionsAndProfiles,
193 paragraph_index: int,
194) -> "BinaryPackage":
195 try:
196 package_name = paragraph["Package"]
197 except KeyError:
198 _error(f'Missing mandatory field "Package" in stanza number {paragraph_index}')
199 # The raise is there to help PyCharm type-checking (which fails at "NoReturn")
200 raise
202 for mandatory_field in _MANDATORY_BINARY_PACKAGE_FIELD:
203 if mandatory_field not in paragraph: 203 ↛ 204line 203 didn't jump to line 204 because the condition on line 203 was never true
204 _error(
205 f'Missing mandatory field "{mandatory_field}" for binary package {package_name}'
206 f" (stanza number {paragraph_index})"
207 )
209 architecture = paragraph["Architecture"]
211 if paragraph_index < 1: 211 ↛ 212line 211 didn't jump to line 212 because the condition on line 211 was never true
212 raise ValueError("stanza index must be 1-indexed (1, 2, ...)")
213 is_main_package = paragraph_index == 1
215 if package_name in excluded_packages: 215 ↛ 216line 215 didn't jump to line 216 because the condition on line 215 was never true
216 should_act_on = False
217 elif package_name in selected_packages: 217 ↛ 218line 217 didn't jump to line 218 because the condition on line 217 was never true
218 should_act_on = True
219 elif architecture == "all":
220 should_act_on = select_arch_all
221 else:
222 should_act_on = select_arch_any
224 profiles_raw = paragraph.get("Build-Profiles", "").strip()
225 if should_act_on and profiles_raw: 225 ↛ 226line 225 didn't jump to line 226 because the condition on line 225 was never true
226 try:
227 should_act_on = active_profiles_match(
228 profiles_raw, build_env.deb_build_profiles
229 )
230 except ValueError as e:
231 _error(f"Invalid Build-Profiles field for {package_name}: {e.args[0]}")
233 return BinaryPackage(
234 paragraph,
235 dpkg_architecture_variables,
236 dpkg_arch_query_table,
237 should_be_acted_on=should_act_on,
238 is_main_package=is_main_package,
239 )
242def _check_binary_arch(
243 arch_table: DpkgArchTable,
244 binary_arch: str,
245 declared_arch: str,
246) -> bool:
247 if binary_arch == "all":
248 return True
249 arch_wildcards = declared_arch.split()
250 for arch_wildcard in arch_wildcards:
251 if arch_table.matches_architecture(binary_arch, arch_wildcard):
252 return True
253 return False
256class BinaryPackage:
257 __slots__ = [
258 "_package_fields",
259 "_dbgsym_binary_package",
260 "_should_be_acted_on",
261 "_dpkg_architecture_variables",
262 "_declared_arch_matches_output_arch",
263 "_is_main_package",
264 "_substvars",
265 "_maintscript_snippets",
266 ]
268 def __init__(
269 self,
270 fields: Union[Mapping[str, str], Deb822ParagraphElement],
271 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
272 dpkg_arch_query: DpkgArchTable,
273 *,
274 is_main_package: bool = False,
275 should_be_acted_on: bool = True,
276 ) -> None:
277 super(BinaryPackage, self).__init__()
278 # Typing-wise, Deb822ParagraphElement is *not* a Mapping[str, str] but it behaves enough
279 # like one that we rely on it and just cast it.
280 self._package_fields = cast("Mapping[str, str]", fields)
281 self._dbgsym_binary_package = None
282 self._should_be_acted_on = should_be_acted_on
283 self._dpkg_architecture_variables = dpkg_architecture_variables
284 self._is_main_package = is_main_package
285 self._declared_arch_matches_output_arch = _check_binary_arch(
286 dpkg_arch_query, self.resolved_architecture, self.declared_architecture
287 )
289 @property
290 def name(self) -> str:
291 return self.fields["Package"]
293 @property
294 def archive_section(self) -> str:
295 value = self.fields.get("Section")
296 if value is None: 296 ↛ 297line 296 didn't jump to line 297 because the condition on line 296 was never true
297 return "Unknown"
298 return value
300 @property
301 def archive_component(self) -> str:
302 component = ""
303 section = self.archive_section
304 if "/" in section:
305 component = section.rsplit("/", 1)[0]
306 # The "main" component is always shortened to ""
307 if component == "main":
308 component = ""
309 return component
311 @property
312 def is_essential(self) -> bool:
313 return self._package_fields.get("Essential") == "yes"
315 @property
316 def is_udeb(self) -> bool:
317 return self.package_type == UDEB_PACKAGE_TYPE
319 @property
320 def should_be_acted_on(self) -> bool:
321 return self._should_be_acted_on and self._declared_arch_matches_output_arch
323 @property
324 def fields(self) -> Mapping[str, str]:
325 return self._package_fields
327 @property
328 def resolved_architecture(self) -> str:
329 arch = self.declared_architecture
330 if arch == "all":
331 return arch
332 if self._x_dh_build_for_type == "target": 332 ↛ 333line 332 didn't jump to line 333 because the condition on line 332 was never true
333 return self._dpkg_architecture_variables["DEB_TARGET_ARCH"]
334 return self._dpkg_architecture_variables.current_host_arch
336 def package_deb_architecture_variable(self, variable_suffix: str) -> str:
337 if self._x_dh_build_for_type == "target": 337 ↛ 338line 337 didn't jump to line 338 because the condition on line 337 was never true
338 return self._dpkg_architecture_variables[f"DEB_TARGET_{variable_suffix}"]
339 return self._dpkg_architecture_variables[f"DEB_HOST_{variable_suffix}"]
341 @property
342 def deb_multiarch(self) -> str:
343 return self.package_deb_architecture_variable("MULTIARCH")
345 @property
346 def _x_dh_build_for_type(self) -> str:
347 v = self._package_fields.get("X-DH-Build-For-Type")
348 if v is None: 348 ↛ 350line 348 didn't jump to line 350 because the condition on line 348 was always true
349 return "host"
350 return v.lower()
352 @property
353 def package_type(self) -> str:
354 """Short for Package-Type (with proper default if absent)"""
355 v = self.fields.get("Package-Type")
356 if v is None:
357 return DEFAULT_PACKAGE_TYPE
358 return v
360 @property
361 def is_main_package(self) -> bool:
362 return self._is_main_package
364 def cross_command(self, command: str) -> str:
365 arch_table = self._dpkg_architecture_variables
366 if self._x_dh_build_for_type == "target":
367 target_gnu_type = arch_table["DEB_TARGET_GNU_TYPE"]
368 if arch_table["DEB_HOST_GNU_TYPE"] != target_gnu_type:
369 return f"{target_gnu_type}-{command}"
370 if arch_table.is_cross_compiling:
371 return f"{arch_table['DEB_HOST_GNU_TYPE']}-{command}"
372 return command
374 @property
375 def declared_architecture(self) -> str:
376 return self.fields["Architecture"]
378 @property
379 def is_arch_all(self) -> bool:
380 return self.declared_architecture == "all"
383class SourcePackage:
384 __slots__ = ("_package_fields",)
386 def __init__(self, fields: Union[Mapping[str, str], Deb822ParagraphElement]):
387 # Typing-wise, Deb822ParagraphElement is *not* a Mapping[str, str] but it behaves enough
388 # like one that we rely on it and just cast it.
389 self._package_fields = cast("Mapping[str, str]", fields)
391 @property
392 def fields(self) -> Mapping[str, str]:
393 return self._package_fields
395 @property
396 def name(self) -> str:
397 return self._package_fields["Source"]