Coverage for src/debputy/package_build/assemble_deb.py: 13%
100 statements
« prev ^ index » next coverage.py v7.6.0, created at 2025-01-27 13:59 +0000
1import json
2import os
3import subprocess
4from typing import Optional, Sequence, List, Tuple
6from debputy import DEBPUTY_ROOT_DIR
7from debputy.commands.debputy_cmd.context import CommandContext
8from debputy.deb_packaging_support import setup_control_files
9from debputy.filesystem_scan import FSRootDir
10from debputy.highlevel_manifest import HighLevelManifest
11from debputy.intermediate_manifest import IntermediateManifest
12from debputy.plugin.api.impl_types import PackageDataTable
13from debputy.util import (
14 escape_shell,
15 _error,
16 compute_output_filename,
17 scratch_dir,
18 ensure_dir,
19 assume_not_none,
20 _info,
21)
23_RRR_DEB_ASSEMBLY_KEYWORD = "debputy/deb-assembly"
24_NOTIFIED_ABOUT_FALLBACK_ASSEMBLY = False
27def _serialize_intermediate_manifest(members: IntermediateManifest) -> str:
28 serial_format = [m.to_manifest() for m in members]
29 return json.dumps(serial_format)
32def determine_assembly_method(
33 package: str,
34 intermediate_manifest: IntermediateManifest,
35) -> Tuple[bool, bool, List[str]]:
36 paths_needing_root = (
37 tm for tm in intermediate_manifest if tm.owner != "root" or tm.group != "root"
38 )
39 matched_path = next(paths_needing_root, None)
40 if matched_path is None:
41 return False, False, []
42 rrr = os.environ.get("DEB_RULES_REQUIRES_ROOT")
43 if rrr and _RRR_DEB_ASSEMBLY_KEYWORD in rrr:
44 gain_root_cmd = os.environ.get("DEB_GAIN_ROOT_CMD")
45 if not gain_root_cmd:
46 _error(
47 "DEB_RULES_REQUIRES_ROOT contains a debputy keyword but DEB_GAIN_ROOT_CMD does not contain a "
48 '"gain root" command'
49 )
50 return True, False, gain_root_cmd.split()
51 if rrr == "no":
52 global _NOTIFIED_ABOUT_FALLBACK_ASSEMBLY
53 if not _NOTIFIED_ABOUT_FALLBACK_ASSEMBLY:
54 _info(
55 'Using internal assembly method due to "Rules-Requires-Root" being "no" and dpkg-deb assembly would'
56 " require (fake)root for binary packages that needs it."
57 )
58 _NOTIFIED_ABOUT_FALLBACK_ASSEMBLY = True
59 return True, True, []
61 _error(
62 f'Due to the path "{matched_path.member_path}" in {package}, the package assembly will require (fake)root.'
63 " However, this command is not run as root nor was debputy requested to use a root command via"
64 f' "Rules-Requires-Root". Please consider adding "{_RRR_DEB_ASSEMBLY_KEYWORD}" to "Rules-Requires-Root"'
65 " in debian/control. Though, due to #1036865, you may have to revert to"
66 ' "Rules-Requires-Root: binary-targets" depending on which version of dpkg you need to support.'
67 ' Alternatively, you can set "Rules-Requires-Root: no" in debian/control and debputy will assemble'
68 " the package anyway. In this case, dpkg-deb will not be used, but the output should be bit-for-bit"
69 " compatible with what debputy would have produced with dpkg-deb (and root/fakeroot)."
70 )
73def assemble_debs(
74 context: CommandContext,
75 manifest: HighLevelManifest,
76 package_data_table: PackageDataTable,
77 is_dh_rrr_only_mode: bool,
78 *,
79 debug_materialization: bool = False,
80) -> None:
81 parsed_args = context.parsed_args
82 output_path = parsed_args.output
83 upstream_args = parsed_args.upstream_args
84 deb_materialize = str(DEBPUTY_ROOT_DIR / "deb_materialization.py")
85 mtime = context.mtime
87 for dctrl_bin in manifest.active_packages:
88 package = dctrl_bin.name
89 dbgsym_package_name = f"{package}-dbgsym"
90 dctrl_data = package_data_table[package]
91 fs_root = dctrl_data.fs_root
92 control_output_fs_path = dctrl_data.control_output_dir.fs_path
93 package_metadata_context = dctrl_data.package_metadata_context
94 if (
95 dbgsym_package_name in package_data_table
96 or "noautodbgsym" in manifest.deb_options_and_profiles.deb_build_options
97 or "noddebs" in manifest.deb_options_and_profiles.deb_build_options
98 ):
99 # Discard the dbgsym part if it conflicts with a real package, or
100 # we were asked not to build it.
101 dctrl_data.dbgsym_info.dbgsym_fs_root = FSRootDir()
102 dctrl_data.dbgsym_info.dbgsym_ids.clear()
103 dbgsym_fs_root = dctrl_data.dbgsym_info.dbgsym_fs_root
104 dbgsym_ids = dctrl_data.dbgsym_info.dbgsym_ids
105 intermediate_manifest = manifest.finalize_data_tar_contents(
106 package, fs_root, mtime
107 )
109 setup_control_files(
110 dctrl_data,
111 manifest,
112 dbgsym_fs_root,
113 dbgsym_ids,
114 package_metadata_context,
115 allow_ctrl_file_management=not is_dh_rrr_only_mode,
116 )
118 needs_root, use_fallback_assembly, gain_root_cmd = determine_assembly_method(
119 package, intermediate_manifest
120 )
122 if not dctrl_bin.is_udeb and any(
123 f for f in dbgsym_fs_root.all_paths() if f.is_file
124 ):
125 # We never built udebs due to #797391. We currently do not generate a control
126 # file for it either for the same reason.
127 dbgsym_root = dctrl_data.dbgsym_info.dbgsym_root_dir
128 if not os.path.isdir(output_path):
129 _error(
130 "Cannot produce a dbgsym package when output path is not a directory."
131 )
132 dbgsym_intermediate_manifest = manifest.finalize_data_tar_contents(
133 dbgsym_package_name,
134 dbgsym_fs_root,
135 mtime,
136 )
137 _assemble_deb(
138 dbgsym_package_name,
139 deb_materialize,
140 dbgsym_intermediate_manifest,
141 mtime,
142 os.path.join(dbgsym_root, "DEBIAN"),
143 output_path,
144 upstream_args,
145 is_udeb=dctrl_bin.is_udeb, # Review this if we ever do dbgsyms for udebs
146 use_fallback_assembly=False,
147 needs_root=False,
148 debug_materialization=debug_materialization,
149 )
151 _assemble_deb(
152 package,
153 deb_materialize,
154 intermediate_manifest,
155 mtime,
156 control_output_fs_path,
157 output_path,
158 upstream_args,
159 is_udeb=dctrl_bin.is_udeb,
160 use_fallback_assembly=use_fallback_assembly,
161 needs_root=needs_root,
162 gain_root_cmd=gain_root_cmd,
163 debug_materialization=debug_materialization,
164 )
167def _assemble_deb(
168 package: str,
169 deb_materialize_cmd: str,
170 intermediate_manifest: IntermediateManifest,
171 mtime: int,
172 control_output_fs_path: str,
173 output_path: str,
174 upstream_args: Optional[List[str]],
175 is_udeb: bool = False,
176 use_fallback_assembly: bool = False,
177 needs_root: bool = False,
178 gain_root_cmd: Optional[Sequence[str]] = None,
179 *,
180 debug_materialization: bool = False,
181) -> None:
182 scratch_root_dir = scratch_dir()
183 materialization_dir = os.path.join(
184 scratch_root_dir, "materialization-dirs", package
185 )
186 ensure_dir(os.path.dirname(materialization_dir))
187 materialize_cmd: List[str] = []
188 assert not use_fallback_assembly or not gain_root_cmd
189 if needs_root and gain_root_cmd:
190 # Only use the gain_root_cmd if we absolutely need it.
191 # Note that gain_root_cmd will be empty unless R³ is set to the relevant keyword
192 # that would make us use targeted promotion. Therefore, we do not need to check other
193 # conditions than the package needing root. (R³: binary-targets implies `needs_root=True`
194 # without a gain_root_cmd)
195 materialize_cmd.extend(gain_root_cmd)
196 materialize_cmd.append(deb_materialize_cmd)
197 if debug_materialization:
198 materialize_cmd.append("--verbose")
199 materialize_cmd.extend(
200 [
201 "materialize-deb",
202 "--intermediate-package-manifest",
203 "-",
204 "--may-move-control-files",
205 "--may-move-data-files",
206 "--source-date-epoch",
207 str(mtime),
208 "--discard-existing-output",
209 control_output_fs_path,
210 materialization_dir,
211 ]
212 )
213 output = output_path
214 if is_udeb:
215 materialize_cmd.append("--udeb")
216 output = os.path.join(
217 output_path, compute_output_filename(control_output_fs_path, True)
218 )
220 assembly_method = "debputy" if needs_root and use_fallback_assembly else "dpkg-deb"
221 combined_materialization_and_assembly = not needs_root
222 if combined_materialization_and_assembly:
223 materialize_cmd.extend(
224 ["--build-method", assembly_method, "--assembled-deb-output", output]
225 )
227 if upstream_args:
228 materialize_cmd.append("--")
229 materialize_cmd.extend(upstream_args)
231 if combined_materialization_and_assembly:
232 _info(
233 f"Materializing and assembling {package} via: {escape_shell(*materialize_cmd)}"
234 )
235 else:
236 _info(f"Materializing {package} via: {escape_shell(*materialize_cmd)}")
237 proc = subprocess.Popen(materialize_cmd, stdin=subprocess.PIPE)
238 proc.communicate(
239 _serialize_intermediate_manifest(intermediate_manifest).encode("utf-8")
240 )
241 if proc.returncode != 0:
242 _error(f"{escape_shell(deb_materialize_cmd)} exited with a non-zero exit code!")
244 if not combined_materialization_and_assembly:
245 build_materialization = [
246 deb_materialize_cmd,
247 "build-materialized-deb",
248 materialization_dir,
249 assembly_method,
250 "--output",
251 output,
252 ]
253 _info(f"Assembling {package} via: {escape_shell(*build_materialization)}")
254 try:
255 subprocess.check_call(build_materialization)
256 except subprocess.CalledProcessError as e:
257 exit_code = f" with exit code {e.returncode}" if e.returncode else ""
258 _error(
259 f"Assembly command for {package} failed{exit_code}. Please review the output of the command"
260 f" for more details on the problem."
261 )