Coverage for src/debputy/deb_packaging_support.py: 24%
842 statements
« prev ^ index » next coverage.py v7.8.2, created at 2026-07-22 10:58 +0000
1import collections
2import contextlib
3import dataclasses
4import datetime
5import functools
6import hashlib
7import itertools
8import operator
9import os
10import re
11import shutil
12import subprocess
13import tempfile
14import textwrap
15import typing
16from contextlib import ExitStack, suppress
17from tempfile import mkstemp
18from typing import (
19 Literal,
20 cast,
21 Any,
22 AbstractSet,
23 TYPE_CHECKING,
24)
25from collections.abc import Iterable, Sequence, Iterator, Mapping
27import debian.deb822
28from debian.changelog import Changelog
29from debian.deb822 import Deb822
30from debputy._deb_options_profiles import DebBuildOptionsAndProfiles
31from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable
32from debputy.elf_util import find_all_elf_files, ELF_MAGIC
33from debputy.exceptions import DebputyDpkgGensymbolsError, PureVirtualPathError
34from debputy.filesystem_scan import (
35 FSControlRootDir,
36 VirtualPathBase,
37 InMemoryVirtualPathBase,
38)
39from debputy.maintscript_snippet import (
40 ALL_CONTROL_SCRIPTS,
41 DPKG_DEB_CONTROL_SCRIPTS,
42 SnippetAnchor,
43 PackageMaintscriptSnippetContainer,
44 SUPPORTED_UDEB_SCRIPTS,
45)
46from debputy.packager_provided_files import PackagerProvidedFile
47from debputy.packages import BinaryPackage, SourcePackage
48from debputy.packaging.alternatives import process_alternatives
49from debputy.packaging.debconf_templates import process_debconf_templates
50from debputy.packaging.makeshlibs import (
51 compute_shlibs,
52 ShlibsContent,
53 generate_shlib_dirs,
54 resolve_reserved_provided_file,
55)
56from debputy.plugin.api.feature_set import PluginProvidedFeatureSet
57from debputy.plugin.api.impl import ServiceRegistryImpl
58from debputy.plugin.api.impl_types import (
59 MetadataOrMaintscriptDetector,
60 PackageDataTable,
61 ServiceManagerDetails,
62)
63from debputy.plugin.api.spec import (
64 FlushableSubstvars,
65 VirtualPath,
66 PackageProcessingContext,
67 ServiceDefinition,
68)
69from debputy.plugins.debputy.binary_package_rules import (
70 ServiceRule,
71 MaintainerProvidedMaintscriptSnippetContainer,
72)
73from debputy.util import (
74 _error,
75 ensure_dir,
76 assume_not_none,
77 resolve_perl_config,
78 perlxs_api_dependency,
79 detect_fakeroot,
80 grouper,
81 _info,
82 xargs,
83 escape_shell,
84 generated_content_dir,
85 print_command,
86 _warn,
87)
89if TYPE_CHECKING:
90 from debputy.highlevel_manifest import (
91 HighLevelManifest,
92 PackageTransformationDefinition,
93 BinaryPackageData,
94 )
96_T64_REGEX = re.compile("^lib.*t64(?:-nss)?$")
97_T64_PROVIDES = "t64:Provides"
100def generate_md5sums_file(
101 control_output_dir: VirtualPathBase,
102 fs_root: VirtualPath,
103) -> None:
104 conffiles = control_output_dir.get("conffiles")
105 exclude = set()
106 if conffiles and conffiles.is_file:
107 with conffiles.open() as fd:
108 for line in fd:
109 if not line.startswith("/"):
110 continue
111 exclude.add("." + line.rstrip("\n"))
112 files_to_checksum = sorted(
113 (
114 path
115 for path in fs_root.all_paths()
116 if path.is_file and path.path not in exclude
117 ),
118 # Sort in the same order as dh_md5sums, which is not quite the same as dpkg/`all_paths()`
119 # Compare `.../doc/...` vs `.../doc-base/...` if you want to see the difference between
120 # the two approaches.
121 key=lambda p: p.path,
122 )
123 if not files_to_checksum:
124 return
125 with control_output_dir.open_child("md5sums", "w") as md5fd:
126 for member in files_to_checksum:
127 path = member.path
128 assert path.startswith("./")
129 path = path[2:]
130 with member.open(byte_io=True) as f:
131 file_hash = hashlib.md5()
132 while chunk := f.read(8192):
133 file_hash.update(chunk)
134 md5fd.write(f"{file_hash.hexdigest()} {path}\n")
137def install_or_generate_conffiles(
138 ctrl_root: InMemoryVirtualPathBase | FSControlRootDir,
139 fs_root: VirtualPath,
140 reserved_packager_provided_files: dict[str, list[PackagerProvidedFile]],
141) -> None:
142 provided_conffiles_file = resolve_reserved_provided_file(
143 "conffiles",
144 reserved_packager_provided_files,
145 )
146 if ( 146 ↛ 151line 146 didn't jump to line 151 because the condition on line 146 was never true
147 provided_conffiles_file
148 and provided_conffiles_file.is_file
149 and provided_conffiles_file.size > 0
150 ):
151 ctrl_root.insert_file_from_fs_path(
152 "conffiles",
153 provided_conffiles_file.fs_path,
154 mode=0o644,
155 reference_path=provided_conffiles_file,
156 )
157 etc_dir = fs_root.lookup("etc")
158 if etc_dir:
159 _add_conffiles(ctrl_root, (p for p in etc_dir.all_paths() if p.is_file))
162PERL_DEP_PROGRAM = 1
163PERL_DEP_INDEP_PM_MODULE = 2
164PERL_DEP_XS_MODULE = 4
165PERL_DEP_ARCH_PM_MODULE = 8
166PERL_DEP_MA_ANY_INCOMPATIBLE_TYPES = ~(PERL_DEP_PROGRAM | PERL_DEP_INDEP_PM_MODULE)
169@functools.lru_cache(2) # In practice, param will be "perl" or "perl-base"
170def _dpkg_perl_version(package: str) -> str:
171 dpkg_version = None
172 lines = (
173 subprocess.check_output(["dpkg", "-s", package])
174 .decode("utf-8")
175 .splitlines(keepends=False)
176 )
177 for line in lines:
178 if line.startswith("Version: "):
179 dpkg_version = line[8:].strip()
180 break
181 assert dpkg_version is not None
182 return dpkg_version
185def handle_perl_code(
186 dctrl_bin: BinaryPackage,
187 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
188 fs_root: InMemoryVirtualPathBase,
189 substvars: FlushableSubstvars,
190) -> None:
191 perl_config_data = resolve_perl_config(dpkg_architecture_variables, dctrl_bin)
192 detected_dep_requirements = 0
194 # MakeMaker always makes lib and share dirs, but typically only one directory is actually used.
195 for perl_inc_dir in (perl_config_data.vendorarch, perl_config_data.vendorlib):
196 p = fs_root.lookup(perl_inc_dir)
197 if p and p.is_dir:
198 p.prune_if_empty_dir()
200 # FIXME: 80% of this belongs in a metadata detector, but that requires us to expose .walk() in the public API,
201 # which will not be today.
202 for d, pm_mode in [
203 (perl_config_data.vendorlib, PERL_DEP_INDEP_PM_MODULE),
204 (perl_config_data.vendorarch, PERL_DEP_ARCH_PM_MODULE),
205 ]:
206 inc_dir = fs_root.lookup(d)
207 if not inc_dir:
208 continue
209 for path in inc_dir.all_paths():
210 if not path.is_file:
211 continue
212 if path.name.endswith(".so"):
213 detected_dep_requirements |= PERL_DEP_XS_MODULE
214 elif path.name.endswith(".pm"):
215 detected_dep_requirements |= pm_mode
217 for path, children in fs_root.walk():
218 if path.path == "./usr/share/doc":
219 children.clear()
220 continue
221 if (
222 not path.is_file
223 or not path.has_fs_path
224 or not (path.is_executable or path.name.endswith(".pl"))
225 ):
226 continue
228 interpreter = path.interpreter()
229 if interpreter is not None and interpreter.command_full_basename == "perl":
230 detected_dep_requirements |= PERL_DEP_PROGRAM
232 if not detected_dep_requirements:
233 return
234 dpackage = "perl"
235 # FIXME: Currently, dh_perl supports perl-base via manual toggle.
237 dependency = dpackage
238 if not (detected_dep_requirements & PERL_DEP_MA_ANY_INCOMPATIBLE_TYPES):
239 dependency += ":any"
241 if detected_dep_requirements & PERL_DEP_XS_MODULE:
242 dpkg_version = _dpkg_perl_version(dpackage)
243 dependency += f" (>= {dpkg_version})"
244 substvars.add_dependency("perl:Depends", dependency)
246 if detected_dep_requirements & (PERL_DEP_XS_MODULE | PERL_DEP_ARCH_PM_MODULE):
247 substvars.add_dependency("perl:Depends", perlxs_api_dependency())
250def usr_local_transformation(dctrl: BinaryPackage, fs_root: VirtualPath) -> None:
251 path = fs_root.lookup("./usr/local")
252 if path and any(path.iterdir()):
253 # There are two key issues:
254 # 1) Getting the generated maintscript carried on to the final maintscript
255 # 2) Making sure that manifest created directories do not trigger the "unused error".
256 _error(
257 f"Replacement of /usr/local paths is currently not supported in debputy (triggered by: {dctrl.name})."
258 )
261def _find_and_analyze_systemd_service_files(
262 fs_root: VirtualPath,
263 systemd_service_dir: Literal["system", "user"],
264) -> Iterable[VirtualPath]:
265 service_dirs = [
266 f"./usr/lib/systemd/{systemd_service_dir}",
267 f"./lib/systemd/{systemd_service_dir}",
268 ]
269 aliases: dict[str, list[str]] = collections.defaultdict(list)
270 seen = set()
271 all_files = []
273 for d in service_dirs:
274 system_dir = fs_root.lookup(d)
275 if not system_dir:
276 continue
277 for child in system_dir.iterdir():
278 if child.is_symlink:
279 dest = os.path.basename(child.readlink())
280 aliases[dest].append(child.name)
281 elif child.is_file and child.name not in seen:
282 seen.add(child.name)
283 all_files.append(child)
285 return all_files
288def detect_systemd_user_service_files(
289 dctrl: BinaryPackage,
290 fs_root: VirtualPath,
291) -> None:
292 for service_file in _find_and_analyze_systemd_service_files(fs_root, "user"):
293 _error(
294 f'Sorry, systemd user services files are not supported at the moment (saw "{service_file.path}"'
295 f" in {dctrl.name})"
296 )
299# Generally, this should match the release date of oldstable or oldoldstable
300_DCH_PRUNE_CUT_OFF_DATE = datetime.date(2019, 7, 6)
301_DCH_MIN_NUM_OF_ENTRIES = 4
304def _prune_dch_file(
305 package: BinaryPackage,
306 path: VirtualPath,
307 is_changelog: bool,
308 keep_versions: set[str] | None,
309 *,
310 trim: bool = True,
311) -> tuple[bool, set[str] | None]:
312 # TODO: Process `d/changelog` once
313 # Note we cannot assume that changelog_file is always `d/changelog` as you can have
314 # per-package changelogs.
315 with path.open() as fd:
316 dch = Changelog(fd)
317 shortened = False
318 important_entries = 0
319 binnmu_entries = []
320 if is_changelog:
321 kept_entries = []
322 for block in dch:
323 if block.other_pairs.get("binary-only", "no") == "yes":
324 # Always keep binNMU entries (they are always in the top) and they do not count
325 # towards our kept_entries limit
326 binnmu_entries.append(block)
327 continue
328 block_date = block.date
329 if block_date is None:
330 _error("The Debian changelog was missing date in sign off line")
331 try:
332 entry_date = datetime.datetime.strptime(
333 block_date, "%a, %d %b %Y %H:%M:%S %z"
334 ).date()
335 except ValueError:
336 _error(
337 f"Invalid date in the changelog entry for version {block.version}: {block_date!r} (Expected format: 'Thu, 26 Feb 2026 00:00:00 +0000')"
338 )
339 if (
340 trim
341 and entry_date < _DCH_PRUNE_CUT_OFF_DATE
342 and important_entries >= _DCH_MIN_NUM_OF_ENTRIES
343 ):
344 shortened = True
345 break
346 # Match debhelper in incrementing after the check.
347 important_entries += 1
348 kept_entries.append(block)
349 else:
350 assert keep_versions is not None
351 # The NEWS files should match the version for the dch to avoid lintian warnings.
352 # If that means we remove all entries in the NEWS file, then we delete the NEWS
353 # file (see #1021607)
354 kept_entries = [b for b in dch if b.version in keep_versions]
355 shortened = len(dch) > len(kept_entries)
356 if shortened and not kept_entries:
357 path.unlink()
358 return True, None
360 if not shortened and not binnmu_entries:
361 return False, None
363 parent_dir = assume_not_none(path.parent_dir)
365 with (
366 path.replace_fs_path_content() as fs_path,
367 open(fs_path, "w", encoding="utf-8") as fd,
368 ):
369 for entry in kept_entries:
370 fd.write(str(entry))
372 if is_changelog and shortened:
373 # For changelog (rather than NEWS) files, add a note about how to
374 # get the full version.
375 msg = textwrap.dedent(f"""\
376 # Older entries have been removed from this changelog.
377 # To read the complete changelog use `apt changelog {package.name}`.
378 """)
379 fd.write(msg)
381 if binnmu_entries:
382 if package.is_arch_all:
383 _error(
384 f"The package {package.name} is architecture all, but it is built during a binNMU. A binNMU build"
385 " must not include architecture all packages"
386 )
388 with (
389 parent_dir.add_file(
390 f"{path.name}.{package.resolved_architecture}"
391 ) as binnmu_changelog,
392 open(
393 binnmu_changelog.fs_path,
394 "w",
395 encoding="utf-8",
396 ) as binnmu_fd,
397 ):
398 for entry in binnmu_entries:
399 binnmu_fd.write(str(entry))
401 if not shortened:
402 return False, None
403 return True, {b.version for b in kept_entries}
406def fixup_debian_changelog_and_news_file(
407 dctrl: BinaryPackage,
408 fs_root: VirtualPath,
409 is_native: bool,
410 build_env: DebBuildOptionsAndProfiles,
411) -> None:
412 doc_dir = fs_root.lookup(f"./usr/share/doc/{dctrl.name}")
413 if not doc_dir:
414 return
415 changelog = doc_dir.get("changelog.Debian")
416 if changelog and is_native:
417 changelog.name = "changelog"
418 elif is_native:
419 changelog = doc_dir.get("changelog")
421 trim = "notrimdch" not in build_env.deb_build_options
423 kept_entries = None
424 pruned_changelog = False
425 if changelog and changelog.has_fs_path:
426 pruned_changelog, kept_entries = _prune_dch_file(
427 dctrl, changelog, True, None, trim=trim
428 )
430 if not trim:
431 return
433 news_file = doc_dir.get("NEWS.Debian")
434 if news_file and news_file.has_fs_path and pruned_changelog:
435 _prune_dch_file(dctrl, news_file, False, kept_entries)
438_UPSTREAM_CHANGELOG_SOURCE_DIRS = [
439 ".",
440 "doc",
441 "docs",
442]
443_UPSTREAM_CHANGELOG_NAMES = {
444 # The value is a priority to match the debhelper order.
445 # - The suffix weights heavier than the basename (because that is what debhelper did)
446 #
447 # We list the name/suffix in order of priority in the code. That makes it easier to
448 # see the priority directly, but it gives the "lowest" value to the most important items
449 f"{n}{s}": (sw, nw)
450 for (nw, n), (sw, s) in itertools.product(
451 enumerate(["changelog", "changes", "history"], start=1),
452 enumerate(["", ".txt", ".md", ".rst", ".org"], start=1),
453 )
454}
457def _detect_upstream_changelog(names: Iterable[str]) -> str | None:
458 matches = []
459 for name in names:
460 match_priority = _UPSTREAM_CHANGELOG_NAMES.get(name.lower())
461 if match_priority is not None:
462 matches.append((name, match_priority))
463 if not matches:
464 return None
465 return min(matches, key=operator.itemgetter(1))[0]
468def install_upstream_changelog(
469 dctrl_bin: BinaryPackage,
470 fs_root: InMemoryVirtualPathBase,
471 source_fs_root: VirtualPath,
472) -> None:
473 doc_dir = f"./usr/share/doc/{dctrl_bin.name}"
474 bdir = fs_root.lookup(doc_dir)
475 if bdir and not bdir.is_dir:
476 # "/usr/share/doc/foo -> bar" symlink. Avoid croaking on those per:
477 # https://salsa.debian.org/debian/debputy/-/issues/49
478 return
480 if bdir:
481 if bdir.get("changelog") or bdir.get("changelog.gz"):
482 # Upstream's build system already provided the changelog with the correct name.
483 # Accept that as the canonical one.
484 return
485 upstream_changelog = _detect_upstream_changelog(
486 p.name for p in bdir.iterdir() if p.is_file and p.has_fs_path and p.size > 0
487 )
488 if upstream_changelog:
489 p = bdir.lookup(upstream_changelog)
490 assert p is not None # Mostly as a typing hint
491 p.name = "changelog"
492 return
493 for dirname in _UPSTREAM_CHANGELOG_SOURCE_DIRS:
494 dir_path = source_fs_root.lookup(dirname)
495 if not dir_path or not dir_path.is_dir:
496 continue
497 changelog_name = _detect_upstream_changelog(
498 p.name
499 for p in dir_path.iterdir()
500 if p.is_file and p.has_fs_path and p.size > 0
501 )
502 if changelog_name:
503 if bdir is None: 503 ↛ 505line 503 didn't jump to line 505 because the condition on line 503 was always true
504 bdir = fs_root.mkdirs(doc_dir)
505 typing.cast(InMemoryVirtualPathBase, bdir).insert_file_from_fs_path(
506 "changelog",
507 dir_path[changelog_name].fs_path,
508 )
509 break
512@dataclasses.dataclass(slots=True)
513class _ElfInfo:
514 path: VirtualPath
515 fs_path: str
516 is_stripped: bool | None = None
517 build_id: str | None = None
518 dbgsym: InMemoryVirtualPathBase | None = None
521def _elf_static_lib_walk_filter[VP: VirtualPath](
522 fs_path: VirtualPath,
523 children: list[VP],
524) -> bool:
525 if (
526 fs_path.name == ".build-id"
527 and assume_not_none(fs_path.parent_dir).name == "debug"
528 ):
529 children.clear()
530 return False
531 # Deal with some special cases, where certain files are not supposed to be stripped in a given directory
532 if "debug/" in fs_path.path or fs_path.name.endswith("debug/"):
533 # FIXME: We need a way to opt out of this per #468333/#1016122
534 # list(children) is because we mutate children and that is not allowed during iteration.
535 for so_file in (f for f in list(children) if f.name.endswith(".so")):
536 children.remove(so_file)
537 if "/guile/" in fs_path.path or fs_path.name == "guile":
538 # list(children) is because we mutate children and that is not allowed during iteration.
539 for go_file in (f for f in list(children) if f.name.endswith(".go")):
540 children.remove(go_file)
541 return True
544@contextlib.contextmanager
545def _all_elf_files(fs_root: VirtualPath) -> Iterator[dict[str, _ElfInfo]]:
546 all_elf_files = find_all_elf_files(
547 fs_root,
548 walk_filter=_elf_static_lib_walk_filter,
549 )
550 if not all_elf_files:
551 yield {}
552 return
553 with ExitStack() as cm_stack:
554 resolved = (
555 (p, cm_stack.enter_context(p.replace_fs_path_content()))
556 for p in all_elf_files
557 )
558 elf_info = {
559 fs_path: _ElfInfo(
560 path=assume_not_none(fs_root.lookup(detached_path.path)),
561 fs_path=fs_path,
562 )
563 for detached_path, fs_path in resolved
564 }
565 _resolve_build_ids(elf_info)
566 yield elf_info
569def _find_all_static_libs(
570 fs_root: InMemoryVirtualPathBase,
571) -> Iterator[InMemoryVirtualPathBase]:
572 for path, children in fs_root.walk():
573 # Matching the logic of dh_strip for now.
574 if not _elf_static_lib_walk_filter(path, children):
575 continue
576 if not path.is_file:
577 continue
578 if path.name.startswith("lib") and path.name.endswith("_g.a"):
579 # _g.a are historically ignored. I do not remember why, but guessing the "_g" is
580 # an encoding of gcc's -g parameter into the filename (with -g meaning "I want debug
581 # symbols")
582 continue
583 if not path.has_fs_path:
584 continue
585 with path.open(byte_io=True) as fd:
586 magic = fd.read(8)
587 if magic not in (b"!<arch>\n", b"!<thin>\n"):
588 continue
589 # Maybe we should see if the first file looks like an index file.
590 # Three random .a samples suggests the index file is named "/"
591 # Not sure if we should skip past it and then do the ELF check or just assume
592 # that "index => static lib".
593 data = fd.read(1024 * 1024)
594 if b"\0" not in data and ELF_MAGIC not in data:
595 continue
596 yield path
599@contextlib.contextmanager
600def _all_static_libs(fs_root: InMemoryVirtualPathBase) -> Iterator[list[str]]:
601 all_static_libs = list(_find_all_static_libs(fs_root))
602 if not all_static_libs:
603 yield []
604 return
605 with ExitStack() as cm_stack:
606 resolved: list[str] = [
607 cm_stack.enter_context(p.replace_fs_path_content()) for p in all_static_libs
608 ]
609 yield resolved
612_FILE_BUILD_ID_RE = re.compile(rb"BuildID(?:\[\S+\])?=([A-Fa-f0-9]+)")
615def _resolve_build_ids(elf_info: dict[str, _ElfInfo]) -> None:
616 static_cmd = ["file", "-00", "-N"]
617 if detect_fakeroot():
618 static_cmd.append("--no-sandbox")
620 for cmd in xargs(static_cmd, (i.fs_path for i in elf_info.values())):
621 _info(f"Looking up build-ids via: {escape_shell(*cmd)}")
622 output = subprocess.check_output(cmd)
624 # Trailing "\0" gives an empty element in the end when splitting, so strip it out
625 lines = output.rstrip(b"\0").split(b"\0")
627 for fs_path_b, verdict in grouper(lines, 2, incomplete="strict"):
628 fs_path = fs_path_b.decode("utf-8")
629 info = elf_info[fs_path]
630 info.is_stripped = b"not stripped" not in verdict
631 m = _FILE_BUILD_ID_RE.search(verdict)
632 if m:
633 info.build_id = m.group(1).decode("utf-8")
636def _make_debug_file(
637 objcopy: str,
638 fs_path: str,
639 build_id: str,
640 dbgsym_fs_root: InMemoryVirtualPathBase,
641) -> InMemoryVirtualPathBase:
642 dbgsym_dirname = f"./usr/lib/debug/.build-id/{build_id[0:2]}/"
643 dbgsym_basename = f"{build_id[2:]}.debug"
644 dbgsym_dir = dbgsym_fs_root.mkdirs(dbgsym_dirname)
645 if dbgsym_basename in dbgsym_dir:
646 return dbgsym_dir[dbgsym_basename]
647 # objcopy is a pain and includes the basename verbatim when you do `--add-gnu-debuglink` without having an option
648 # to overwrite the physical basename. So we have to ensure that the physical basename matches the installed
649 # basename.
650 try:
651 with dbgsym_dir.add_file(
652 dbgsym_basename,
653 unlink_if_exists=False,
654 fs_basename_matters=True,
655 # Ensure that the debug files are namespaced by the `objcopy` that created them.
656 # This avoids arch-confusion in case two different archs happens to produce the same elf ID
657 # (in theory possible with X-DH-Build-For-Type)
658 subdir_key=f"dbgsym-build-ids/{objcopy}/{build_id[0:2]}",
659 ) as dbgsym:
660 try:
661 subprocess.check_call(
662 [
663 objcopy,
664 "--only-keep-debug",
665 "--compress-debug-sections",
666 fs_path,
667 dbgsym.fs_path,
668 ]
669 )
670 except subprocess.CalledProcessError:
671 full_command = (
672 f"{objcopy} --only-keep-debug --compress-debug-sections"
673 f" {escape_shell(fs_path, dbgsym.fs_path)}"
674 )
675 _error(
676 f"Attempting to create a .debug file failed. Please review the error message from {objcopy} to"
677 f" understand what went wrong. Full command was: {full_command}"
678 )
679 except FileExistsError as e:
680 dbgsym = dbgsym_dir.insert_file_from_fs_path(
681 dbgsym_basename,
682 e.filename,
683 exist_ok=True,
684 )
685 return dbgsym
688def _strip_binary(strip: str, options: list[str], paths: Iterable[str]) -> None:
689 # We assume the paths are obtained via `p.replace_fs_path_content()`,
690 # which is the case at the time of written and should remain so forever.
691 it = iter(paths)
692 first = next(it, None)
693 if first is None:
694 return
695 static_cmd = [strip]
696 static_cmd.extend(options)
698 for cmd in xargs(static_cmd, itertools.chain((first,), it)):
699 _info(f"Removing unnecessary ELF debug info via: {escape_shell(*cmd)}")
700 try:
701 subprocess.check_call(
702 cmd,
703 stdin=subprocess.DEVNULL,
704 restore_signals=True,
705 )
706 except subprocess.CalledProcessError:
707 _error(
708 f"Attempting to remove ELF debug info failed. Please review the error from {strip} above"
709 f" understand what went wrong."
710 )
713def _attach_debug(
714 objcopy: str, elf_binary: VirtualPath, dbgsym: InMemoryVirtualPathBase
715) -> None:
716 dbgsym_fs_path: str
717 with dbgsym.replace_fs_path_content() as dbgsym_fs_path:
718 cmd = [objcopy, "--add-gnu-debuglink", dbgsym_fs_path, elf_binary.fs_path]
719 print_command(*cmd)
720 try:
721 subprocess.check_call(cmd)
722 except subprocess.CalledProcessError:
723 _error(
724 f"Attempting to attach ELF debug link to ELF binary failed. Please review the error from {objcopy}"
725 f" above understand what went wrong."
726 )
729@functools.lru_cache
730def _has_tool(tool: str) -> bool:
731 return shutil.which(tool) is not None
734def _run_dwz(
735 dctrl: BinaryPackage,
736 dbgsym_fs_root: InMemoryVirtualPathBase,
737 unstripped_elf_info: list[_ElfInfo],
738) -> None:
739 if not unstripped_elf_info or dctrl.is_udeb or not _has_tool("dwz"):
740 return
741 dwz_cmd = ["dwz"]
742 dwz_ma_dir_name = f"usr/lib/debug/.dwz/{dctrl.deb_multiarch}"
743 dwz_ma_basename = f"{dctrl.name}.debug"
744 multifile = f"{dwz_ma_dir_name}/{dwz_ma_basename}"
745 build_time_multifile = None
746 if len(unstripped_elf_info) > 1:
747 fs_content_dir = generated_content_dir()
748 fd, build_time_multifile = mkstemp(suffix=dwz_ma_basename, dir=fs_content_dir)
749 os.close(fd)
750 dwz_cmd.append(f"-m{build_time_multifile}")
751 dwz_cmd.append(f"-M/{multifile}")
753 # TODO: configuration for disabling multi-file and tweaking memory limits
755 dwz_cmd.extend(e.fs_path for e in unstripped_elf_info)
757 _info(f"Deduplicating ELF debug info via: {escape_shell(*dwz_cmd)}")
758 try:
759 subprocess.check_call(dwz_cmd)
760 except subprocess.CalledProcessError:
761 _error(
762 "Attempting to deduplicate ELF info via dwz failed. Please review the output from dwz above"
763 " to understand what went wrong."
764 )
765 if build_time_multifile is not None and os.stat(build_time_multifile).st_size > 0:
766 dwz_dir = dbgsym_fs_root.mkdirs(dwz_ma_dir_name)
767 dwz_dir.insert_file_from_fs_path(
768 dwz_ma_basename,
769 build_time_multifile,
770 mode=0o644,
771 require_copy_on_write=False,
772 follow_symlinks=False,
773 )
776def relocate_dwarves_into_dbgsym_packages(
777 dctrl: BinaryPackage,
778 package_fs_root: InMemoryVirtualPathBase,
779 dbgsym_fs_root: InMemoryVirtualPathBase,
780 *,
781 run_dwz: bool = False,
782) -> list[str]:
783 # FIXME: hardlinks
784 with _all_static_libs(package_fs_root) as all_static_files:
785 if all_static_files:
786 strip = dctrl.cross_command("strip")
787 _strip_binary(
788 strip,
789 [
790 "--strip-debug",
791 "--remove-section=.comment",
792 "--remove-section=.note",
793 "--enable-deterministic-archives",
794 "-R",
795 ".gnu.lto_*",
796 "-R",
797 ".gnu.debuglto_*",
798 "-N",
799 "__gnu_lto_slim",
800 "-N",
801 "__gnu_lto_v1",
802 ],
803 all_static_files,
804 )
806 with _all_elf_files(package_fs_root) as all_elf_files:
807 if not all_elf_files:
808 return []
809 objcopy = dctrl.cross_command("objcopy")
810 strip = dctrl.cross_command("strip")
811 unstripped_elf_info = [e for e in all_elf_files.values() if not e.is_stripped]
813 if run_dwz:
814 _run_dwz(dctrl, dbgsym_fs_root, unstripped_elf_info)
816 for elf_info in unstripped_elf_info:
817 elf_info.dbgsym = _make_debug_file(
818 objcopy,
819 elf_info.fs_path,
820 assume_not_none(elf_info.build_id),
821 dbgsym_fs_root,
822 )
824 # Note: When run strip, we do so also on already stripped ELF binaries because that is what debhelper does!
825 # Executables (defined by mode)
826 _strip_binary(
827 strip,
828 ["--remove-section=.comment", "--remove-section=.note"],
829 (i.fs_path for i in all_elf_files.values() if i.path.is_executable),
830 )
832 # Libraries (defined by mode)
833 _strip_binary(
834 strip,
835 ["--remove-section=.comment", "--remove-section=.note", "--strip-unneeded"],
836 (i.fs_path for i in all_elf_files.values() if not i.path.is_executable),
837 )
839 for elf_info in unstripped_elf_info:
840 _attach_debug(
841 objcopy,
842 assume_not_none(elf_info.path),
843 assume_not_none(elf_info.dbgsym),
844 )
846 # Set for uniqueness
847 all_debug_info = sorted(
848 {assume_not_none(i.build_id) for i in unstripped_elf_info}
849 )
851 dbgsym_doc_dir = dbgsym_fs_root.mkdirs("./usr/share/doc/")
852 dbgsym_doc_dir.add_symlink(f"{dctrl.name}-dbgsym", dctrl.name)
853 return all_debug_info
856def run_package_processors(
857 manifest: "HighLevelManifest",
858 package_metadata_context: PackageProcessingContext,
859 fs_root: VirtualPath,
860) -> None:
861 pppps = manifest.plugin_provided_feature_set.package_processors_in_order()
862 binary_package = package_metadata_context.binary_package
863 for pppp in pppps:
864 if not pppp.applies_to(binary_package):
865 continue
866 pppp.run_package_processor(fs_root, None, package_metadata_context)
869def cross_package_control_files(
870 package_data_table: PackageDataTable,
871 manifest: "HighLevelManifest",
872) -> None:
873 errors = []
874 combined_shlibs = ShlibsContent()
875 shlibs_dir = None
876 shlib_dirs: list[str] = []
877 shlibs_local = manifest.debian_dir.get("shlibs.local")
878 if shlibs_local and shlibs_local.is_file:
879 with shlibs_local.open() as fd:
880 combined_shlibs.add_entries_from_shlibs_file(fd)
882 debputy_plugin_metadata = manifest.plugin_provided_feature_set.plugin_data[
883 "debputy"
884 ]
886 for binary_package_data in package_data_table:
887 binary_package = binary_package_data.binary_package
888 if (
889 binary_package.is_arch_all
890 or not binary_package.should_be_acted_on
891 or binary_package.is_udeb
892 ):
893 continue
894 fs_root = binary_package_data.fs_root
895 package_metadata_context = binary_package_data.package_metadata_context
896 package_state = manifest.package_state_for(binary_package.name)
897 related_udeb_package = package_metadata_context.related_udeb_package
899 udeb_package_name = related_udeb_package.name if related_udeb_package else None
900 ctrl = binary_package_data.ctrl_creator.for_plugin(
901 debputy_plugin_metadata,
902 "compute_shlibs",
903 )
904 try:
905 soname_info_list = compute_shlibs(
906 package_metadata_context,
907 binary_package_data.control_output_dir.fs_path,
908 fs_root,
909 manifest,
910 udeb_package_name,
911 ctrl,
912 package_state.reserved_packager_provided_files,
913 combined_shlibs,
914 )
915 except DebputyDpkgGensymbolsError as e:
916 errors.append(e.message)
917 else:
918 if soname_info_list:
919 if shlibs_dir is None:
920 shlibs_dir = generated_content_dir(
921 subdir_key="_shlibs_materialization_dir"
922 )
923 generate_shlib_dirs(
924 binary_package,
925 shlibs_dir,
926 soname_info_list,
927 shlib_dirs,
928 )
929 if errors:
930 for error in errors:
931 _warn(error)
932 _error("Stopping due to the errors above")
934 generated_shlibs_local = None
935 if combined_shlibs:
936 if shlibs_dir is None:
937 shlibs_dir = generated_content_dir(subdir_key="_shlibs_materialization_dir")
938 generated_shlibs_local = os.path.join(shlibs_dir, "shlibs.local")
939 with open(generated_shlibs_local, "w", encoding="utf-8") as fd:
940 combined_shlibs.write_to(fd)
941 _info(f"Generated {generated_shlibs_local} for dpkg-shlibdeps")
943 for binary_package_data in package_data_table:
944 binary_package = binary_package_data.binary_package
945 if binary_package.is_arch_all or not binary_package.should_be_acted_on:
946 continue
947 binary_package_data.ctrl_creator.shlibs_details = (
948 generated_shlibs_local,
949 shlib_dirs,
950 )
953def _relevant_service_definitions(
954 service_rule: ServiceRule,
955 service_managers: list[str] | frozenset[str],
956 by_service_manager_key: Mapping[
957 tuple[str, str, str, str], tuple[ServiceManagerDetails, ServiceDefinition[Any]]
958 ],
959 aliases: Mapping[str, Sequence[tuple[str, str, str, str]]],
960) -> Iterable[tuple[tuple[str, str, str, str], ServiceDefinition[Any]]]:
961 as_keys = aliases[service_rule.service]
963 pending_queue = {
964 key
965 for key in as_keys
966 if key in by_service_manager_key
967 and service_rule.applies_to_service_manager(key[-1])
968 }
969 seen_keys = set()
971 if not pending_queue:
972 service_manager_names = ", ".join(sorted(service_managers))
973 _error(
974 f"No none of the service managers ({service_manager_names}) detected a service named"
975 f" {service_rule.service} (type: {service_rule.type_of_service}, scope: {service_rule.service_scope}),"
976 f" but the manifest definition at {service_rule.definition_source} requested that."
977 )
979 while pending_queue:
980 next_key = pending_queue.pop()
981 seen_keys.add(next_key)
982 _, definition = by_service_manager_key[next_key]
983 yield next_key, definition
984 for name in definition.names:
985 for target_key in aliases[name]:
986 if (
987 target_key not in seen_keys
988 and service_rule.applies_to_service_manager(target_key[-1])
989 ):
990 pending_queue.add(target_key)
993def handle_service_management(
994 binary_package_data: "BinaryPackageData",
995 manifest: "HighLevelManifest",
996 package_metadata_context: PackageProcessingContext,
997 fs_root: VirtualPath,
998 feature_set: PluginProvidedFeatureSet,
999) -> None:
1001 by_service_manager_key = {}
1002 aliases_by_name = collections.defaultdict(list)
1004 state = manifest.package_state_for(binary_package_data.binary_package.name)
1005 all_service_managers = list(feature_set.service_managers)
1006 requested_service_rules = state.requested_service_rules
1007 for requested_service_rule in requested_service_rules:
1008 if not requested_service_rule.service_managers:
1009 continue
1010 for manager in requested_service_rule.service_managers:
1011 if manager not in feature_set.service_managers:
1012 # FIXME: Missing definition source; move to parsing.
1013 _error(
1014 f"Unknown service manager {manager} used at {requested_service_rule.definition_source}"
1015 )
1017 for service_manager_details in feature_set.service_managers.values():
1018 service_registry: ServiceRegistryImpl = ServiceRegistryImpl(
1019 service_manager_details
1020 )
1021 service_manager_details.service_detector(
1022 fs_root,
1023 service_registry,
1024 package_metadata_context,
1025 )
1027 service_definitions = service_registry.detected_services
1028 if not service_definitions:
1029 continue
1031 for plugin_provided_definition in service_definitions:
1032 key = (
1033 plugin_provided_definition.name,
1034 plugin_provided_definition.type_of_service,
1035 plugin_provided_definition.service_scope,
1036 service_manager_details.service_manager,
1037 )
1038 by_service_manager_key[key] = (
1039 service_manager_details,
1040 plugin_provided_definition,
1041 )
1043 for name in plugin_provided_definition.names:
1044 aliases_by_name[name].append(key)
1046 for requested_service_rule in requested_service_rules:
1047 explicit_service_managers = requested_service_rule.service_managers is not None
1048 related_service_managers = requested_service_rule.service_managers or frozenset(
1049 all_service_managers
1050 )
1051 seen_service_managers = set()
1052 for service_key, service_definition in _relevant_service_definitions(
1053 requested_service_rule,
1054 related_service_managers,
1055 by_service_manager_key,
1056 aliases_by_name,
1057 ):
1058 sm = service_key[-1]
1059 seen_service_managers.add(sm)
1060 by_service_manager_key[service_key] = (
1061 by_service_manager_key[service_key][0],
1062 requested_service_rule.apply_to_service_definition(service_definition),
1063 )
1064 if (
1065 explicit_service_managers
1066 and seen_service_managers != related_service_managers
1067 ):
1068 missing_sms = ", ".join(
1069 sorted(related_service_managers - seen_service_managers)
1070 )
1071 _error(
1072 f"The rule {requested_service_rule.definition_source} explicitly requested which service managers"
1073 f" it should apply to. However, the following service managers did not provide a service of that"
1074 f" name, type and scope: {missing_sms}. Please check the rule is correct and either provide the"
1075 f" missing service or update the definition match the relevant services."
1076 )
1078 per_service_manager = {}
1080 for (
1081 service_manager_details,
1082 plugin_provided_definition,
1083 ) in by_service_manager_key.values():
1084 service_manager = service_manager_details.service_manager
1085 if service_manager not in per_service_manager:
1086 per_service_manager[service_manager] = (
1087 service_manager_details,
1088 [plugin_provided_definition],
1089 )
1090 else:
1091 per_service_manager[service_manager][1].append(plugin_provided_definition)
1093 for (
1094 service_manager_details,
1095 final_service_definitions,
1096 ) in per_service_manager.values():
1097 ctrl = binary_package_data.ctrl_creator.for_plugin(
1098 service_manager_details.plugin_metadata,
1099 service_manager_details.service_manager,
1100 default_snippet_anchor=SnippetAnchor.SERVICE,
1101 )
1102 _info(f"Applying {final_service_definitions}")
1103 service_manager_details.service_integrator(
1104 final_service_definitions,
1105 ctrl,
1106 package_metadata_context,
1107 )
1110def setup_control_files(
1111 binary_package_data: "BinaryPackageData",
1112 manifest: "HighLevelManifest",
1113 dbgsym_fs_root: VirtualPath,
1114 dbgsym_ids: list[str],
1115 package_metadata_context: PackageProcessingContext,
1116 *,
1117 allow_ctrl_file_management: bool = True,
1118) -> None:
1119 binary_package = package_metadata_context.binary_package
1120 control_output_dir = binary_package_data.control_output_dir
1121 control_output_fs_path = control_output_dir.fs_path
1122 fs_root = binary_package_data.fs_root
1123 package_state = manifest.package_state_for(binary_package.name)
1125 feature_set: PluginProvidedFeatureSet = manifest.plugin_provided_feature_set
1126 metadata_maintscript_detectors = feature_set.metadata_maintscript_detectors
1127 substvars = binary_package_data.substvars
1129 provided_maintscript_snippets = package_metadata_context.manifest_configuration(
1130 binary_package, MaintainerProvidedMaintscriptSnippetContainer
1131 )
1132 snippets = DPKG_DEB_CONTROL_SCRIPTS
1133 generated_triggers = list(binary_package_data.ctrl_creator.generated_triggers())
1135 if binary_package.is_udeb:
1136 snippets = SUPPORTED_UDEB_SCRIPTS
1138 if allow_ctrl_file_management:
1139 process_alternatives(
1140 binary_package,
1141 fs_root,
1142 package_state.reserved_packager_provided_files,
1143 package_state.maintscript_snippets,
1144 substvars,
1145 )
1146 process_debconf_templates(
1147 binary_package,
1148 package_state.reserved_packager_provided_files,
1149 package_state.maintscript_snippets,
1150 substvars,
1151 control_output_fs_path,
1152 )
1154 handle_service_management(
1155 binary_package_data,
1156 manifest,
1157 package_metadata_context,
1158 fs_root,
1159 feature_set,
1160 )
1162 plugin_detector_definition: MetadataOrMaintscriptDetector
1163 for plugin_detector_definition in itertools.chain.from_iterable(
1164 metadata_maintscript_detectors.values()
1165 ):
1166 if not plugin_detector_definition.applies_to(binary_package):
1167 continue
1168 ctrl = binary_package_data.ctrl_creator.for_plugin(
1169 plugin_detector_definition.plugin_metadata,
1170 plugin_detector_definition.detector_id,
1171 )
1172 plugin_detector_definition.run_detector(
1173 fs_root, ctrl, package_metadata_context
1174 )
1176 if provided_maintscript_snippets:
1177 package_state.maintscript_snippets.apply_maintainer_provided_snippets(
1178 provided_maintscript_snippets
1179 )
1181 for script in snippets:
1182 _generate_snippet(
1183 control_output_fs_path,
1184 script,
1185 package_state.maintscript_snippets,
1186 )
1188 else:
1189 if provided_maintscript_snippets:
1190 raise AssertionError(
1191 "Internal error: Manifest allowed maintscript snippet in integration mode that does not support it"
1192 )
1193 state = manifest.package_state_for(binary_package_data.binary_package.name)
1194 if state.requested_service_rules:
1195 service_source = state.requested_service_rules[0].definition_source
1196 _error(
1197 f"Use of service definitions (such as {service_source}) is not supported in this integration mode"
1198 )
1199 for script, snippet_container in package_state.maintscript_snippets.items():
1200 for snippet in snippet_container.all_snippets():
1201 source = snippet.definition_source
1202 _error(
1203 f"This integration mode cannot use maintscript snippets"
1204 f' (since dh_installdeb has already been called). However, "{source}" triggered'
1205 f" a snippet for {script}. Please remove the offending definition if it is from"
1206 f" the manifest or file a bug if it is caused by a built-in rule."
1207 )
1209 for trigger in generated_triggers:
1210 source = f"{trigger.provider.plugin_name}:{trigger.provider_source_id}"
1211 _error(
1212 f"This integration mode must not generate triggers"
1213 f' (since dh_installdeb has already been called). However, "{source}" created'
1214 f" a trigger. Please remove the offending definition if it is from"
1215 f" the manifest or file a bug if it is caused by a built-in rule."
1216 )
1218 shlibdeps_definition = [
1219 d
1220 for d in metadata_maintscript_detectors["debputy"]
1221 if d.detector_id == "dpkg-shlibdeps"
1222 ][0]
1224 ctrl = binary_package_data.ctrl_creator.for_plugin(
1225 shlibdeps_definition.plugin_metadata,
1226 shlibdeps_definition.detector_id,
1227 )
1228 shlibdeps_definition.run_detector(fs_root, ctrl, package_metadata_context)
1230 dh_staging_dir = os.path.join("debian", binary_package.name, "DEBIAN")
1231 try:
1232 with os.scandir(dh_staging_dir) as it:
1233 existing_control_files = [
1234 f.path
1235 for f in it
1236 if f.is_file(follow_symlinks=False)
1237 and f.name not in ("control", "md5sums")
1238 ]
1239 except FileNotFoundError:
1240 existing_control_files = []
1242 if existing_control_files:
1243 cmd = ["cp", "-a"]
1244 cmd.extend(existing_control_files)
1245 cmd.append(control_output_fs_path)
1246 print_command(*cmd)
1247 subprocess.check_call(cmd)
1249 if binary_package.is_udeb:
1250 _generate_control_files(
1251 binary_package_data,
1252 package_state,
1253 control_output_dir,
1254 fs_root,
1255 substvars,
1256 # We never built udebs due to #797391, so skip over this information,
1257 # when creating the udeb
1258 None,
1259 None,
1260 )
1261 return
1263 if generated_triggers:
1264 assert allow_ctrl_file_management
1265 dest_file = os.path.join(control_output_fs_path, "triggers")
1266 with open(dest_file, "a", encoding="utf-8") as fd:
1267 fd.writelines(textwrap.dedent(f"""\
1268 # Added by {t.provider_source_id} from {t.provider.plugin_name}
1269 {t.dpkg_trigger_type} {t.dpkg_trigger_target}
1270 """) for t in generated_triggers)
1271 os.chmod(fd.fileno(), 0o644)
1273 if allow_ctrl_file_management:
1274 install_or_generate_conffiles(
1275 control_output_dir,
1276 fs_root,
1277 package_state.reserved_packager_provided_files,
1278 )
1280 _generate_control_files(
1281 binary_package_data,
1282 package_state,
1283 control_output_dir,
1284 fs_root,
1285 substvars,
1286 dbgsym_fs_root,
1287 dbgsym_ids,
1288 )
1291def _generate_snippet(
1292 control_output_dir: str,
1293 script: str,
1294 maintscript_snippets: PackageMaintscriptSnippetContainer,
1295) -> None:
1296 debputy_snippets = maintscript_snippets.get(script)
1297 if debputy_snippets is None:
1298 return
1299 reverse = script in ("prerm", "postrm")
1300 snippets = [
1301 debputy_snippets.generate_snippet(
1302 snippet_anchor=sa,
1303 reverse=reverse,
1304 )
1305 for sa in SnippetAnchor
1306 ]
1307 snippets_in_order = reversed(snippets) if reverse else snippets
1308 full_content = "".join(f"{s}\n" for s in filter(None, snippets_in_order))
1309 if not full_content:
1310 return
1311 filename = os.path.join(control_output_dir, script)
1312 with open(filename, "w") as fd:
1313 fd.write("#!/bin/sh\nset -e\n\n")
1314 if debputy_snippets.needs_debconf():
1315 fd.write(textwrap.dedent("""\
1316 # Snippet source: debputy (dependency on debconf)
1317 if [ -e /usr/share/debconf/confmodule ]; then
1318 . /usr/share/debconf/confmodule
1319 fi
1321 """))
1322 fd.write(full_content)
1323 os.chmod(fd.fileno(), 0o755) # noqa: python:S2612
1326def _add_conffiles(
1327 ctrl_root: VirtualPathBase,
1328 conffile_matches: Iterable[VirtualPath],
1329) -> None:
1330 it = iter(conffile_matches)
1331 first = next(it, None)
1332 if first is None:
1333 return
1334 conffiles = itertools.chain([first], it)
1335 with ctrl_root.open_child("conffiles", "at") as fd:
1336 for conffile_match in conffiles:
1337 conffile = conffile_match.absolute
1338 assert conffile_match.is_file
1339 fd.write(f"{conffile}\n")
1342def _ensure_base_substvars_defined(substvars: FlushableSubstvars) -> None:
1343 for substvar in ("misc:Depends", "misc:Pre-Depends"):
1344 if substvar not in substvars:
1345 substvars[substvar] = ""
1348def compute_installed_size(fs_root: VirtualPath) -> int:
1349 """Emulate dpkg-gencontrol's code for computing the default Installed-Size"""
1350 size_in_kb = 0
1351 hard_links = set()
1352 for path in fs_root.all_paths():
1353 if path.is_symlink or path.is_file:
1354 try:
1355 # If it is a VirtualPathBase instance, the use its `.stat()` method
1356 # since it might have the stat cached as a minor optimization on disk
1357 # access. Other than that, the `os.lstat` fallback is sufficient.
1358 if isinstance(path, VirtualPathBase): 1358 ↛ 1361line 1358 didn't jump to line 1361 because the condition on line 1358 was always true
1359 st = path.stat()
1360 else:
1361 st = os.lstat(path.fs_path)
1362 if st.st_nlink > 1:
1363 hl_key = (st.st_dev, st.st_ino)
1364 if hl_key in hard_links:
1365 continue
1366 hard_links.add(hl_key)
1367 size = st.st_size
1368 except PureVirtualPathError:
1369 # We just assume it is not a hard link when the path is purely virtual
1370 size = path.size
1371 path_size = (size + 1023) // 1024
1372 else:
1373 path_size = 1
1374 size_in_kb += path_size
1375 return size_in_kb
1378def _generate_dbgsym_control_file_if_relevant(
1379 binary_package: BinaryPackage,
1380 dbgsym_fs_root: VirtualPath,
1381 dbgsym_control_dir: FSControlRootDir,
1382 dbgsym_ids: str,
1383 multi_arch: str | None,
1384 dctrl: str,
1385 extra_common_params: Sequence[str],
1386) -> None:
1387 section = binary_package.archive_section
1388 component = ""
1389 extra_params = []
1390 if section is not None and "/" in section and not section.startswith("main/"):
1391 component = section.split("/", 1)[1] + "/"
1392 if multi_arch != "same":
1393 extra_params.append("-UMulti-Arch")
1394 else:
1395 extra_params.append(f"-DMulti-Arch={multi_arch}")
1396 extra_params.append("-UReplaces")
1397 extra_params.append("-UBreaks")
1398 dbgsym_control_fs_path = dbgsym_control_dir.fs_path
1399 ensure_dir(dbgsym_control_fs_path)
1400 # Pass it via cmd-line to make it more visible that we are providing the
1401 # value. It also prevents the dbgsym package from picking up this value.
1402 total_size = compute_installed_size(dbgsym_fs_root) + compute_installed_size(
1403 dbgsym_control_dir
1404 )
1405 extra_params.append(f"-VInstalled-Size={total_size}")
1406 extra_params.extend(extra_common_params)
1408 package = binary_package.name
1409 package_selector = (
1410 binary_package.name
1411 if dctrl == "debian/control"
1412 else f"{binary_package.name}-dbgsym"
1413 )
1414 dpkg_cmd = [
1415 "dpkg-gencontrol",
1416 f"-p{package_selector}",
1417 # FIXME: Support d/<pkg>.changelog at some point.
1418 "-ldebian/changelog",
1419 "-T/dev/null",
1420 f"-c{dctrl}",
1421 f"-O{dbgsym_control_fs_path}/control",
1422 # Use a placeholder for -P to ensure failure if we forgot to override a path parameter
1423 "-P/non-existent",
1424 f"-DPackage={package}-dbgsym",
1425 "-DDepends=" + package + " (= ${binary:Version})",
1426 f"-DDescription=debug symbols for {package}",
1427 f"-DSection={component}debug",
1428 f"-DBuild-Ids={dbgsym_ids}",
1429 "-UPre-Depends",
1430 "-URecommends",
1431 "-USuggests",
1432 "-UEnhances",
1433 "-UProvides",
1434 "-UEssential",
1435 "-UConflicts",
1436 "-DPriority=optional",
1437 "-UHomepage",
1438 "-UImportant",
1439 "-UBuilt-Using",
1440 "-UStatic-Built-Using",
1441 "-DAuto-Built-Package=debug-symbols",
1442 "-UProtected",
1443 *extra_params,
1444 ]
1445 print_command(*dpkg_cmd)
1446 try:
1447 subprocess.check_call(dpkg_cmd)
1448 except subprocess.CalledProcessError:
1449 _error(
1450 f"Attempting to generate DEBIAN/control file for {package}-dbgsym failed. Please review the output from "
1451 " dpkg-gencontrol above to understand what went wrong."
1452 )
1453 os.chmod(os.path.join(dbgsym_control_fs_path, "control"), 0o644)
1456def _all_parent_directories_of(directories: Iterable[str]) -> set[str]:
1457 result = {"."}
1458 for path in directories:
1459 current = os.path.dirname(path)
1460 while current and current not in result:
1461 result.add(current)
1462 current = os.path.dirname(current)
1463 return result
1466def _compute_multi_arch_for_arch_all_doc(
1467 binary_package: BinaryPackage,
1468 fs_root: InMemoryVirtualPathBase,
1469) -> str | None:
1470 if not binary_package.name.endswith(("-doc", "-docs")):
1471 # We limit by package name, since there are tricks involving a `Multi-Arch: no` depending on a
1472 # `Multi-Arch: same` to emulate `Multi-Arch: allowed`. Said `Multi-Arch: no` can have no contents.
1473 #
1474 # That case seems unrealistic for -doc/-docs packages and accordingly the limitation here.
1475 return None
1476 acceptable_no_descend_paths = {
1477 "./usr/share/doc",
1478 }
1479 acceptable_files = {f"./usr/share/lintian/overrides/{binary_package.name}"}
1480 if _any_unacceptable_paths(
1481 fs_root,
1482 acceptable_no_descend_paths=acceptable_no_descend_paths,
1483 acceptable_files=acceptable_files,
1484 ):
1485 return None
1486 return "foreign"
1489def _any_unacceptable_paths(
1490 fs_root: InMemoryVirtualPathBase,
1491 *,
1492 acceptable_no_descend_paths: list[str] | AbstractSet[str] = frozenset(),
1493 acceptable_files: list[str] | AbstractSet[str] = frozenset(),
1494) -> bool:
1495 acceptable_intermediate_dirs = _all_parent_directories_of(
1496 itertools.chain(acceptable_no_descend_paths, acceptable_files)
1497 )
1498 for fs_path, children in fs_root.walk():
1499 path = fs_path.path
1500 if path in acceptable_no_descend_paths:
1501 children.clear()
1502 continue
1503 if path in acceptable_intermediate_dirs or path in acceptable_files:
1504 continue
1505 return True
1506 return False
1509def auto_compute_multi_arch(
1510 binary_package: BinaryPackage,
1511 control_output_dir: VirtualPath,
1512 fs_root: InMemoryVirtualPathBase,
1513) -> str | None:
1514 resolved_arch = binary_package.resolved_architecture
1515 if any(
1516 script
1517 for script in ALL_CONTROL_SCRIPTS
1518 if (p := control_output_dir.get(script)) is not None and p.is_file
1519 ):
1520 return None
1522 if resolved_arch == "all":
1523 return _compute_multi_arch_for_arch_all_doc(binary_package, fs_root)
1525 resolved_multiarch = binary_package.deb_multiarch
1526 assert resolved_arch != "all"
1527 acceptable_no_descend_paths = {
1528 f"./usr/lib/{resolved_multiarch}",
1529 f"./usr/include/{resolved_multiarch}",
1530 }
1531 acceptable_files = {
1532 f"./usr/share/doc/{binary_package.name}/{basename}"
1533 for basename in (
1534 "copyright",
1535 "changelog.gz",
1536 "changelog.Debian.gz",
1537 f"changelog.Debian.{resolved_arch}.gz",
1538 "NEWS.Debian",
1539 "NEWS.Debian.gz",
1540 "README.Debian",
1541 "README.Debian.gz",
1542 )
1543 }
1545 # Note that the lintian-overrides file is deliberately omitted from the allow-list. We would have to know that the
1546 # override does not use architecture segments. With pure debputy, this is guaranteed (debputy
1547 # does not allow lintian-overrides with architecture segment). However, with a mixed debhelper + debputy,
1548 # `dh_lintian` allows it with compat 13 or older.
1550 if _any_unacceptable_paths(
1551 fs_root,
1552 acceptable_no_descend_paths=acceptable_no_descend_paths,
1553 acceptable_files=acceptable_files,
1554 ):
1555 return None
1557 return "same"
1560@functools.lru_cache
1561def _has_t64_enabled() -> bool:
1562 try:
1563 output = subprocess.check_output(
1564 ["dpkg-buildflags", "--query-features", "abi"]
1565 ).decode()
1566 except (subprocess.CalledProcessError, FileNotFoundError):
1567 return False
1569 for stanza in Deb822.iter_paragraphs(output):
1570 if stanza.get("Feature") == "time64" and stanza.get("Enabled") == "yes":
1571 return True
1572 return False
1575def _t64_migration_substvar(
1576 binary_package: BinaryPackage,
1577 control_output_dir: VirtualPath,
1578 substvars: FlushableSubstvars,
1579) -> None:
1580 name = binary_package.name
1581 compat_name = binary_package.fields.get("X-Time64-Compat")
1582 if compat_name is None and not _T64_REGEX.match(name):
1583 return
1585 if not any(
1586 p.is_file
1587 for n in ["symbols", "shlibs"]
1588 if (p := control_output_dir.get(n)) is not None
1589 ):
1590 return
1592 if compat_name is None:
1593 compat_name = name.replace("t64", "", 1)
1594 if compat_name == name:
1595 raise AssertionError(
1596 f"Failed to derive a t64 compat name for {name}. Please file a bug against debputy."
1597 " As a work around, you can explicitly provide a X-Time64-Compat header in debian/control"
1598 " where you specify the desired compat name."
1599 )
1601 arch_bits = binary_package.package_deb_architecture_variable("ARCH_BITS")
1603 if arch_bits != "32" or not _has_t64_enabled():
1604 substvars.add_dependency(
1605 _T64_PROVIDES,
1606 f"{compat_name} (= ${ binary:Version} )",
1607 )
1608 elif _T64_PROVIDES not in substvars:
1609 substvars[_T64_PROVIDES] = ""
1612@functools.lru_cache
1613def dpkg_field_list_pkg_dep() -> Sequence[str]:
1614 try:
1615 output = subprocess.check_output(
1616 [
1617 "perl",
1618 "-MDpkg::Control::Fields",
1619 "-e",
1620 r'print "$_\n" for field_list_pkg_dep',
1621 ]
1622 )
1623 except (FileNotFoundError, subprocess.CalledProcessError):
1624 _error("Could not run perl -MDpkg::Control::Fields to get a list of fields")
1625 return output.decode("utf-8").splitlines(keepends=False)
1628_SUBSTVARS_FIELDS_NOT_SUPPORTED_BY_DPKG = {
1629 "Commands",
1630}
1633@functools.lru_cache
1634def all_auto_substvars() -> Sequence[str]:
1635 result = [x for x in dpkg_field_list_pkg_dep()]
1636 result.extend(_SUBSTVARS_FIELDS_NOT_SUPPORTED_BY_DPKG)
1637 return tuple(result)
1640def _handle_auto_substvars(
1641 source: SourcePackage,
1642 dctrl_file: BinaryPackage,
1643 substvars: FlushableSubstvars,
1644 has_dbgsym: bool,
1645) -> str | None:
1646 auto_substvars_fields = all_auto_substvars()
1647 auto_substvars_fields_lc = {x.lower(): x for x in auto_substvars_fields}
1648 substvar_fields = collections.defaultdict(set)
1649 needs_dbgsym_stanza = False
1650 for substvar_name, substvar in substvars.as_substvar.items():
1651 if ":" not in substvar_name:
1652 continue
1653 if substvar.assignment_operator in ("$=", "!="):
1654 # Will create incorrect results if there is a dbgsym and we do nothing
1655 needs_dbgsym_stanza = True
1657 if substvar.assignment_operator == "$=":
1658 # Automatically handled; no need for manual merging.
1659 continue
1660 _, field = substvar_name.rsplit(":", 1)
1661 field_lc = field.lower()
1662 if field_lc not in auto_substvars_fields_lc:
1663 continue
1664 substvar_fields[field_lc].add("${" + substvar_name + "}")
1666 if not has_dbgsym:
1667 needs_dbgsym_stanza = False
1669 if not substvar_fields and not needs_dbgsym_stanza:
1670 return None
1672 replacement_stanza = debian.deb822.Deb822(dctrl_file.fields)
1674 for field_name in auto_substvars_fields:
1675 field_name_lc = field_name.lower()
1676 addendum = substvar_fields.get(field_name_lc)
1677 if addendum is None:
1678 # No merging required
1679 continue
1680 substvars_part = ", ".join(sorted(addendum))
1681 existing_value = replacement_stanza.get(field_name)
1683 if existing_value is None or existing_value.isspace():
1684 final_value = substvars_part
1685 else:
1686 existing_value = existing_value.rstrip().rstrip(",")
1687 final_value = f"{existing_value}, {substvars_part}"
1688 replacement_stanza[field_name] = final_value
1689 canonical_field_name = auto_substvars_fields_lc.get(field_name_lc)
1690 # If `dpkg` does not know the field, we need to inject `XB-` in front
1691 # of it.
1692 if (
1693 canonical_field_name
1694 and canonical_field_name in _SUBSTVARS_FIELDS_NOT_SUPPORTED_BY_DPKG
1695 ):
1696 replacement_stanza[f"XB-{canonical_field_name}"] = replacement_stanza[
1697 field_name
1698 ]
1699 del replacement_stanza[field_name]
1701 with suppress(KeyError):
1702 replacement_stanza.order_last("Description")
1704 tmpdir = generated_content_dir(package=dctrl_file)
1705 with tempfile.NamedTemporaryFile(
1706 mode="wb",
1707 dir=tmpdir,
1708 suffix="__DEBIAN_control",
1709 delete=False,
1710 ) as fd:
1711 try:
1712 cast("Any", source.fields).dump(fd)
1713 except AttributeError:
1714 debian.deb822.Deb822(source.fields).dump(fd)
1715 fd.write(b"\n")
1716 replacement_stanza.dump(fd)
1718 if has_dbgsym:
1719 # Minimal stanza to avoid substvars warnings. Most fields are still set
1720 # via -D.
1721 dbgsym_stanza = Deb822()
1722 dbgsym_stanza["Package"] = f"{dctrl_file.name}-dbgsym"
1723 dbgsym_stanza["Architecture"] = dctrl_file.fields["Architecture"]
1724 dbgsym_stanza["Description"] = f"debug symbols for {dctrl_file.name}"
1725 fd.write(b"\n")
1726 dbgsym_stanza.dump(fd)
1728 return fd.name
1731def _generate_control_files(
1732 binary_package_data: "BinaryPackageData",
1733 package_state: "PackageTransformationDefinition",
1734 control_output_dir: FSControlRootDir,
1735 fs_root: InMemoryVirtualPathBase,
1736 substvars: FlushableSubstvars,
1737 dbgsym_root_fs: VirtualPath | None,
1738 dbgsym_build_ids: list[str] | None,
1739) -> None:
1740 binary_package = binary_package_data.binary_package
1741 source_package = binary_package_data.source_package
1742 package_name = binary_package.name
1743 extra_common_params = []
1744 extra_params_specific = []
1745 _ensure_base_substvars_defined(substvars)
1746 if "Installed-Size" not in substvars:
1747 # Pass it via cmd-line to make it more visible that we are providing the
1748 # value. It also prevents the dbgsym package from picking up this value.
1749 total_size = compute_installed_size(fs_root) + compute_installed_size(
1750 control_output_dir
1751 )
1752 extra_params_specific.append(f"-VInstalled-Size={total_size}")
1754 ma_value = binary_package.fields.get("Multi-Arch")
1755 if not binary_package.is_udeb and ma_value is None:
1756 ma_value = auto_compute_multi_arch(binary_package, control_output_dir, fs_root)
1757 if ma_value is not None:
1758 _info(
1759 f'The package "{binary_package.name}" looks like it should be "Multi-Arch: {ma_value}" based'
1760 ' on the contents and there is no explicit "Multi-Arch" field. Setting the Multi-Arch field'
1761 ' accordingly in the binary. If this auto-correction is wrong, please add "Multi-Arch: no" to the'
1762 ' relevant part of "debian/control" to disable this feature.'
1763 )
1764 # We want this to apply to the `-dbgsym` package as well to avoid
1765 # lintian `debug-package-for-multi-arch-same-pkg-not-coinstallable`
1766 extra_common_params.append(f"-DMulti-Arch={ma_value}")
1767 elif ma_value == "no":
1768 extra_common_params.append("-UMulti-Arch")
1770 dbgsym_ids = " ".join(dbgsym_build_ids) if dbgsym_build_ids else ""
1771 if package_state.binary_version is not None:
1772 extra_common_params.append(f"-v{package_state.binary_version}")
1774 _t64_migration_substvar(binary_package, control_output_dir, substvars)
1776 with substvars.flush() as flushed_substvars:
1777 has_dbgsym = dbgsym_root_fs is not None and any(
1778 f for f in dbgsym_root_fs.all_paths() if f.is_file
1779 )
1780 dctrl_file = _handle_auto_substvars(
1781 source_package,
1782 binary_package,
1783 substvars,
1784 has_dbgsym,
1785 )
1786 if dctrl_file is None:
1787 dctrl_file = "debian/control"
1789 if has_dbgsym:
1790 assert dbgsym_root_fs is not None # mypy hint
1791 dbgsym_ctrl_dir = binary_package_data.dbgsym_info.dbgsym_ctrl_dir
1792 _generate_dbgsym_control_file_if_relevant(
1793 binary_package,
1794 dbgsym_root_fs,
1795 dbgsym_ctrl_dir,
1796 dbgsym_ids,
1797 ma_value,
1798 dctrl_file,
1799 extra_common_params,
1800 )
1801 generate_md5sums_file(
1802 dbgsym_ctrl_dir,
1803 dbgsym_root_fs,
1804 )
1805 elif dbgsym_ids:
1806 extra_common_params.append(f"-DBuild-Ids={dbgsym_ids}")
1808 ctrl_file = os.path.join(control_output_dir.fs_path, "control")
1809 dpkg_cmd = [
1810 "dpkg-gencontrol",
1811 f"-p{package_name}",
1812 # FIXME: Support d/<pkg>.changelog at some point.
1813 "-ldebian/changelog",
1814 f"-c{dctrl_file}",
1815 f"-T{flushed_substvars}",
1816 f"-O{ctrl_file}",
1817 # Use a placeholder for -P to ensure failure if we forgot to override a path parameter
1818 "-P/non-existent",
1819 *extra_common_params,
1820 *extra_params_specific,
1821 ]
1822 print_command(*dpkg_cmd)
1823 try:
1824 subprocess.check_call(dpkg_cmd)
1825 except subprocess.CalledProcessError:
1826 _error(
1827 f"Attempting to generate DEBIAN/control file for {package_name} failed. Please review the output from "
1828 " dpkg-gencontrol above to understand what went wrong."
1829 )
1830 os.chmod(ctrl_file, 0o644)
1832 if not binary_package.is_udeb:
1833 generate_md5sums_file(control_output_dir, fs_root)