Coverage for src/debputy/dh_migration/migrators_impl.py: 81%
725 statements
« prev ^ index » next coverage.py v7.8.2, created at 2025-09-07 09:27 +0000
1import collections
2import dataclasses
3import functools
4import json
5import os
6import re
7import subprocess
8from itertools import product, chain
9from typing import (
10 Iterable,
11 Optional,
12 Tuple,
13 List,
14 Set,
15 Mapping,
16 Any,
17 Union,
18 Callable,
19 TypeVar,
20 Dict,
21 Container,
22)
24from debian.deb822 import Deb822
26from debputy import DEBPUTY_DOC_ROOT_DIR
27from debputy.architecture_support import dpkg_architecture_table
28from debputy.deb_packaging_support import dpkg_field_list_pkg_dep
29from debputy.dh.debhelper_emulation import (
30 dhe_filedoublearray,
31 DHConfigFileLine,
32 dhe_pkgfile,
33)
34from debputy.dh.dh_assistant import (
35 read_dh_addon_sequences,
36)
37from debputy.dh_migration.models import (
38 ConflictingChange,
39 FeatureMigration,
40 UnsupportedFeature,
41 AcceptableMigrationIssues,
42 DHMigrationSubstitution,
43)
44from debputy.highlevel_manifest import (
45 MutableYAMLSymlink,
46 HighLevelManifest,
47 MutableYAMLConffileManagementItem,
48 AbstractMutableYAMLInstallRule,
49)
50from debputy.installations import MAN_GUESS_FROM_BASENAME, MAN_GUESS_LANG_FROM_PATH
51from debputy.packages import BinaryPackage
52from debputy.plugin.api import VirtualPath
53from debputy.plugin.api.spec import (
54 INTEGRATION_MODE_DH_DEBPUTY_RRR,
55 INTEGRATION_MODE_DH_DEBPUTY,
56 DebputyIntegrationMode,
57 INTEGRATION_MODE_FULL,
58)
59from debputy.util import (
60 _error,
61 PKGVERSION_REGEX,
62 PKGNAME_REGEX,
63 _normalize_path,
64 assume_not_none,
65 has_glob_magic,
66)
69class ContainsEverything:
71 def __contains__(self, item: str) -> bool:
72 return True
75# Align with debputy.py
76DH_COMMANDS_REPLACED: Mapping[DebputyIntegrationMode, Container[str]] = {
77 INTEGRATION_MODE_DH_DEBPUTY_RRR: frozenset(
78 {
79 "dh_fixperms",
80 "dh_shlibdeps",
81 "dh_gencontrol",
82 "dh_md5sums",
83 "dh_builddeb",
84 }
85 ),
86 INTEGRATION_MODE_DH_DEBPUTY: frozenset(
87 {
88 "dh_install",
89 "dh_installdocs",
90 "dh_installchangelogs",
91 "dh_installexamples",
92 "dh_installman",
93 "dh_installcatalogs",
94 "dh_installcron",
95 "dh_installdebconf",
96 "dh_installemacsen",
97 "dh_installifupdown",
98 "dh_installinfo",
99 "dh_installinit",
100 "dh_installsysusers",
101 "dh_installtmpfiles",
102 "dh_installsystemd",
103 "dh_installsystemduser",
104 "dh_installmenu",
105 "dh_installmime",
106 "dh_installmodules",
107 "dh_installlogcheck",
108 "dh_installlogrotate",
109 "dh_installpam",
110 "dh_installppp",
111 "dh_installudev",
112 "dh_installgsettings",
113 "dh_installinitramfs",
114 "dh_installalternatives",
115 "dh_bugfiles",
116 "dh_ucf",
117 "dh_lintian",
118 "dh_icons",
119 "dh_usrlocal",
120 "dh_perl",
121 "dh_link",
122 "dh_installwm",
123 "dh_installxfonts",
124 "dh_strip_nondeterminism",
125 "dh_compress",
126 "dh_fixperms",
127 "dh_dwz",
128 "dh_strip",
129 "dh_makeshlibs",
130 "dh_shlibdeps",
131 "dh_missing",
132 "dh_installdeb",
133 "dh_gencontrol",
134 "dh_md5sums",
135 "dh_builddeb",
136 }
137 ),
138 INTEGRATION_MODE_FULL: ContainsEverything(),
139}
141_GS_DOC = f"{DEBPUTY_DOC_ROOT_DIR}/GETTING-STARTED-WITH-dh-debputy.md"
142MIGRATION_AID_FOR_OVERRIDDEN_COMMANDS = {
143 "dh_installinit": f"{_GS_DOC}#covert-your-overrides-for-dh_installsystemd-dh_installinit-if-any",
144 "dh_installsystemd": f"{_GS_DOC}#covert-your-overrides-for-dh_installsystemd-dh_installinit-if-any",
145 "dh_fixperms": f"{_GS_DOC}#convert-your-overrides-or-excludes-for-dh_fixperms-if-any",
146 "dh_gencontrol": f"{_GS_DOC}#convert-your-overrides-for-dh_gencontrol-if-any",
147}
150@dataclasses.dataclass(frozen=True, slots=True)
151class UnsupportedDHConfig:
152 dh_config_basename: str
153 dh_tool: str
154 bug_950723_prefix_matching: bool = False
155 is_missing_migration: bool = False
158@dataclasses.dataclass(frozen=True, slots=True)
159class DHSequenceMigration:
160 debputy_plugin: str
161 remove_dh_sequence: bool = True
162 must_use_zz_debputy: bool = False
165UNSUPPORTED_DH_CONFIGS_AND_TOOLS_FOR_ZZ_DEBPUTY = [
166 UnsupportedDHConfig("config", "dh_installdebconf"),
167 UnsupportedDHConfig("templates", "dh_installdebconf"),
168 UnsupportedDHConfig("emacsen-compat", "dh_installemacsen"),
169 UnsupportedDHConfig("emacsen-install", "dh_installemacsen"),
170 UnsupportedDHConfig("emacsen-remove", "dh_installemacsen"),
171 UnsupportedDHConfig("emacsen-startup", "dh_installemacsen"),
172 # The `upstart` file should be long dead, but we might as well detect it.
173 UnsupportedDHConfig("upstart", "dh_installinit"),
174 # dh_installsystemduser
175 UnsupportedDHConfig(
176 "user.path", "dh_installsystemduser", bug_950723_prefix_matching=False
177 ),
178 UnsupportedDHConfig(
179 "user.path", "dh_installsystemduser", bug_950723_prefix_matching=True
180 ),
181 UnsupportedDHConfig(
182 "user.service", "dh_installsystemduser", bug_950723_prefix_matching=False
183 ),
184 UnsupportedDHConfig(
185 "user.service", "dh_installsystemduser", bug_950723_prefix_matching=True
186 ),
187 UnsupportedDHConfig(
188 "user.socket", "dh_installsystemduser", bug_950723_prefix_matching=False
189 ),
190 UnsupportedDHConfig(
191 "user.socket", "dh_installsystemduser", bug_950723_prefix_matching=True
192 ),
193 UnsupportedDHConfig(
194 "user.target", "dh_installsystemduser", bug_950723_prefix_matching=False
195 ),
196 UnsupportedDHConfig(
197 "user.target", "dh_installsystemduser", bug_950723_prefix_matching=True
198 ),
199 UnsupportedDHConfig(
200 "user.timer", "dh_installsystemduser", bug_950723_prefix_matching=False
201 ),
202 UnsupportedDHConfig(
203 "user.timer", "dh_installsystemduser", bug_950723_prefix_matching=True
204 ),
205 UnsupportedDHConfig("menu", "dh_installmenu"),
206 UnsupportedDHConfig("menu-method", "dh_installmenu"),
207 UnsupportedDHConfig("ucf", "dh_ucf"),
208 UnsupportedDHConfig("wm", "dh_installwm"),
209 UnsupportedDHConfig("triggers", "dh_installdeb"),
210 UnsupportedDHConfig("postinst", "dh_installdeb"),
211 UnsupportedDHConfig("postrm", "dh_installdeb"),
212 UnsupportedDHConfig("preinst", "dh_installdeb"),
213 UnsupportedDHConfig("prerm", "dh_installdeb"),
214 UnsupportedDHConfig("menutest", "dh_installdeb"),
215 UnsupportedDHConfig("isinstallable", "dh_installdeb"),
216]
217SUPPORTED_DH_ADDONS_WITH_ZZ_DEBPUTY = frozenset(
218 {
219 # debputy's own
220 "debputy",
221 "zz-debputy",
222 # debhelper provided sequences that should work.
223 "single-binary",
224 }
225)
226DH_ADDONS_TO_REMOVE_FOR_ZZ_DEBPUTY = frozenset(
227 [
228 # The `zz-debputy` add-on replaces the `zz-debputy-rrr` plugin.
229 "zz-debputy-rrr",
230 # Sequences debputy directly replaces
231 "dwz",
232 "elf-tools",
233 "installinitramfs",
234 "installsysusers",
235 "doxygen",
236 # Sequences that are embedded fully into debputy
237 "bash-completion",
238 "shell-completions",
239 "sodeps",
240 ]
241)
242DH_ADDONS_TO_PLUGINS = {
243 "gnome": DHSequenceMigration(
244 "gnome",
245 # The sequence still provides a command for the clean sequence
246 remove_dh_sequence=False,
247 must_use_zz_debputy=True,
248 ),
249 "grantlee": DHSequenceMigration(
250 "grantlee",
251 remove_dh_sequence=True,
252 must_use_zz_debputy=True,
253 ),
254 "numpy3": DHSequenceMigration(
255 "numpy3",
256 # The sequence provides (build-time) dependencies that we cannot provide
257 remove_dh_sequence=False,
258 must_use_zz_debputy=True,
259 ),
260 "perl-openssl": DHSequenceMigration(
261 "perl-openssl",
262 # The sequence provides (build-time) dependencies that we cannot provide
263 remove_dh_sequence=False,
264 must_use_zz_debputy=True,
265 ),
266}
269def _dh_config_file(
270 debian_dir: VirtualPath,
271 dctrl_bin: BinaryPackage,
272 basename: str,
273 helper_name: str,
274 acceptable_migration_issues: AcceptableMigrationIssues,
275 feature_migration: FeatureMigration,
276 manifest: HighLevelManifest,
277 support_executable_files: bool = False,
278 allow_dh_exec_rename: bool = False,
279 pkgfile_lookup: bool = True,
280 remove_on_migration: bool = True,
281) -> Union[Tuple[None, None], Tuple[VirtualPath, Iterable[DHConfigFileLine]]]:
282 mutable_manifest = assume_not_none(manifest.mutable_manifest)
283 dh_config_file = (
284 dhe_pkgfile(debian_dir, dctrl_bin, basename)
285 if pkgfile_lookup
286 else debian_dir.get(basename)
287 )
288 if dh_config_file is None or dh_config_file.is_dir:
289 return None, None
290 if dh_config_file.is_executable and not support_executable_files:
291 primary_key = f"executable-{helper_name}-config"
292 if (
293 primary_key in acceptable_migration_issues
294 or "any-executable-dh-configs" in acceptable_migration_issues
295 ):
296 feature_migration.warn(
297 f'TODO: MANUAL MIGRATION of executable dh config "{dh_config_file}" is required.'
298 )
299 return None, None
300 raise UnsupportedFeature(
301 f"Executable configuration files not supported (found: {dh_config_file}).",
302 [primary_key, "any-executable-dh-configs"],
303 )
305 if remove_on_migration:
306 feature_migration.remove_on_success(dh_config_file.fs_path)
307 substitution = DHMigrationSubstitution(
308 dpkg_architecture_table(),
309 acceptable_migration_issues,
310 feature_migration,
311 mutable_manifest,
312 )
313 content = dhe_filedoublearray(
314 dh_config_file,
315 substitution,
316 allow_dh_exec_rename=allow_dh_exec_rename,
317 )
318 return dh_config_file, content
321def _validate_rm_mv_conffile(
322 package: str,
323 config_line: DHConfigFileLine,
324) -> Tuple[str, str, Optional[str], Optional[str], Optional[str]]:
325 cmd, *args = config_line.tokens
326 if "--" in config_line.tokens: 326 ↛ 327line 326 didn't jump to line 327 because the condition on line 326 was never true
327 raise ValueError(
328 f'The maintscripts file "{config_line.config_file.path}" for {package} includes a "--" in line'
329 f" {config_line.line_no}. The offending line is: {config_line.original_line}"
330 )
331 if cmd == "rm_conffile":
332 min_args = 1
333 max_args = 3
334 else:
335 min_args = 2
336 max_args = 4
337 if len(args) > max_args or len(args) < min_args: 337 ↛ 338line 337 didn't jump to line 338 because the condition on line 337 was never true
338 raise ValueError(
339 f'The "{cmd}" command takes at least {min_args} and at most {max_args} arguments. However,'
340 f' in "{config_line.config_file.path}" line {config_line.line_no} (for {package}), there'
341 f" are {len(args)} arguments. The offending line is: {config_line.original_line}"
342 )
344 obsolete_conffile = args[0]
345 new_conffile = args[1] if cmd == "mv_conffile" else None
346 prior_version = args[min_args] if len(args) > min_args else None
347 owning_package = args[min_args + 1] if len(args) > min_args + 1 else None
348 if not obsolete_conffile.startswith("/"): 348 ↛ 349line 348 didn't jump to line 349 because the condition on line 348 was never true
349 raise ValueError(
350 f'The (old-)conffile parameter for {cmd} must be absolute (i.e., start with "/"). However,'
351 f' in "{config_line.config_file.path}" line {config_line.line_no} (for {package}), it was specified'
352 f' as "{obsolete_conffile}". The offending line is: {config_line.original_line}'
353 )
354 if new_conffile is not None and not new_conffile.startswith("/"): 354 ↛ 355line 354 didn't jump to line 355 because the condition on line 354 was never true
355 raise ValueError(
356 f'The new-conffile parameter for {cmd} must be absolute (i.e., start with "/"). However,'
357 f' in "{config_line.config_file.path}" line {config_line.line_no} (for {package}), it was specified'
358 f' as "{new_conffile}". The offending line is: {config_line.original_line}'
359 )
360 if prior_version is not None and not PKGVERSION_REGEX.fullmatch(prior_version): 360 ↛ 361line 360 didn't jump to line 361 because the condition on line 360 was never true
361 raise ValueError(
362 f"The prior-version parameter for {cmd} must be a valid package version (i.e., match"
363 f' {PKGVERSION_REGEX}). However, in "{config_line.config_file.path}" line {config_line.line_no}'
364 f' (for {package}), it was specified as "{prior_version}". The offending line is:'
365 f" {config_line.original_line}"
366 )
367 if owning_package is not None and not PKGNAME_REGEX.fullmatch(owning_package): 367 ↛ 368line 367 didn't jump to line 368 because the condition on line 367 was never true
368 raise ValueError(
369 f"The package parameter for {cmd} must be a valid package name (i.e., match {PKGNAME_REGEX})."
370 f' However, in "{config_line.config_file.path}" line {config_line.line_no} (for {package}), it'
371 f' was specified as "{owning_package}". The offending line is: {config_line.original_line}'
372 )
373 return cmd, obsolete_conffile, new_conffile, prior_version, owning_package
376_BASH_COMPLETION_RE = re.compile(
377 r"""
378 (^|[|&;])\s*complete.*-[A-Za-z].*
379 | \$\(.*\)
380 | \s*compgen.*-[A-Za-z].*
381 | \s*if.*;.*then/
382""",
383 re.VERBOSE,
384)
387def migrate_bash_completion(
388 debian_dir: VirtualPath,
389 manifest: HighLevelManifest,
390 acceptable_migration_issues: AcceptableMigrationIssues,
391 feature_migration: FeatureMigration,
392 _migration_target: Optional[DebputyIntegrationMode],
393) -> None:
394 feature_migration.tagline = "dh_bash-completion files"
395 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
396 mutable_manifest = assume_not_none(manifest.mutable_manifest)
397 installations = mutable_manifest.installations(create_if_absent=False)
399 for dctrl_bin in manifest.all_packages:
400 dh_file = dhe_pkgfile(debian_dir, dctrl_bin, "bash-completion")
401 if dh_file is None:
402 continue
403 is_bash_completion_file = False
404 with dh_file.open() as fd:
405 for line in fd:
406 line = line.strip()
407 if not line or line[0] == "#": 407 ↛ 408line 407 didn't jump to line 408 because the condition on line 407 was never true
408 continue
409 if _BASH_COMPLETION_RE.search(line):
410 is_bash_completion_file = True
411 break
412 if not is_bash_completion_file:
413 _, content = _dh_config_file(
414 debian_dir,
415 dctrl_bin,
416 "bash-completion",
417 "dh_bash-completion",
418 acceptable_migration_issues,
419 feature_migration,
420 manifest,
421 support_executable_files=True,
422 )
423 else:
424 content = None
426 if content:
427 install_dest_sources: List[str] = []
428 install_as_rules: List[Tuple[str, str]] = []
429 for dhe_line in content:
430 if len(dhe_line.tokens) > 2: 430 ↛ 431line 430 didn't jump to line 431 because the condition on line 430 was never true
431 raise UnsupportedFeature(
432 f"The dh_bash-completion file {dh_file.path} more than two words on"
433 f' line {dhe_line.line_no} (line: "{dhe_line.original_line}").'
434 )
435 source = dhe_line.tokens[0]
436 dest_basename = (
437 dhe_line.tokens[1]
438 if len(dhe_line.tokens) > 1
439 else os.path.basename(source)
440 )
441 if source.startswith("debian/") and not has_glob_magic(source):
442 if dctrl_bin.name != dest_basename:
443 dest_path = (
444 f"debian/{dctrl_bin.name}.{dest_basename}.bash-completion"
445 )
446 else:
447 dest_path = f"debian/{dest_basename}.bash-completion"
448 feature_migration.rename_on_success(source, dest_path)
449 elif len(dhe_line.tokens) == 1:
450 install_dest_sources.append(source)
451 else:
452 install_as_rules.append((source, dest_basename))
454 if install_dest_sources: 454 ↛ 468line 454 didn't jump to line 468 because the condition on line 454 was always true
455 sources: Union[List[str], str] = (
456 install_dest_sources
457 if len(install_dest_sources) > 1
458 else install_dest_sources[0]
459 )
460 installations.append(
461 AbstractMutableYAMLInstallRule.install_dest(
462 sources=sources,
463 dest_dir="{{path:BASH_COMPLETION_DIR}}",
464 into=dctrl_bin.name if not is_single_binary else None,
465 )
466 )
468 for source, dest_basename in install_as_rules:
469 installations.append(
470 AbstractMutableYAMLInstallRule.install_as(
471 source=source,
472 install_as="{{path:BASH_COMPLETION_DIR}}/" + dest_basename,
473 into=dctrl_bin.name if not is_single_binary else None,
474 )
475 )
478_SHELL_COMPLETIONS_RE = re.compile(r"^\s*\S+\s+\S+\s+\S")
481def migrate_shell_completions(
482 debian_dir: VirtualPath,
483 manifest: HighLevelManifest,
484 acceptable_migration_issues: AcceptableMigrationIssues,
485 feature_migration: FeatureMigration,
486 _migration_target: Optional[DebputyIntegrationMode],
487) -> None:
488 feature_migration.tagline = "dh_shell_completions files"
489 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
490 mutable_manifest = assume_not_none(manifest.mutable_manifest)
491 installations = mutable_manifest.installations(create_if_absent=False)
492 # Note: The bash completion script used `bash-completion` whereas `dh_shell_completions` uses
493 # `...-completions` (note the trailing `s`). In `debputy`, we always use the singular notation
494 # because we use "one file, one completion (ruleset)".
495 completions = ["bash", "fish", "zsh"]
497 for completion, dctrl_bin in product(completions, manifest.all_packages):
498 dh_file = dhe_pkgfile(debian_dir, dctrl_bin, f"{completion}-completions")
499 if dh_file is None:
500 continue
501 is_completion_file = False
502 with dh_file.open() as fd:
503 for line in fd:
504 line = line.strip()
505 if not line or line[0] == "#":
506 continue
507 if _SHELL_COMPLETIONS_RE.search(line):
508 is_completion_file = True
509 break
510 if is_completion_file:
511 dest_path = f"debian/{dctrl_bin.name}.{completion}-completion"
512 feature_migration.rename_on_success(dh_file.fs_path, dest_path)
513 continue
515 _, content = _dh_config_file(
516 debian_dir,
517 dctrl_bin,
518 f"{completion}-completions",
519 "dh_shell_completions",
520 acceptable_migration_issues,
521 feature_migration,
522 manifest,
523 remove_on_migration=True,
524 )
526 if content: 526 ↛ 497line 526 didn't jump to line 497 because the condition on line 526 was always true
527 install_dest_sources: List[str] = []
528 install_as_rules: List[Tuple[str, str]] = []
529 for dhe_line in content:
530 if len(dhe_line.tokens) > 2: 530 ↛ 531line 530 didn't jump to line 531 because the condition on line 530 was never true
531 raise UnsupportedFeature(
532 f"The dh_shell_completions file {dh_file.path} more than two words on"
533 f' line {dhe_line.line_no} (line: "{dhe_line.original_line}").'
534 )
535 source = dhe_line.tokens[0]
536 dest_basename = (
537 dhe_line.tokens[1]
538 if len(dhe_line.tokens) > 1
539 else os.path.basename(source)
540 )
541 if source.startswith("debian/") and not has_glob_magic(source):
542 if dctrl_bin.name != dest_basename:
543 dest_path = f"debian/{dctrl_bin.name}.{dest_basename}.{completion}-completion"
544 else:
545 dest_path = f"debian/{dest_basename}.{completion}-completion"
546 feature_migration.rename_on_success(source, dest_path)
547 elif len(dhe_line.tokens) == 1:
548 install_dest_sources.append(source)
549 else:
550 install_as_rules.append((source, dest_basename))
552 completion_dir_variable = (
553 "{{path:" + f"{completion.upper()}_COMPLETION_DIR" + "}}"
554 )
556 if install_dest_sources: 556 ↛ 570line 556 didn't jump to line 570 because the condition on line 556 was always true
557 sources: Union[List[str], str] = (
558 install_dest_sources
559 if len(install_dest_sources) > 1
560 else install_dest_sources[0]
561 )
562 installations.append(
563 AbstractMutableYAMLInstallRule.install_dest(
564 sources=sources,
565 dest_dir=completion_dir_variable,
566 into=dctrl_bin.name if not is_single_binary else None,
567 )
568 )
570 for source, dest_basename in install_as_rules:
571 installations.append(
572 AbstractMutableYAMLInstallRule.install_as(
573 source=source,
574 install_as=f"{completion_dir_variable}/{dest_basename}",
575 into=dctrl_bin.name if not is_single_binary else None,
576 )
577 )
580def migrate_dh_installsystemd_files(
581 debian_dir: VirtualPath,
582 manifest: HighLevelManifest,
583 _acceptable_migration_issues: AcceptableMigrationIssues,
584 feature_migration: FeatureMigration,
585 _migration_target: Optional[DebputyIntegrationMode],
586) -> None:
587 feature_migration.tagline = "dh_installsystemd files"
588 for dctrl_bin in manifest.all_packages:
589 for stem in [
590 "path",
591 "service",
592 "socket",
593 "target",
594 "timer",
595 ]:
596 pkgfile = dhe_pkgfile(
597 debian_dir, dctrl_bin, stem, bug_950723_prefix_matching=True
598 )
599 if not pkgfile:
600 continue
601 if not pkgfile.name.endswith(f".{stem}") or "@." not in pkgfile.name: 601 ↛ 602line 601 didn't jump to line 602 because the condition on line 601 was never true
602 raise UnsupportedFeature(
603 f'Unable to determine the correct name for {pkgfile.fs_path}. It should be a ".@{stem}"'
604 f" file now (foo@.service => foo.@service)"
605 )
606 newname = pkgfile.name.replace("@.", ".")
607 newname = newname[: -len(stem)] + f"@{stem}"
608 feature_migration.rename_on_success(
609 pkgfile.fs_path, os.path.join(debian_dir.fs_path, newname)
610 )
613def migrate_clean_file(
614 debian_dir: VirtualPath,
615 manifest: HighLevelManifest,
616 acceptable_migration_issues: AcceptableMigrationIssues,
617 feature_migration: FeatureMigration,
618 _migration_target: Optional[DebputyIntegrationMode],
619) -> None:
620 feature_migration.tagline = "debian/clean"
621 clean_file = debian_dir.get("clean")
622 if clean_file is None:
623 return
625 substitution = DHMigrationSubstitution(
626 dpkg_architecture_table(),
627 acceptable_migration_issues,
628 feature_migration,
629 manifest.mutable_manifest,
630 )
631 content = dhe_filedoublearray(
632 clean_file,
633 substitution,
634 )
636 remove_during_clean_rules = manifest.mutable_manifest.remove_during_clean(
637 create_if_absent=False
638 )
639 tokens = chain.from_iterable(c.tokens for c in content)
640 rules_before = len(remove_during_clean_rules)
641 remove_during_clean_rules.extend(tokens)
642 rules_after = len(remove_during_clean_rules)
643 feature_migration.successful_manifest_changes += rules_after - rules_before
644 feature_migration.remove_on_success(clean_file.fs_path)
647def migrate_maintscript(
648 debian_dir: VirtualPath,
649 manifest: HighLevelManifest,
650 acceptable_migration_issues: AcceptableMigrationIssues,
651 feature_migration: FeatureMigration,
652 _migration_target: Optional[DebputyIntegrationMode],
653) -> None:
654 feature_migration.tagline = "dh_installdeb files"
655 mutable_manifest = assume_not_none(manifest.mutable_manifest)
656 for dctrl_bin in manifest.all_packages:
657 mainscript_file, content = _dh_config_file(
658 debian_dir,
659 dctrl_bin,
660 "maintscript",
661 "dh_installdeb",
662 acceptable_migration_issues,
663 feature_migration,
664 manifest,
665 )
667 if mainscript_file is None:
668 continue
669 assert content is not None
671 package_definition = mutable_manifest.package(dctrl_bin.name)
672 conffiles = {
673 it.obsolete_conffile: it
674 for it in package_definition.conffile_management_items()
675 }
676 seen_conffiles = set()
678 for dhe_line in content:
679 cmd = dhe_line.tokens[0]
680 if cmd not in {"rm_conffile", "mv_conffile"}: 680 ↛ 681line 680 didn't jump to line 681 because the condition on line 680 was never true
681 raise UnsupportedFeature(
682 f"The dh_installdeb file {mainscript_file.path} contains the (currently)"
683 f' unsupported command "{cmd}" on line {dhe_line.line_no}'
684 f' (line: "{dhe_line.original_line}")'
685 )
687 try:
688 (
689 _,
690 obsolete_conffile,
691 new_conffile,
692 prior_to_version,
693 owning_package,
694 ) = _validate_rm_mv_conffile(dctrl_bin.name, dhe_line)
695 except ValueError as e:
696 _error(
697 f"Validation error in {mainscript_file} on line {dhe_line.line_no}. The error was: {e.args[0]}."
698 )
700 if obsolete_conffile in seen_conffiles: 700 ↛ 701line 700 didn't jump to line 701 because the condition on line 700 was never true
701 raise ConflictingChange(
702 f'The {mainscript_file} file defines actions for "{obsolete_conffile}" twice!'
703 f" Please ensure that it is defined at most once in that file."
704 )
705 seen_conffiles.add(obsolete_conffile)
707 if cmd == "rm_conffile":
708 item = MutableYAMLConffileManagementItem.rm_conffile(
709 obsolete_conffile,
710 prior_to_version,
711 owning_package,
712 )
713 else:
714 assert cmd == "mv_conffile"
715 item = MutableYAMLConffileManagementItem.mv_conffile(
716 obsolete_conffile,
717 assume_not_none(new_conffile),
718 prior_to_version,
719 owning_package,
720 )
722 existing_def = conffiles.get(item.obsolete_conffile)
723 if existing_def is not None: 723 ↛ 724line 723 didn't jump to line 724 because the condition on line 723 was never true
724 if not (
725 item.command == existing_def.command
726 and item.new_conffile == existing_def.new_conffile
727 and item.prior_to_version == existing_def.prior_to_version
728 and item.owning_package == existing_def.owning_package
729 ):
730 raise ConflictingChange(
731 f"The maintscript defines the action {item.command} for"
732 f' "{obsolete_conffile}" in {mainscript_file}, but there is another'
733 f" conffile management definition for same path defined already (in the"
734 f" existing manifest or an migration e.g., inside {mainscript_file})"
735 )
736 continue
738 package_definition.add_conffile_management(item)
739 feature_migration.successful_manifest_changes += 1
742@dataclasses.dataclass(slots=True)
743class SourcesAndConditional:
744 dest_dir: Optional[str] = None
745 sources: List[str] = dataclasses.field(default_factory=list)
746 conditional: Optional[Union[str, Mapping[str, Any]]] = None
749def _strip_d_tmp(p: str) -> str:
750 if p.startswith("debian/tmp/") and len(p) > 11:
751 return p[11:]
752 return p
755def migrate_install_file(
756 debian_dir: VirtualPath,
757 manifest: HighLevelManifest,
758 acceptable_migration_issues: AcceptableMigrationIssues,
759 feature_migration: FeatureMigration,
760 _migration_target: Optional[DebputyIntegrationMode],
761) -> None:
762 feature_migration.tagline = "dh_install config files"
763 mutable_manifest = assume_not_none(manifest.mutable_manifest)
764 installations = mutable_manifest.installations(create_if_absent=False)
765 priority_lines = []
766 remaining_install_lines = []
767 warn_about_fixmes_in_dest_dir = False
769 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
771 for dctrl_bin in manifest.all_packages:
772 install_file, content = _dh_config_file(
773 debian_dir,
774 dctrl_bin,
775 "install",
776 "dh_install",
777 acceptable_migration_issues,
778 feature_migration,
779 manifest,
780 support_executable_files=True,
781 allow_dh_exec_rename=True,
782 )
783 if not install_file or not content:
784 continue
785 current_sources = []
786 sources_by_destdir: Dict[Tuple[str, Tuple[str, ...]], SourcesAndConditional] = (
787 {}
788 )
789 install_as_rules = []
790 multi_dest = collections.defaultdict(list)
791 seen_sources = set()
792 multi_dest_sources: Set[str] = set()
794 for dhe_line in content:
795 special_rule = None
796 if "=>" in dhe_line.tokens:
797 if dhe_line.tokens[0] == "=>" and len(dhe_line.tokens) == 2:
798 # This rule must be as early as possible to retain the semantics
799 path = _strip_d_tmp(
800 _normalize_path(dhe_line.tokens[1], with_prefix=False)
801 )
802 special_rule = AbstractMutableYAMLInstallRule.install_dest(
803 path,
804 dctrl_bin.name if not is_single_binary else None,
805 dest_dir=None,
806 when=dhe_line.conditional(),
807 )
808 elif len(dhe_line.tokens) != 3: 808 ↛ 809line 808 didn't jump to line 809 because the condition on line 808 was never true
809 _error(
810 f"Validation error in {install_file.path} on line {dhe_line.line_no}. Cannot migrate dh-exec"
811 ' renames that is not exactly "SOURCE => TARGET" or "=> TARGET".'
812 )
813 else:
814 install_rule = AbstractMutableYAMLInstallRule.install_as(
815 _strip_d_tmp(
816 _normalize_path(dhe_line.tokens[0], with_prefix=False)
817 ),
818 _normalize_path(dhe_line.tokens[2], with_prefix=False),
819 dctrl_bin.name if not is_single_binary else None,
820 when=dhe_line.conditional(),
821 )
822 install_as_rules.append(install_rule)
823 else:
824 if len(dhe_line.tokens) > 1:
825 sources = list(
826 _strip_d_tmp(_normalize_path(w, with_prefix=False))
827 for w in dhe_line.tokens[:-1]
828 )
829 dest_dir = _normalize_path(dhe_line.tokens[-1], with_prefix=False)
830 else:
831 sources = list(
832 _strip_d_tmp(_normalize_path(w, with_prefix=False))
833 for w in dhe_line.tokens
834 )
835 dest_dir = None
837 multi_dest_sources.update(s for s in sources if s in seen_sources)
838 seen_sources.update(sources)
840 if dest_dir is None and dhe_line.conditional() is None:
841 current_sources.extend(sources)
842 continue
843 key = (dest_dir, dhe_line.conditional_key())
844 ctor = functools.partial(
845 SourcesAndConditional,
846 dest_dir=dest_dir,
847 conditional=dhe_line.conditional(),
848 )
849 md = _fetch_or_create(
850 sources_by_destdir,
851 key,
852 ctor,
853 )
854 md.sources.extend(sources)
856 if special_rule:
857 priority_lines.append(special_rule)
859 remaining_install_lines.extend(install_as_rules)
861 for md in sources_by_destdir.values():
862 if multi_dest_sources:
863 sources = [s for s in md.sources if s not in multi_dest_sources]
864 already_installed = (s for s in md.sources if s in multi_dest_sources)
865 for s in already_installed:
866 # The sources are ignored, so we can reuse the object as-is
867 multi_dest[s].append(md)
868 if not sources:
869 continue
870 else:
871 sources = md.sources
872 install_rule = AbstractMutableYAMLInstallRule.install_dest(
873 sources[0] if len(sources) == 1 else sources,
874 dctrl_bin.name if not is_single_binary else None,
875 dest_dir=md.dest_dir,
876 when=md.conditional,
877 )
878 remaining_install_lines.append(install_rule)
880 if current_sources:
881 if multi_dest_sources:
882 sources = [s for s in current_sources if s not in multi_dest_sources]
883 already_installed = (
884 s for s in current_sources if s in multi_dest_sources
885 )
886 for s in already_installed:
887 # The sources are ignored, so we can reuse the object as-is
888 dest_dir = os.path.dirname(s)
889 if has_glob_magic(dest_dir):
890 warn_about_fixmes_in_dest_dir = True
891 dest_dir = f"FIXME: {dest_dir} (could not reliably compute the dest dir)"
892 multi_dest[s].append(
893 SourcesAndConditional(
894 dest_dir=dest_dir,
895 conditional=None,
896 )
897 )
898 else:
899 sources = current_sources
901 if sources:
902 install_rule = AbstractMutableYAMLInstallRule.install_dest(
903 sources[0] if len(sources) == 1 else sources,
904 dctrl_bin.name if not is_single_binary else None,
905 dest_dir=None,
906 )
907 remaining_install_lines.append(install_rule)
909 if multi_dest:
910 for source, dest_and_conditionals in multi_dest.items():
911 dest_dirs = [dac.dest_dir for dac in dest_and_conditionals]
912 # We assume the conditional is the same.
913 conditional = next(
914 iter(
915 dac.conditional
916 for dac in dest_and_conditionals
917 if dac.conditional is not None
918 ),
919 None,
920 )
921 remaining_install_lines.append(
922 AbstractMutableYAMLInstallRule.multi_dest_install(
923 source,
924 dest_dirs,
925 dctrl_bin.name if not is_single_binary else None,
926 when=conditional,
927 )
928 )
930 if priority_lines:
931 installations.extend(priority_lines)
933 if remaining_install_lines:
934 installations.extend(remaining_install_lines)
936 feature_migration.successful_manifest_changes += len(priority_lines) + len(
937 remaining_install_lines
938 )
939 if warn_about_fixmes_in_dest_dir:
940 feature_migration.warn(
941 "TODO: FIXME left in dest-dir(s) of some installation rules."
942 " Please review these and remove the FIXME (plus correct as necessary)"
943 )
946def migrate_installdocs_file(
947 debian_dir: VirtualPath,
948 manifest: HighLevelManifest,
949 acceptable_migration_issues: AcceptableMigrationIssues,
950 feature_migration: FeatureMigration,
951 _migration_target: Optional[DebputyIntegrationMode],
952) -> None:
953 feature_migration.tagline = "dh_installdocs config files"
954 mutable_manifest = assume_not_none(manifest.mutable_manifest)
955 installations = mutable_manifest.installations(create_if_absent=False)
957 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
959 for dctrl_bin in manifest.all_packages:
960 install_file, content = _dh_config_file(
961 debian_dir,
962 dctrl_bin,
963 "docs",
964 "dh_installdocs",
965 acceptable_migration_issues,
966 feature_migration,
967 manifest,
968 support_executable_files=True,
969 )
970 if not install_file:
971 continue
972 assert content is not None
973 docs: List[str] = []
974 for dhe_line in content:
975 if dhe_line.arch_filter or dhe_line.build_profile_filter: 975 ↛ 976line 975 didn't jump to line 976 because the condition on line 975 was never true
976 _error(
977 f"Unable to migrate line {dhe_line.line_no} of {install_file.path}."
978 " Missing support for conditions."
979 )
980 docs.extend(_normalize_path(w, with_prefix=False) for w in dhe_line.tokens)
982 if not docs: 982 ↛ 983line 982 didn't jump to line 983 because the condition on line 982 was never true
983 continue
984 feature_migration.successful_manifest_changes += 1
985 install_rule = AbstractMutableYAMLInstallRule.install_docs(
986 docs if len(docs) > 1 else docs[0],
987 dctrl_bin.name if not is_single_binary else None,
988 )
989 installations.create_definition_if_missing()
990 installations.append(install_rule)
993def migrate_installexamples_file(
994 debian_dir: VirtualPath,
995 manifest: HighLevelManifest,
996 acceptable_migration_issues: AcceptableMigrationIssues,
997 feature_migration: FeatureMigration,
998 _migration_target: Optional[DebputyIntegrationMode],
999) -> None:
1000 feature_migration.tagline = "dh_installexamples config files"
1001 mutable_manifest = assume_not_none(manifest.mutable_manifest)
1002 installations = mutable_manifest.installations(create_if_absent=False)
1003 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
1005 for dctrl_bin in manifest.all_packages:
1006 install_file, content = _dh_config_file(
1007 debian_dir,
1008 dctrl_bin,
1009 "examples",
1010 "dh_installexamples",
1011 acceptable_migration_issues,
1012 feature_migration,
1013 manifest,
1014 support_executable_files=True,
1015 )
1016 if not install_file:
1017 continue
1018 assert content is not None
1019 examples: List[str] = []
1020 for dhe_line in content:
1021 if dhe_line.arch_filter or dhe_line.build_profile_filter: 1021 ↛ 1022line 1021 didn't jump to line 1022 because the condition on line 1021 was never true
1022 _error(
1023 f"Unable to migrate line {dhe_line.line_no} of {install_file.path}."
1024 " Missing support for conditions."
1025 )
1026 examples.extend(
1027 _normalize_path(w, with_prefix=False) for w in dhe_line.tokens
1028 )
1030 if not examples: 1030 ↛ 1031line 1030 didn't jump to line 1031 because the condition on line 1030 was never true
1031 continue
1032 feature_migration.successful_manifest_changes += 1
1033 install_rule = AbstractMutableYAMLInstallRule.install_examples(
1034 examples if len(examples) > 1 else examples[0],
1035 dctrl_bin.name if not is_single_binary else None,
1036 )
1037 installations.create_definition_if_missing()
1038 installations.append(install_rule)
1041@dataclasses.dataclass(slots=True)
1042class InfoFilesDefinition:
1043 sources: List[str] = dataclasses.field(default_factory=list)
1044 conditional: Optional[Union[str, Mapping[str, Any]]] = None
1047def migrate_installinfo_file(
1048 debian_dir: VirtualPath,
1049 manifest: HighLevelManifest,
1050 acceptable_migration_issues: AcceptableMigrationIssues,
1051 feature_migration: FeatureMigration,
1052 _migration_target: Optional[DebputyIntegrationMode],
1053) -> None:
1054 feature_migration.tagline = "dh_installinfo config files"
1055 mutable_manifest = assume_not_none(manifest.mutable_manifest)
1056 installations = mutable_manifest.installations(create_if_absent=False)
1057 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
1059 for dctrl_bin in manifest.all_packages:
1060 info_file, content = _dh_config_file(
1061 debian_dir,
1062 dctrl_bin,
1063 "info",
1064 "dh_installinfo",
1065 acceptable_migration_issues,
1066 feature_migration,
1067 manifest,
1068 support_executable_files=True,
1069 )
1070 if not info_file:
1071 continue
1072 assert content is not None
1073 info_files_by_condition: Dict[Tuple[str, ...], InfoFilesDefinition] = {}
1074 for dhe_line in content:
1075 key = dhe_line.conditional_key()
1076 ctr = functools.partial(
1077 InfoFilesDefinition, conditional=dhe_line.conditional()
1078 )
1079 info_def = _fetch_or_create(
1080 info_files_by_condition,
1081 key,
1082 ctr,
1083 )
1084 info_def.sources.extend(
1085 _normalize_path(w, with_prefix=False) for w in dhe_line.tokens
1086 )
1088 if not info_files_by_condition: 1088 ↛ 1089line 1088 didn't jump to line 1089 because the condition on line 1088 was never true
1089 continue
1090 feature_migration.successful_manifest_changes += 1
1091 installations.create_definition_if_missing()
1092 for info_def in info_files_by_condition.values():
1093 info_files = info_def.sources
1094 install_rule = AbstractMutableYAMLInstallRule.install_docs(
1095 info_files if len(info_files) > 1 else info_files[0],
1096 dctrl_bin.name if not is_single_binary else None,
1097 dest_dir="{{path:GNU_INFO_DIR}}",
1098 when=info_def.conditional,
1099 )
1100 installations.append(install_rule)
1103@dataclasses.dataclass(slots=True)
1104class ManpageDefinition:
1105 sources: List[str] = dataclasses.field(default_factory=list)
1106 language: Optional[str] = None
1107 conditional: Optional[Union[str, Mapping[str, Any]]] = None
1110DK = TypeVar("DK")
1111DV = TypeVar("DV")
1114def _fetch_or_create(d: Dict[DK, DV], key: DK, factory: Callable[[], DV]) -> DV:
1115 v = d.get(key)
1116 if v is None:
1117 v = factory()
1118 d[key] = v
1119 return v
1122def migrate_installman_file(
1123 debian_dir: VirtualPath,
1124 manifest: HighLevelManifest,
1125 acceptable_migration_issues: AcceptableMigrationIssues,
1126 feature_migration: FeatureMigration,
1127 _migration_target: Optional[DebputyIntegrationMode],
1128) -> None:
1129 feature_migration.tagline = "dh_installman config files"
1130 mutable_manifest = assume_not_none(manifest.mutable_manifest)
1131 installations = mutable_manifest.installations(create_if_absent=False)
1132 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
1133 warn_about_basename = False
1135 for dctrl_bin in manifest.all_packages:
1136 manpages_file, content = _dh_config_file(
1137 debian_dir,
1138 dctrl_bin,
1139 "manpages",
1140 "dh_installman",
1141 acceptable_migration_issues,
1142 feature_migration,
1143 manifest,
1144 support_executable_files=True,
1145 allow_dh_exec_rename=True,
1146 )
1147 if not manpages_file:
1148 continue
1149 assert content is not None
1151 vanilla_definitions = []
1152 install_as_rules = []
1153 complex_definitions: Dict[
1154 Tuple[Optional[str], Tuple[str, ...]], ManpageDefinition
1155 ] = {}
1156 install_rule: AbstractMutableYAMLInstallRule
1157 for dhe_line in content:
1158 if "=>" in dhe_line.tokens: 1158 ↛ 1161line 1158 didn't jump to line 1161 because the condition on line 1158 was never true
1159 # dh-exec allows renaming features. For `debputy`, we degenerate it into an `install` (w. `as`) feature
1160 # without any of the `install-man` features.
1161 if dhe_line.tokens[0] == "=>" and len(dhe_line.tokens) == 2:
1162 _error(
1163 f'Unsupported "=> DEST" rule for error in {manpages_file.path} on line {dhe_line.line_no}."'
1164 f' Cannot migrate dh-exec renames that is not exactly "SOURCE => TARGET" for d/manpages files.'
1165 )
1166 elif len(dhe_line.tokens) != 3:
1167 _error(
1168 f"Validation error in {manpages_file.path} on line {dhe_line.line_no}. Cannot migrate dh-exec"
1169 ' renames that is not exactly "SOURCE => TARGET" or "=> TARGET".'
1170 )
1171 else:
1172 install_rule = AbstractMutableYAMLInstallRule.install_doc_as(
1173 _normalize_path(dhe_line.tokens[0], with_prefix=False),
1174 _normalize_path(dhe_line.tokens[2], with_prefix=False),
1175 dctrl_bin.name if not is_single_binary else None,
1176 when=dhe_line.conditional(),
1177 )
1178 install_as_rules.append(install_rule)
1179 continue
1181 sources = [_normalize_path(w, with_prefix=False) for w in dhe_line.tokens]
1182 needs_basename = any(
1183 MAN_GUESS_FROM_BASENAME.search(x)
1184 and not MAN_GUESS_LANG_FROM_PATH.search(x)
1185 for x in sources
1186 )
1187 if needs_basename or dhe_line.conditional() is not None:
1188 if needs_basename: 1188 ↛ 1192line 1188 didn't jump to line 1192 because the condition on line 1188 was always true
1189 warn_about_basename = True
1190 language = "derive-from-basename"
1191 else:
1192 language = None
1193 key = (language, dhe_line.conditional_key())
1194 ctor = functools.partial(
1195 ManpageDefinition,
1196 language=language,
1197 conditional=dhe_line.conditional(),
1198 )
1199 manpage_def = _fetch_or_create(
1200 complex_definitions,
1201 key,
1202 ctor,
1203 )
1204 manpage_def.sources.extend(sources)
1205 else:
1206 vanilla_definitions.extend(sources)
1208 if not install_as_rules and not vanilla_definitions and not complex_definitions: 1208 ↛ 1209line 1208 didn't jump to line 1209 because the condition on line 1208 was never true
1209 continue
1210 feature_migration.successful_manifest_changes += 1
1211 installations.create_definition_if_missing()
1212 installations.extend(install_as_rules)
1213 if vanilla_definitions: 1213 ↛ 1225line 1213 didn't jump to line 1225 because the condition on line 1213 was always true
1214 man_source = (
1215 vanilla_definitions
1216 if len(vanilla_definitions) > 1
1217 else vanilla_definitions[0]
1218 )
1219 install_rule = AbstractMutableYAMLInstallRule.install_man(
1220 man_source,
1221 dctrl_bin.name if not is_single_binary else None,
1222 None,
1223 )
1224 installations.append(install_rule)
1225 for manpage_def in complex_definitions.values():
1226 sources = manpage_def.sources
1227 install_rule = AbstractMutableYAMLInstallRule.install_man(
1228 sources if len(sources) > 1 else sources[0],
1229 dctrl_bin.name if not is_single_binary else None,
1230 manpage_def.language,
1231 when=manpage_def.conditional,
1232 )
1233 installations.append(install_rule)
1235 if warn_about_basename:
1236 feature_migration.warn(
1237 'Detected man pages that might rely on "derive-from-basename" logic. Please double check'
1238 " that the generated `install-man` rules are correct"
1239 )
1242def migrate_not_installed_file(
1243 debian_dir: VirtualPath,
1244 manifest: HighLevelManifest,
1245 acceptable_migration_issues: AcceptableMigrationIssues,
1246 feature_migration: FeatureMigration,
1247 _migration_target: Optional[DebputyIntegrationMode],
1248) -> None:
1249 feature_migration.tagline = "dh_missing's not-installed config file"
1250 mutable_manifest = assume_not_none(manifest.mutable_manifest)
1251 installations = mutable_manifest.installations(create_if_absent=False)
1252 main_binary = [p for p in manifest.all_packages if p.is_main_package][0]
1254 missing_file, content = _dh_config_file(
1255 debian_dir,
1256 main_binary,
1257 "not-installed",
1258 "dh_missing",
1259 acceptable_migration_issues,
1260 feature_migration,
1261 manifest,
1262 support_executable_files=False,
1263 pkgfile_lookup=False,
1264 )
1265 discard_rules: List[str] = []
1266 if missing_file:
1267 assert content is not None
1268 for dhe_line in content:
1269 discard_rules.extend(
1270 _normalize_path(w, with_prefix=False) for w in dhe_line.tokens
1271 )
1273 if discard_rules:
1274 feature_migration.successful_manifest_changes += 1
1275 install_rule = AbstractMutableYAMLInstallRule.discard(
1276 discard_rules if len(discard_rules) > 1 else discard_rules[0],
1277 )
1278 installations.create_definition_if_missing()
1279 installations.append(install_rule)
1282def detect_pam_files(
1283 debian_dir: VirtualPath,
1284 manifest: HighLevelManifest,
1285 _acceptable_migration_issues: AcceptableMigrationIssues,
1286 feature_migration: FeatureMigration,
1287 _migration_target: Optional[DebputyIntegrationMode],
1288) -> None:
1289 feature_migration.tagline = "detect dh_installpam files (min dh compat)"
1290 for dctrl_bin in manifest.all_packages:
1291 dh_config_file = dhe_pkgfile(debian_dir, dctrl_bin, "pam")
1292 if dh_config_file is not None:
1293 feature_migration.assumed_compat = 14
1294 break
1297def migrate_tmpfile(
1298 debian_dir: VirtualPath,
1299 manifest: HighLevelManifest,
1300 _acceptable_migration_issues: AcceptableMigrationIssues,
1301 feature_migration: FeatureMigration,
1302 _migration_target: Optional[DebputyIntegrationMode],
1303) -> None:
1304 feature_migration.tagline = "dh_installtmpfiles config files"
1305 for dctrl_bin in manifest.all_packages:
1306 dh_config_file = dhe_pkgfile(debian_dir, dctrl_bin, "tmpfile")
1307 if dh_config_file is not None:
1308 target = (
1309 dh_config_file.name.replace(".tmpfile", ".tmpfiles")
1310 if "." in dh_config_file.name
1311 else "tmpfiles"
1312 )
1313 _rename_file_if_exists(
1314 debian_dir,
1315 dh_config_file.name,
1316 target,
1317 feature_migration,
1318 )
1321def migrate_lintian_overrides_files(
1322 debian_dir: VirtualPath,
1323 manifest: HighLevelManifest,
1324 acceptable_migration_issues: AcceptableMigrationIssues,
1325 feature_migration: FeatureMigration,
1326 _migration_target: Optional[DebputyIntegrationMode],
1327) -> None:
1328 feature_migration.tagline = "dh_lintian config files"
1329 for dctrl_bin in manifest.all_packages:
1330 # We do not support executable lintian-overrides and `_dh_config_file` handles all of that.
1331 # Therefore, the return value is irrelevant to us.
1332 _dh_config_file(
1333 debian_dir,
1334 dctrl_bin,
1335 "lintian-overrides",
1336 "dh_lintian",
1337 acceptable_migration_issues,
1338 feature_migration,
1339 manifest,
1340 support_executable_files=False,
1341 remove_on_migration=False,
1342 )
1345def migrate_links_files(
1346 debian_dir: VirtualPath,
1347 manifest: HighLevelManifest,
1348 acceptable_migration_issues: AcceptableMigrationIssues,
1349 feature_migration: FeatureMigration,
1350 _migration_target: Optional[DebputyIntegrationMode],
1351) -> None:
1352 feature_migration.tagline = "dh_link files"
1353 mutable_manifest = assume_not_none(manifest.mutable_manifest)
1354 for dctrl_bin in manifest.all_packages:
1355 links_file, content = _dh_config_file(
1356 debian_dir,
1357 dctrl_bin,
1358 "links",
1359 "dh_link",
1360 acceptable_migration_issues,
1361 feature_migration,
1362 manifest,
1363 support_executable_files=True,
1364 )
1366 if links_file is None:
1367 continue
1368 assert content is not None
1370 package_definition = mutable_manifest.package(dctrl_bin.name)
1371 defined_symlink = {
1372 symlink.symlink_path: symlink.symlink_target
1373 for symlink in package_definition.symlinks()
1374 }
1376 seen_symlinks: Set[str] = set()
1378 for dhe_line in content:
1379 if len(dhe_line.tokens) != 2: 1379 ↛ 1380line 1379 didn't jump to line 1380 because the condition on line 1379 was never true
1380 raise UnsupportedFeature(
1381 f"The dh_link file {links_file.fs_path} did not have exactly two paths on line"
1382 f' {dhe_line.line_no} (line: "{dhe_line.original_line}"'
1383 )
1384 target, source = dhe_line.tokens
1385 if source in seen_symlinks: 1385 ↛ 1387line 1385 didn't jump to line 1387 because the condition on line 1385 was never true
1386 # According to #934499, this has happened in the wild already
1387 raise ConflictingChange(
1388 f"The {links_file.fs_path} file defines the link path {source} twice! Please ensure"
1389 " that it is defined at most once in that file"
1390 )
1391 seen_symlinks.add(source)
1392 # Symlinks in .links are always considered absolute, but you were not required to have a leading slash.
1393 # However, in the debputy manifest, you can have relative links, so we should ensure it is explicitly
1394 # absolute.
1395 if not target.startswith("/"): 1395 ↛ 1397line 1395 didn't jump to line 1397 because the condition on line 1395 was always true
1396 target = "/" + target
1397 existing_target = defined_symlink.get(source)
1398 if existing_target is not None: 1398 ↛ 1399line 1398 didn't jump to line 1399 because the condition on line 1398 was never true
1399 if existing_target != target:
1400 raise ConflictingChange(
1401 f'The symlink "{source}" points to "{target}" in {links_file}, but there is'
1402 f' another symlink with same path pointing to "{existing_target}" defined'
1403 " already (in the existing manifest or an migration e.g., inside"
1404 f" {links_file.fs_path})"
1405 )
1406 continue
1407 condition = dhe_line.conditional()
1408 package_definition.add_symlink(
1409 MutableYAMLSymlink.new_symlink(
1410 source,
1411 target,
1412 condition,
1413 )
1414 )
1415 feature_migration.successful_manifest_changes += 1
1418def migrate_misspelled_readme_debian_files(
1419 debian_dir: VirtualPath,
1420 manifest: HighLevelManifest,
1421 acceptable_migration_issues: AcceptableMigrationIssues,
1422 feature_migration: FeatureMigration,
1423 _migration_target: Optional[DebputyIntegrationMode],
1424) -> None:
1425 feature_migration.tagline = "misspelled README.Debian files"
1426 for dctrl_bin in manifest.all_packages:
1427 readme, _ = _dh_config_file(
1428 debian_dir,
1429 dctrl_bin,
1430 "README.debian",
1431 "dh_installdocs",
1432 acceptable_migration_issues,
1433 feature_migration,
1434 manifest,
1435 support_executable_files=False,
1436 remove_on_migration=False,
1437 )
1438 if readme is None:
1439 continue
1440 new_name = readme.name.replace("README.debian", "README.Debian")
1441 assert readme.name != new_name
1442 _rename_file_if_exists(
1443 debian_dir,
1444 readme.name,
1445 new_name,
1446 feature_migration,
1447 )
1450def migrate_doc_base_files(
1451 debian_dir: VirtualPath,
1452 manifest: HighLevelManifest,
1453 _: AcceptableMigrationIssues,
1454 feature_migration: FeatureMigration,
1455 _migration_target: Optional[DebputyIntegrationMode],
1456) -> None:
1457 feature_migration.tagline = "doc-base files"
1458 # ignore the dh_make ".EX" file if one should still be present. The dh_installdocs tool ignores it too.
1459 possible_effected_doc_base_files = [
1460 f
1461 for f in debian_dir.iterdir
1462 if (
1463 (".doc-base." in f.name or f.name.startswith("doc-base."))
1464 and not f.name.endswith("doc-base.EX")
1465 )
1466 ]
1467 known_packages = {d.name: d for d in manifest.all_packages}
1468 main_package = [d for d in manifest.all_packages if d.is_main_package][0]
1469 for doc_base_file in possible_effected_doc_base_files:
1470 parts = doc_base_file.name.split(".")
1471 owning_package = known_packages.get(parts[0])
1472 if owning_package is None: 1472 ↛ 1473line 1472 didn't jump to line 1473 because the condition on line 1472 was never true
1473 owning_package = main_package
1474 package_part = None
1475 else:
1476 package_part = parts[0]
1477 parts = parts[1:]
1479 if not parts or parts[0] != "doc-base": 1479 ↛ 1481line 1479 didn't jump to line 1481 because the condition on line 1479 was never true
1480 # Not a doc-base file after all
1481 continue
1483 if len(parts) > 1: 1483 ↛ 1490line 1483 didn't jump to line 1490 because the condition on line 1483 was always true
1484 name_part = ".".join(parts[1:])
1485 if package_part is None: 1485 ↛ 1487line 1485 didn't jump to line 1487 because the condition on line 1485 was never true
1486 # Named files must have a package prefix
1487 package_part = owning_package.name
1488 else:
1489 # No rename needed
1490 continue
1492 new_basename = ".".join(filter(None, (package_part, name_part, "doc-base")))
1493 _rename_file_if_exists(
1494 debian_dir,
1495 doc_base_file.name,
1496 new_basename,
1497 feature_migration,
1498 )
1501def migrate_dh_hook_targets(
1502 debian_dir: VirtualPath,
1503 _: HighLevelManifest,
1504 acceptable_migration_issues: AcceptableMigrationIssues,
1505 feature_migration: FeatureMigration,
1506 migration_target: Optional[DebputyIntegrationMode],
1507) -> None:
1508 feature_migration.tagline = "dh hook targets"
1509 source_root = os.path.dirname(debian_dir.fs_path)
1510 if source_root == "":
1511 source_root = "."
1512 detected_hook_targets = json.loads(
1513 subprocess.check_output(
1514 ["dh_assistant", "detect-hook-targets"],
1515 cwd=source_root,
1516 ).decode("utf-8")
1517 )
1518 sample_hook_target: Optional[str] = None
1519 assert migration_target is not None
1520 replaced_commands = DH_COMMANDS_REPLACED[migration_target]
1522 for hook_target_def in detected_hook_targets["hook-targets"]:
1523 if hook_target_def["is-empty"]:
1524 continue
1525 command = hook_target_def["command"]
1526 if command not in replaced_commands:
1527 continue
1528 hook_target = hook_target_def["target-name"]
1529 advice = MIGRATION_AID_FOR_OVERRIDDEN_COMMANDS.get(command)
1530 if advice is None:
1531 if sample_hook_target is None:
1532 sample_hook_target = hook_target
1533 feature_migration.warn(
1534 f"TODO: MANUAL MIGRATION required for hook target {hook_target}"
1535 )
1536 else:
1537 feature_migration.warn(
1538 f"TODO: MANUAL MIGRATION required for hook target {hook_target}. Please see {advice}"
1539 f" for migration advice."
1540 )
1541 if (
1542 feature_migration.warnings
1543 and "dh-hook-targets" not in acceptable_migration_issues
1544 and sample_hook_target is not None
1545 ):
1546 raise UnsupportedFeature(
1547 f"The debian/rules file contains one or more non empty dh hook targets that will not"
1548 f" be run with the requested debputy dh sequence with no known migration advice. One of these would be"
1549 f" {sample_hook_target}.",
1550 ["dh-hook-targets"],
1551 )
1554def detect_unsupported_zz_debputy_features(
1555 debian_dir: VirtualPath,
1556 manifest: HighLevelManifest,
1557 acceptable_migration_issues: AcceptableMigrationIssues,
1558 feature_migration: FeatureMigration,
1559 _migration_target: Optional[DebputyIntegrationMode],
1560) -> None:
1561 feature_migration.tagline = "Known unsupported features"
1563 for unsupported_config in UNSUPPORTED_DH_CONFIGS_AND_TOOLS_FOR_ZZ_DEBPUTY:
1564 _unsupported_debhelper_config_file(
1565 debian_dir,
1566 manifest,
1567 unsupported_config,
1568 acceptable_migration_issues,
1569 feature_migration,
1570 )
1573def detect_obsolete_substvars(
1574 debian_dir: VirtualPath,
1575 _manifest: HighLevelManifest,
1576 _acceptable_migration_issues: AcceptableMigrationIssues,
1577 feature_migration: FeatureMigration,
1578 _migration_target: Optional[DebputyIntegrationMode],
1579) -> None:
1580 feature_migration.tagline = (
1581 "Check for obsolete ${foo:var} variables in debian/control"
1582 )
1583 ctrl_file = debian_dir.get("control")
1584 if not ctrl_file: 1584 ↛ 1585line 1584 didn't jump to line 1585 because the condition on line 1584 was never true
1585 feature_migration.warn(
1586 "Cannot find debian/control. Detection of obsolete substvars could not be performed."
1587 )
1588 return
1589 with ctrl_file.open() as fd:
1590 ctrl = list(Deb822.iter_paragraphs(fd))
1592 relationship_fields = dpkg_field_list_pkg_dep()
1593 relationship_fields_lc = frozenset(x.lower() for x in relationship_fields)
1595 for p in ctrl[1:]:
1596 seen_obsolete_relationship_substvars = set()
1597 obsolete_fields = set()
1598 is_essential = p.get("Essential") == "yes"
1599 for df in relationship_fields:
1600 field: Optional[str] = p.get(df)
1601 if field is None:
1602 continue
1603 df_lc = df.lower()
1604 number_of_relations = 0
1605 obsolete_substvars_in_field = set()
1606 for d in (d.strip() for d in field.strip().split(",")):
1607 if not d:
1608 continue
1609 number_of_relations += 1
1610 if not d.startswith("${"):
1611 continue
1612 try:
1613 end_idx = d.index("}")
1614 except ValueError:
1615 continue
1616 substvar_name = d[2:end_idx]
1617 if ":" not in substvar_name: 1617 ↛ 1618line 1617 didn't jump to line 1618 because the condition on line 1617 was never true
1618 continue
1619 _, field = substvar_name.rsplit(":", 1)
1620 field_lc = field.lower()
1621 if field_lc not in relationship_fields_lc: 1621 ↛ 1622line 1621 didn't jump to line 1622 because the condition on line 1621 was never true
1622 continue
1623 is_obsolete = field_lc == df_lc
1624 if (
1625 not is_obsolete
1626 and is_essential
1627 and substvar_name.lower() == "shlibs:depends"
1628 and df_lc == "pre-depends"
1629 ):
1630 is_obsolete = True
1632 if is_obsolete:
1633 obsolete_substvars_in_field.add(d)
1635 if number_of_relations == len(obsolete_substvars_in_field):
1636 obsolete_fields.add(df)
1637 else:
1638 seen_obsolete_relationship_substvars.update(obsolete_substvars_in_field)
1640 package = p.get("Package", "(Missing package name!?)")
1641 fo = feature_migration.fo
1642 if obsolete_fields:
1643 fields = ", ".join(obsolete_fields)
1644 feature_migration.warn(
1645 f"The following relationship fields can be removed from {package}: {fields}."
1646 f" (The content in them would be applied automatically. Note: {fo.bts('1067653')})"
1647 )
1648 if seen_obsolete_relationship_substvars:
1649 v = ", ".join(sorted(seen_obsolete_relationship_substvars))
1650 feature_migration.warn(
1651 f"The following relationship substitution variables can be removed from {package}: {v}"
1652 f" (Note: {fo.bts('1067653')})"
1653 )
1656def detect_dh_addons_zz_debputy_rrr(
1657 debian_dir: VirtualPath,
1658 _manifest: HighLevelManifest,
1659 _acceptable_migration_issues: AcceptableMigrationIssues,
1660 feature_migration: FeatureMigration,
1661 _migration_target: Optional[DebputyIntegrationMode],
1662) -> None:
1663 feature_migration.tagline = "Check for dh-sequence-addons"
1664 r = read_dh_addon_sequences(debian_dir)
1665 if r is None:
1666 feature_migration.warn(
1667 "Cannot find debian/control. Detection of unsupported/missing dh-sequence addon"
1668 " could not be performed. Please ensure the package will Build-Depend on dh-sequence-zz-debputy-rrr."
1669 )
1670 return
1672 bd_sequences, dr_sequences, _ = r
1674 remaining_sequences = bd_sequences | dr_sequences
1675 saw_dh_debputy = "zz-debputy-rrr" in remaining_sequences
1677 if not saw_dh_debputy:
1678 feature_migration.warn("Missing Build-Depends on dh-sequence-zz-debputy-rrr")
1681def detect_dh_addons_with_full_integration(
1682 _debian_dir: VirtualPath,
1683 _manifest: HighLevelManifest,
1684 _acceptable_migration_issues: AcceptableMigrationIssues,
1685 feature_migration: FeatureMigration,
1686 _migration_target: Optional[DebputyIntegrationMode],
1687) -> None:
1688 feature_migration.tagline = "Check for dh-sequence-addons and Build-Depends"
1689 feature_migration.warn(
1690 "TODO: Not implemented: Please remove any dh-sequence Build-Dependency"
1691 )
1692 feature_migration.warn(
1693 "TODO: Not implemented: Please ensure there is a Build-Dependency on `debputy (>= 0.1.45~)"
1694 )
1695 feature_migration.warn(
1696 "TODO: Not implemented: Please ensure there is a Build-Dependency on `dpkg-dev (>= 1.22.7~)"
1697 )
1700def detect_dh_addons_with_zz_integration(
1701 debian_dir: VirtualPath,
1702 _manifest: HighLevelManifest,
1703 acceptable_migration_issues: AcceptableMigrationIssues,
1704 feature_migration: FeatureMigration,
1705 _migration_target: Optional[DebputyIntegrationMode],
1706) -> None:
1707 feature_migration.tagline = "Check for dh-sequence-addons"
1708 r = read_dh_addon_sequences(debian_dir)
1709 if r is None:
1710 feature_migration.warn(
1711 "Cannot find debian/control. Detection of unsupported/missing dh-sequence addon"
1712 " could not be performed. Please ensure the package will Build-Depend on dh-sequence-zz-debputy"
1713 " and not rely on any other debhelper sequence addons except those debputy explicitly supports."
1714 )
1715 return
1717 assert _migration_target != INTEGRATION_MODE_FULL
1719 bd_sequences, dr_sequences, _ = r
1721 remaining_sequences = bd_sequences | dr_sequences
1722 saw_dh_debputy = (
1723 "debputy" in remaining_sequences or "zz-debputy" in remaining_sequences
1724 )
1725 saw_zz_debputy = "zz-debputy" in remaining_sequences
1726 must_use_zz_debputy = False
1727 remaining_sequences -= SUPPORTED_DH_ADDONS_WITH_ZZ_DEBPUTY
1728 for sequence in remaining_sequences & DH_ADDONS_TO_PLUGINS.keys():
1729 migration = DH_ADDONS_TO_PLUGINS[sequence]
1730 feature_migration.require_plugin(migration.debputy_plugin)
1731 if migration.remove_dh_sequence: 1731 ↛ 1732line 1731 didn't jump to line 1732 because the condition on line 1731 was never true
1732 if migration.must_use_zz_debputy:
1733 must_use_zz_debputy = True
1734 if sequence in bd_sequences:
1735 feature_migration.warn(
1736 f"TODO: MANUAL MIGRATION - Remove build-dependency on dh-sequence-{sequence}"
1737 f" (replaced by debputy-plugin-{migration.debputy_plugin})"
1738 )
1739 else:
1740 feature_migration.warn(
1741 f"TODO: MANUAL MIGRATION - Remove --with {sequence} from dh in d/rules"
1742 f" (replaced by debputy-plugin-{migration.debputy_plugin})"
1743 )
1745 remaining_sequences -= DH_ADDONS_TO_PLUGINS.keys()
1747 alt_key = "unsupported-dh-sequences"
1748 for sequence in remaining_sequences & DH_ADDONS_TO_REMOVE_FOR_ZZ_DEBPUTY: 1748 ↛ 1749line 1748 didn't jump to line 1749 because the loop on line 1748 never started
1749 if sequence in bd_sequences:
1750 feature_migration.warn(
1751 f"TODO: MANUAL MIGRATION - Remove build dependency on dh-sequence-{sequence}"
1752 )
1753 else:
1754 feature_migration.warn(
1755 f"TODO: MANUAL MIGRATION - Remove --with {sequence} from dh in d/rules"
1756 )
1758 remaining_sequences -= DH_ADDONS_TO_REMOVE_FOR_ZZ_DEBPUTY
1760 for sequence in remaining_sequences:
1761 key = f"unsupported-dh-sequence-{sequence}"
1762 msg = f'The dh addon "{sequence}" is not known to work with dh-debputy and might malfunction'
1763 if (
1764 key not in acceptable_migration_issues
1765 and alt_key not in acceptable_migration_issues
1766 ):
1767 raise UnsupportedFeature(msg, [key, alt_key])
1768 feature_migration.warn(msg)
1770 if not saw_dh_debputy:
1771 feature_migration.warn("Missing Build-Depends on dh-sequence-zz-debputy")
1772 elif must_use_zz_debputy and not saw_zz_debputy: 1772 ↛ 1773line 1772 didn't jump to line 1773 because the condition on line 1772 was never true
1773 feature_migration.warn(
1774 "Please use the zz-debputy sequence rather than the debputy (needed due to dh add-on load order)"
1775 )
1778def _rename_file_if_exists(
1779 debian_dir: VirtualPath,
1780 source: str,
1781 dest: str,
1782 feature_migration: FeatureMigration,
1783) -> None:
1784 source_path = debian_dir.get(source)
1785 dest_path = debian_dir.get(dest)
1786 spath = (
1787 source_path.path
1788 if source_path is not None
1789 else os.path.join(debian_dir.path, source)
1790 )
1791 dpath = (
1792 dest_path.path if dest_path is not None else os.path.join(debian_dir.path, dest)
1793 )
1794 if source_path is not None and source_path.is_file:
1795 if dest_path is not None:
1796 if not dest_path.is_file:
1797 feature_migration.warnings.append(
1798 f'TODO: MANUAL MIGRATION - there is a "{spath}" (file) and "{dpath}" (not a file).'
1799 f' The migration wanted to replace "{spath}" with "{dpath}", but since "{dpath}" is not'
1800 " a file, this step is left as a manual migration."
1801 )
1802 return
1803 if (
1804 subprocess.call(["cmp", "-s", source_path.fs_path, dest_path.fs_path])
1805 != 0
1806 ):
1807 feature_migration.warnings.append(
1808 f'TODO: MANUAL MIGRATION - there is a "{source_path.path}" and "{dest_path.path}"'
1809 f" file. Normally these files are for the same package and there would only be one of"
1810 f" them. In this case, they both exist but their content differs. Be advised that"
1811 f' debputy tool will use the "{dest_path.path}".'
1812 )
1813 else:
1814 feature_migration.remove_on_success(dest_path.fs_path)
1815 else:
1816 feature_migration.rename_on_success(
1817 source_path.fs_path,
1818 os.path.join(debian_dir.fs_path, dest),
1819 )
1820 elif source_path is not None: 1820 ↛ exitline 1820 didn't return from function '_rename_file_if_exists' because the condition on line 1820 was always true
1821 feature_migration.warnings.append(
1822 f'TODO: MANUAL MIGRATION - The migration would normally have renamed "{spath}" to "{dpath}".'
1823 f' However, the migration assumed "{spath}" would be a file and it is not. Therefore, this step'
1824 " as a manual migration."
1825 )
1828def _find_dh_config_file_for_any_pkg(
1829 debian_dir: VirtualPath,
1830 manifest: HighLevelManifest,
1831 unsupported_config: UnsupportedDHConfig,
1832) -> Iterable[VirtualPath]:
1833 for dctrl_bin in manifest.all_packages:
1834 dh_config_file = dhe_pkgfile(
1835 debian_dir,
1836 dctrl_bin,
1837 unsupported_config.dh_config_basename,
1838 bug_950723_prefix_matching=unsupported_config.bug_950723_prefix_matching,
1839 )
1840 if dh_config_file is not None:
1841 yield dh_config_file
1844def _unsupported_debhelper_config_file(
1845 debian_dir: VirtualPath,
1846 manifest: HighLevelManifest,
1847 unsupported_config: UnsupportedDHConfig,
1848 acceptable_migration_issues: AcceptableMigrationIssues,
1849 feature_migration: FeatureMigration,
1850) -> None:
1851 dh_config_files = list(
1852 _find_dh_config_file_for_any_pkg(debian_dir, manifest, unsupported_config)
1853 )
1854 if not dh_config_files:
1855 return
1856 dh_tool = unsupported_config.dh_tool
1857 basename = unsupported_config.dh_config_basename
1858 file_stem = (
1859 f"@{basename}" if unsupported_config.bug_950723_prefix_matching else basename
1860 )
1861 dh_config_file = dh_config_files[0]
1862 if unsupported_config.is_missing_migration:
1863 feature_migration.warn(
1864 f'Missing migration support for the "{dh_config_file.path}" debhelper config file'
1865 f" (used by {dh_tool}). Manual migration may be feasible depending on the exact features"
1866 " required."
1867 )
1868 return
1869 primary_key = f"unsupported-dh-config-file-{file_stem}"
1870 secondary_key = "any-unsupported-dh-config-file"
1871 if (
1872 primary_key not in acceptable_migration_issues
1873 and secondary_key not in acceptable_migration_issues
1874 ):
1875 msg = (
1876 f'The "{dh_config_file.path}" debhelper config file (used by {dh_tool} is currently not'
1877 " supported by debputy."
1878 )
1879 raise UnsupportedFeature(
1880 msg,
1881 [primary_key, secondary_key],
1882 )
1883 for dh_config_file in dh_config_files:
1884 feature_migration.warn(
1885 f'TODO: MANUAL MIGRATION - Use of unsupported "{dh_config_file.path}" file (used by {dh_tool})'
1886 )