Coverage for src/debputy/dh_migration/migrators_impl.py: 82%
712 statements
« prev ^ index » next coverage.py v7.6.0, created at 2025-03-24 16:38 +0000
1import collections
2import dataclasses
3import functools
4import json
5import os
6import re
7import subprocess
8from itertools import product
9from typing import (
10 Iterable,
11 Optional,
12 Tuple,
13 List,
14 Set,
15 Mapping,
16 Any,
17 Union,
18 Callable,
19 TypeVar,
20 Dict,
21 Container,
22)
24from debian.deb822 import Deb822
26from debputy import DEBPUTY_DOC_ROOT_DIR
27from debputy.architecture_support import dpkg_architecture_table
28from debputy.deb_packaging_support import dpkg_field_list_pkg_dep
29from debputy.dh.debhelper_emulation import (
30 dhe_filedoublearray,
31 DHConfigFileLine,
32 dhe_pkgfile,
33)
34from debputy.dh.dh_assistant import (
35 read_dh_addon_sequences,
36)
37from debputy.dh_migration.models import (
38 ConflictingChange,
39 FeatureMigration,
40 UnsupportedFeature,
41 AcceptableMigrationIssues,
42 DHMigrationSubstitution,
43)
44from debputy.highlevel_manifest import (
45 MutableYAMLSymlink,
46 HighLevelManifest,
47 MutableYAMLConffileManagementItem,
48 AbstractMutableYAMLInstallRule,
49)
50from debputy.installations import MAN_GUESS_FROM_BASENAME, MAN_GUESS_LANG_FROM_PATH
51from debputy.packages import BinaryPackage
52from debputy.plugin.api import VirtualPath
53from debputy.plugin.api.spec import (
54 INTEGRATION_MODE_DH_DEBPUTY_RRR,
55 INTEGRATION_MODE_DH_DEBPUTY,
56 DebputyIntegrationMode,
57 INTEGRATION_MODE_FULL,
58)
59from debputy.util import (
60 _error,
61 PKGVERSION_REGEX,
62 PKGNAME_REGEX,
63 _normalize_path,
64 assume_not_none,
65 has_glob_magic,
66)
69class ContainsEverything:
71 def __contains__(self, item: str) -> bool:
72 return True
75# Align with debputy.py
76DH_COMMANDS_REPLACED: Mapping[DebputyIntegrationMode, Container[str]] = {
77 INTEGRATION_MODE_DH_DEBPUTY_RRR: frozenset(
78 {
79 "dh_fixperms",
80 "dh_shlibdeps",
81 "dh_gencontrol",
82 "dh_md5sums",
83 "dh_builddeb",
84 }
85 ),
86 INTEGRATION_MODE_DH_DEBPUTY: frozenset(
87 {
88 "dh_install",
89 "dh_installdocs",
90 "dh_installchangelogs",
91 "dh_installexamples",
92 "dh_installman",
93 "dh_installcatalogs",
94 "dh_installcron",
95 "dh_installdebconf",
96 "dh_installemacsen",
97 "dh_installifupdown",
98 "dh_installinfo",
99 "dh_installinit",
100 "dh_installsysusers",
101 "dh_installtmpfiles",
102 "dh_installsystemd",
103 "dh_installsystemduser",
104 "dh_installmenu",
105 "dh_installmime",
106 "dh_installmodules",
107 "dh_installlogcheck",
108 "dh_installlogrotate",
109 "dh_installpam",
110 "dh_installppp",
111 "dh_installudev",
112 "dh_installgsettings",
113 "dh_installinitramfs",
114 "dh_installalternatives",
115 "dh_bugfiles",
116 "dh_ucf",
117 "dh_lintian",
118 "dh_icons",
119 "dh_usrlocal",
120 "dh_perl",
121 "dh_link",
122 "dh_installwm",
123 "dh_installxfonts",
124 "dh_strip_nondeterminism",
125 "dh_compress",
126 "dh_fixperms",
127 "dh_dwz",
128 "dh_strip",
129 "dh_makeshlibs",
130 "dh_shlibdeps",
131 "dh_missing",
132 "dh_installdeb",
133 "dh_gencontrol",
134 "dh_md5sums",
135 "dh_builddeb",
136 }
137 ),
138 INTEGRATION_MODE_FULL: ContainsEverything(),
139}
141_GS_DOC = f"{DEBPUTY_DOC_ROOT_DIR}/GETTING-STARTED-WITH-dh-debputy.md"
142MIGRATION_AID_FOR_OVERRIDDEN_COMMANDS = {
143 "dh_installinit": f"{_GS_DOC}#covert-your-overrides-for-dh_installsystemd-dh_installinit-if-any",
144 "dh_installsystemd": f"{_GS_DOC}#covert-your-overrides-for-dh_installsystemd-dh_installinit-if-any",
145 "dh_fixperms": f"{_GS_DOC}#convert-your-overrides-or-excludes-for-dh_fixperms-if-any",
146 "dh_gencontrol": f"{_GS_DOC}#convert-your-overrides-for-dh_gencontrol-if-any",
147}
150@dataclasses.dataclass(frozen=True, slots=True)
151class UnsupportedDHConfig:
152 dh_config_basename: str
153 dh_tool: str
154 bug_950723_prefix_matching: bool = False
155 is_missing_migration: bool = False
158@dataclasses.dataclass(frozen=True, slots=True)
159class DHSequenceMigration:
160 debputy_plugin: str
161 remove_dh_sequence: bool = True
162 must_use_zz_debputy: bool = False
165UNSUPPORTED_DH_CONFIGS_AND_TOOLS_FOR_ZZ_DEBPUTY = [
166 UnsupportedDHConfig("config", "dh_installdebconf"),
167 UnsupportedDHConfig("templates", "dh_installdebconf"),
168 UnsupportedDHConfig("emacsen-compat", "dh_installemacsen"),
169 UnsupportedDHConfig("emacsen-install", "dh_installemacsen"),
170 UnsupportedDHConfig("emacsen-remove", "dh_installemacsen"),
171 UnsupportedDHConfig("emacsen-startup", "dh_installemacsen"),
172 # The `upstart` file should be long dead, but we might as well detect it.
173 UnsupportedDHConfig("upstart", "dh_installinit"),
174 # dh_installsystemduser
175 UnsupportedDHConfig(
176 "user.path", "dh_installsystemduser", bug_950723_prefix_matching=False
177 ),
178 UnsupportedDHConfig(
179 "user.path", "dh_installsystemduser", bug_950723_prefix_matching=True
180 ),
181 UnsupportedDHConfig(
182 "user.service", "dh_installsystemduser", bug_950723_prefix_matching=False
183 ),
184 UnsupportedDHConfig(
185 "user.service", "dh_installsystemduser", bug_950723_prefix_matching=True
186 ),
187 UnsupportedDHConfig(
188 "user.socket", "dh_installsystemduser", bug_950723_prefix_matching=False
189 ),
190 UnsupportedDHConfig(
191 "user.socket", "dh_installsystemduser", bug_950723_prefix_matching=True
192 ),
193 UnsupportedDHConfig(
194 "user.target", "dh_installsystemduser", bug_950723_prefix_matching=False
195 ),
196 UnsupportedDHConfig(
197 "user.target", "dh_installsystemduser", bug_950723_prefix_matching=True
198 ),
199 UnsupportedDHConfig(
200 "user.timer", "dh_installsystemduser", bug_950723_prefix_matching=False
201 ),
202 UnsupportedDHConfig(
203 "user.timer", "dh_installsystemduser", bug_950723_prefix_matching=True
204 ),
205 UnsupportedDHConfig("udev", "dh_installudev"),
206 UnsupportedDHConfig("menu", "dh_installmenu"),
207 UnsupportedDHConfig("menu-method", "dh_installmenu"),
208 UnsupportedDHConfig("ucf", "dh_ucf"),
209 UnsupportedDHConfig("wm", "dh_installwm"),
210 UnsupportedDHConfig("triggers", "dh_installdeb"),
211 UnsupportedDHConfig("postinst", "dh_installdeb"),
212 UnsupportedDHConfig("postrm", "dh_installdeb"),
213 UnsupportedDHConfig("preinst", "dh_installdeb"),
214 UnsupportedDHConfig("prerm", "dh_installdeb"),
215 UnsupportedDHConfig("menutest", "dh_installdeb"),
216 UnsupportedDHConfig("isinstallable", "dh_installdeb"),
217]
218SUPPORTED_DH_ADDONS = frozenset(
219 {
220 # debputy's own
221 "debputy",
222 "zz-debputy",
223 # debhelper provided sequences that should work.
224 "single-binary",
225 }
226)
227DH_ADDONS_TO_REMOVE = frozenset(
228 [
229 # Sequences debputy directly replaces
230 "dwz",
231 "elf-tools",
232 "installinitramfs",
233 "installsysusers",
234 "doxygen",
235 # Sequences that are embedded fully into debputy
236 "bash-completion",
237 "shell-completions",
238 "sodeps",
239 ]
240)
241DH_ADDONS_TO_PLUGINS = {
242 "gnome": DHSequenceMigration(
243 "gnome",
244 # The sequence still provides a command for the clean sequence
245 remove_dh_sequence=False,
246 must_use_zz_debputy=True,
247 ),
248 "grantlee": DHSequenceMigration(
249 "grantlee",
250 remove_dh_sequence=True,
251 must_use_zz_debputy=True,
252 ),
253 "numpy3": DHSequenceMigration(
254 "numpy3",
255 # The sequence provides (build-time) dependencies that we cannot provide
256 remove_dh_sequence=False,
257 must_use_zz_debputy=True,
258 ),
259 "perl-openssl": DHSequenceMigration(
260 "perl-openssl",
261 # The sequence provides (build-time) dependencies that we cannot provide
262 remove_dh_sequence=False,
263 must_use_zz_debputy=True,
264 ),
265}
268def _dh_config_file(
269 debian_dir: VirtualPath,
270 dctrl_bin: BinaryPackage,
271 basename: str,
272 helper_name: str,
273 acceptable_migration_issues: AcceptableMigrationIssues,
274 feature_migration: FeatureMigration,
275 manifest: HighLevelManifest,
276 support_executable_files: bool = False,
277 allow_dh_exec_rename: bool = False,
278 pkgfile_lookup: bool = True,
279 remove_on_migration: bool = True,
280) -> Union[Tuple[None, None], Tuple[VirtualPath, Iterable[DHConfigFileLine]]]:
281 mutable_manifest = assume_not_none(manifest.mutable_manifest)
282 dh_config_file = (
283 dhe_pkgfile(debian_dir, dctrl_bin, basename)
284 if pkgfile_lookup
285 else debian_dir.get(basename)
286 )
287 if dh_config_file is None or dh_config_file.is_dir:
288 return None, None
289 if dh_config_file.is_executable and not support_executable_files:
290 primary_key = f"executable-{helper_name}-config"
291 if (
292 primary_key in acceptable_migration_issues
293 or "any-executable-dh-configs" in acceptable_migration_issues
294 ):
295 feature_migration.warn(
296 f'TODO: MANUAL MIGRATION of executable dh config "{dh_config_file}" is required.'
297 )
298 return None, None
299 raise UnsupportedFeature(
300 f"Executable configuration files not supported (found: {dh_config_file}).",
301 [primary_key, "any-executable-dh-configs"],
302 )
304 if remove_on_migration:
305 feature_migration.remove_on_success(dh_config_file.fs_path)
306 substitution = DHMigrationSubstitution(
307 dpkg_architecture_table(),
308 acceptable_migration_issues,
309 feature_migration,
310 mutable_manifest,
311 )
312 content = dhe_filedoublearray(
313 dh_config_file,
314 substitution,
315 allow_dh_exec_rename=allow_dh_exec_rename,
316 )
317 return dh_config_file, content
320def _validate_rm_mv_conffile(
321 package: str,
322 config_line: DHConfigFileLine,
323) -> Tuple[str, str, Optional[str], Optional[str], Optional[str]]:
324 cmd, *args = config_line.tokens
325 if "--" in config_line.tokens: 325 ↛ 326line 325 didn't jump to line 326 because the condition on line 325 was never true
326 raise ValueError(
327 f'The maintscripts file "{config_line.config_file.path}" for {package} includes a "--" in line'
328 f" {config_line.line_no}. The offending line is: {config_line.original_line}"
329 )
330 if cmd == "rm_conffile":
331 min_args = 1
332 max_args = 3
333 else:
334 min_args = 2
335 max_args = 4
336 if len(args) > max_args or len(args) < min_args: 336 ↛ 337line 336 didn't jump to line 337 because the condition on line 336 was never true
337 raise ValueError(
338 f'The "{cmd}" command takes at least {min_args} and at most {max_args} arguments. However,'
339 f' in "{config_line.config_file.path}" line {config_line.line_no} (for {package}), there'
340 f" are {len(args)} arguments. The offending line is: {config_line.original_line}"
341 )
343 obsolete_conffile = args[0]
344 new_conffile = args[1] if cmd == "mv_conffile" else None
345 prior_version = args[min_args] if len(args) > min_args else None
346 owning_package = args[min_args + 1] if len(args) > min_args + 1 else None
347 if not obsolete_conffile.startswith("/"): 347 ↛ 348line 347 didn't jump to line 348 because the condition on line 347 was never true
348 raise ValueError(
349 f'The (old-)conffile parameter for {cmd} must be absolute (i.e., start with "/"). However,'
350 f' in "{config_line.config_file.path}" line {config_line.line_no} (for {package}), it was specified'
351 f' as "{obsolete_conffile}". The offending line is: {config_line.original_line}'
352 )
353 if new_conffile is not None and not new_conffile.startswith("/"): 353 ↛ 354line 353 didn't jump to line 354 because the condition on line 353 was never true
354 raise ValueError(
355 f'The new-conffile parameter for {cmd} must be absolute (i.e., start with "/"). However,'
356 f' in "{config_line.config_file.path}" line {config_line.line_no} (for {package}), it was specified'
357 f' as "{new_conffile}". The offending line is: {config_line.original_line}'
358 )
359 if prior_version is not None and not PKGVERSION_REGEX.fullmatch(prior_version): 359 ↛ 360line 359 didn't jump to line 360 because the condition on line 359 was never true
360 raise ValueError(
361 f"The prior-version parameter for {cmd} must be a valid package version (i.e., match"
362 f' {PKGVERSION_REGEX}). However, in "{config_line.config_file.path}" line {config_line.line_no}'
363 f' (for {package}), it was specified as "{prior_version}". The offending line is:'
364 f" {config_line.original_line}"
365 )
366 if owning_package is not None and not PKGNAME_REGEX.fullmatch(owning_package): 366 ↛ 367line 366 didn't jump to line 367 because the condition on line 366 was never true
367 raise ValueError(
368 f"The package parameter for {cmd} must be a valid package name (i.e., match {PKGNAME_REGEX})."
369 f' However, in "{config_line.config_file.path}" line {config_line.line_no} (for {package}), it'
370 f' was specified as "{owning_package}". The offending line is: {config_line.original_line}'
371 )
372 return cmd, obsolete_conffile, new_conffile, prior_version, owning_package
375_BASH_COMPLETION_RE = re.compile(
376 r"""
377 (^|[|&;])\s*complete.*-[A-Za-z].*
378 | \$\(.*\)
379 | \s*compgen.*-[A-Za-z].*
380 | \s*if.*;.*then/
381""",
382 re.VERBOSE,
383)
386def migrate_bash_completion(
387 debian_dir: VirtualPath,
388 manifest: HighLevelManifest,
389 acceptable_migration_issues: AcceptableMigrationIssues,
390 feature_migration: FeatureMigration,
391 _migration_target: DebputyIntegrationMode,
392) -> None:
393 feature_migration.tagline = "dh_bash-completion files"
394 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
395 mutable_manifest = assume_not_none(manifest.mutable_manifest)
396 installations = mutable_manifest.installations(create_if_absent=False)
398 for dctrl_bin in manifest.all_packages:
399 dh_file = dhe_pkgfile(debian_dir, dctrl_bin, "bash-completion")
400 if dh_file is None:
401 continue
402 is_bash_completion_file = False
403 with dh_file.open() as fd:
404 for line in fd:
405 line = line.strip()
406 if not line or line[0] == "#": 406 ↛ 407line 406 didn't jump to line 407 because the condition on line 406 was never true
407 continue
408 if _BASH_COMPLETION_RE.search(line):
409 is_bash_completion_file = True
410 break
411 if not is_bash_completion_file:
412 _, content = _dh_config_file(
413 debian_dir,
414 dctrl_bin,
415 "bash-completion",
416 "dh_bash-completion",
417 acceptable_migration_issues,
418 feature_migration,
419 manifest,
420 support_executable_files=True,
421 )
422 else:
423 content = None
425 if content:
426 install_dest_sources: List[str] = []
427 install_as_rules: List[Tuple[str, str]] = []
428 for dhe_line in content:
429 if len(dhe_line.tokens) > 2: 429 ↛ 430line 429 didn't jump to line 430 because the condition on line 429 was never true
430 raise UnsupportedFeature(
431 f"The dh_bash-completion file {dh_file.path} more than two words on"
432 f' line {dhe_line.line_no} (line: "{dhe_line.original_line}").'
433 )
434 source = dhe_line.tokens[0]
435 dest_basename = (
436 dhe_line.tokens[1]
437 if len(dhe_line.tokens) > 1
438 else os.path.basename(source)
439 )
440 if source.startswith("debian/") and not has_glob_magic(source):
441 if dctrl_bin.name != dest_basename:
442 dest_path = (
443 f"debian/{dctrl_bin.name}.{dest_basename}.bash-completion"
444 )
445 else:
446 dest_path = f"debian/{dest_basename}.bash-completion"
447 feature_migration.rename_on_success(source, dest_path)
448 elif len(dhe_line.tokens) == 1:
449 install_dest_sources.append(source)
450 else:
451 install_as_rules.append((source, dest_basename))
453 if install_dest_sources: 453 ↛ 467line 453 didn't jump to line 467 because the condition on line 453 was always true
454 sources: Union[List[str], str] = (
455 install_dest_sources
456 if len(install_dest_sources) > 1
457 else install_dest_sources[0]
458 )
459 installations.append(
460 AbstractMutableYAMLInstallRule.install_dest(
461 sources=sources,
462 dest_dir="{{path:BASH_COMPLETION_DIR}}",
463 into=dctrl_bin.name if not is_single_binary else None,
464 )
465 )
467 for source, dest_basename in install_as_rules:
468 installations.append(
469 AbstractMutableYAMLInstallRule.install_as(
470 source=source,
471 install_as="{{path:BASH_COMPLETION_DIR}}/" + dest_basename,
472 into=dctrl_bin.name if not is_single_binary else None,
473 )
474 )
477_SHELL_COMPLETIONS_RE = re.compile(r"^\s*\S+\s+\S+\s+\S")
480def migrate_shell_completions(
481 debian_dir: VirtualPath,
482 manifest: HighLevelManifest,
483 acceptable_migration_issues: AcceptableMigrationIssues,
484 feature_migration: FeatureMigration,
485 _migration_target: DebputyIntegrationMode,
486) -> None:
487 feature_migration.tagline = "dh_shell_completions files"
488 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
489 mutable_manifest = assume_not_none(manifest.mutable_manifest)
490 installations = mutable_manifest.installations(create_if_absent=False)
491 # Note: The bash completion script used `bash-completion` whereas `dh_shell_completions` uses
492 # `...-completions` (note the trailing `s`). In `debputy`, we always use the singular notation
493 # because we use "one file, one completion (ruleset)".
494 completions = ["bash", "fish", "zsh"]
496 for completion, dctrl_bin in product(completions, manifest.all_packages):
497 dh_file = dhe_pkgfile(debian_dir, dctrl_bin, f"{completion}-completions")
498 if dh_file is None:
499 continue
500 is_completion_file = False
501 with dh_file.open() as fd:
502 for line in fd:
503 line = line.strip()
504 if not line or line[0] == "#":
505 continue
506 if _SHELL_COMPLETIONS_RE.search(line):
507 is_completion_file = True
508 break
509 if is_completion_file:
510 dest_path = f"debian/{dctrl_bin.name}.{completion}-completion"
511 feature_migration.rename_on_success(dh_file.fs_path, dest_path)
512 continue
514 _, content = _dh_config_file(
515 debian_dir,
516 dctrl_bin,
517 f"{completion}-completions",
518 "dh_shell_completions",
519 acceptable_migration_issues,
520 feature_migration,
521 manifest,
522 remove_on_migration=True,
523 )
525 if content: 525 ↛ 496line 525 didn't jump to line 496 because the condition on line 525 was always true
526 install_dest_sources: List[str] = []
527 install_as_rules: List[Tuple[str, str]] = []
528 for dhe_line in content:
529 if len(dhe_line.tokens) > 2: 529 ↛ 530line 529 didn't jump to line 530 because the condition on line 529 was never true
530 raise UnsupportedFeature(
531 f"The dh_shell_completions file {dh_file.path} more than two words on"
532 f' line {dhe_line.line_no} (line: "{dhe_line.original_line}").'
533 )
534 source = dhe_line.tokens[0]
535 dest_basename = (
536 dhe_line.tokens[1]
537 if len(dhe_line.tokens) > 1
538 else os.path.basename(source)
539 )
540 if source.startswith("debian/") and not has_glob_magic(source):
541 if dctrl_bin.name != dest_basename:
542 dest_path = f"debian/{dctrl_bin.name}.{dest_basename}.{completion}-completion"
543 else:
544 dest_path = f"debian/{dest_basename}.{completion}-completion"
545 feature_migration.rename_on_success(source, dest_path)
546 elif len(dhe_line.tokens) == 1:
547 install_dest_sources.append(source)
548 else:
549 install_as_rules.append((source, dest_basename))
551 completion_dir_variable = (
552 "{{path:" + f"{completion.upper()}_COMPLETION_DIR" + "}}"
553 )
555 if install_dest_sources: 555 ↛ 569line 555 didn't jump to line 569 because the condition on line 555 was always true
556 sources: Union[List[str], str] = (
557 install_dest_sources
558 if len(install_dest_sources) > 1
559 else install_dest_sources[0]
560 )
561 installations.append(
562 AbstractMutableYAMLInstallRule.install_dest(
563 sources=sources,
564 dest_dir=completion_dir_variable,
565 into=dctrl_bin.name if not is_single_binary else None,
566 )
567 )
569 for source, dest_basename in install_as_rules:
570 installations.append(
571 AbstractMutableYAMLInstallRule.install_as(
572 source=source,
573 install_as=f"{completion_dir_variable}/{dest_basename}",
574 into=dctrl_bin.name if not is_single_binary else None,
575 )
576 )
579def migrate_dh_installsystemd_files(
580 debian_dir: VirtualPath,
581 manifest: HighLevelManifest,
582 _acceptable_migration_issues: AcceptableMigrationIssues,
583 feature_migration: FeatureMigration,
584 _migration_target: DebputyIntegrationMode,
585) -> None:
586 feature_migration.tagline = "dh_installsystemd files"
587 for dctrl_bin in manifest.all_packages:
588 for stem in [
589 "path",
590 "service",
591 "socket",
592 "target",
593 "timer",
594 ]:
595 pkgfile = dhe_pkgfile(
596 debian_dir, dctrl_bin, stem, bug_950723_prefix_matching=True
597 )
598 if not pkgfile:
599 continue
600 if not pkgfile.name.endswith(f".{stem}") or "@." not in pkgfile.name: 600 ↛ 601line 600 didn't jump to line 601 because the condition on line 600 was never true
601 raise UnsupportedFeature(
602 f'Unable to determine the correct name for {pkgfile.fs_path}. It should be a ".@{stem}"'
603 f" file now (foo@.service => foo.@service)"
604 )
605 newname = pkgfile.name.replace("@.", ".")
606 newname = newname[: -len(stem)] + f"@{stem}"
607 feature_migration.rename_on_success(
608 pkgfile.fs_path, os.path.join(debian_dir.fs_path, newname)
609 )
612def migrate_maintscript(
613 debian_dir: VirtualPath,
614 manifest: HighLevelManifest,
615 acceptable_migration_issues: AcceptableMigrationIssues,
616 feature_migration: FeatureMigration,
617 _migration_target: DebputyIntegrationMode,
618) -> None:
619 feature_migration.tagline = "dh_installdeb files"
620 mutable_manifest = assume_not_none(manifest.mutable_manifest)
621 for dctrl_bin in manifest.all_packages:
622 mainscript_file, content = _dh_config_file(
623 debian_dir,
624 dctrl_bin,
625 "maintscript",
626 "dh_installdeb",
627 acceptable_migration_issues,
628 feature_migration,
629 manifest,
630 )
632 if mainscript_file is None:
633 continue
634 assert content is not None
636 package_definition = mutable_manifest.package(dctrl_bin.name)
637 conffiles = {
638 it.obsolete_conffile: it
639 for it in package_definition.conffile_management_items()
640 }
641 seen_conffiles = set()
643 for dhe_line in content:
644 cmd = dhe_line.tokens[0]
645 if cmd not in {"rm_conffile", "mv_conffile"}: 645 ↛ 646line 645 didn't jump to line 646 because the condition on line 645 was never true
646 raise UnsupportedFeature(
647 f"The dh_installdeb file {mainscript_file.path} contains the (currently)"
648 f' unsupported command "{cmd}" on line {dhe_line.line_no}'
649 f' (line: "{dhe_line.original_line}")'
650 )
652 try:
653 (
654 _,
655 obsolete_conffile,
656 new_conffile,
657 prior_to_version,
658 owning_package,
659 ) = _validate_rm_mv_conffile(dctrl_bin.name, dhe_line)
660 except ValueError as e:
661 _error(
662 f"Validation error in {mainscript_file} on line {dhe_line.line_no}. The error was: {e.args[0]}."
663 )
665 if obsolete_conffile in seen_conffiles: 665 ↛ 666line 665 didn't jump to line 666 because the condition on line 665 was never true
666 raise ConflictingChange(
667 f'The {mainscript_file} file defines actions for "{obsolete_conffile}" twice!'
668 f" Please ensure that it is defined at most once in that file."
669 )
670 seen_conffiles.add(obsolete_conffile)
672 if cmd == "rm_conffile":
673 item = MutableYAMLConffileManagementItem.rm_conffile(
674 obsolete_conffile,
675 prior_to_version,
676 owning_package,
677 )
678 else:
679 assert cmd == "mv_conffile"
680 item = MutableYAMLConffileManagementItem.mv_conffile(
681 obsolete_conffile,
682 assume_not_none(new_conffile),
683 prior_to_version,
684 owning_package,
685 )
687 existing_def = conffiles.get(item.obsolete_conffile)
688 if existing_def is not None: 688 ↛ 689line 688 didn't jump to line 689 because the condition on line 688 was never true
689 if not (
690 item.command == existing_def.command
691 and item.new_conffile == existing_def.new_conffile
692 and item.prior_to_version == existing_def.prior_to_version
693 and item.owning_package == existing_def.owning_package
694 ):
695 raise ConflictingChange(
696 f"The maintscript defines the action {item.command} for"
697 f' "{obsolete_conffile}" in {mainscript_file}, but there is another'
698 f" conffile management definition for same path defined already (in the"
699 f" existing manifest or an migration e.g., inside {mainscript_file})"
700 )
701 feature_migration.already_present += 1
702 continue
704 package_definition.add_conffile_management(item)
705 feature_migration.successful_manifest_changes += 1
708@dataclasses.dataclass(slots=True)
709class SourcesAndConditional:
710 dest_dir: Optional[str] = None
711 sources: List[str] = dataclasses.field(default_factory=list)
712 conditional: Optional[Union[str, Mapping[str, Any]]] = None
715def _strip_d_tmp(p: str) -> str:
716 if p.startswith("debian/tmp/") and len(p) > 11:
717 return p[11:]
718 return p
721def migrate_install_file(
722 debian_dir: VirtualPath,
723 manifest: HighLevelManifest,
724 acceptable_migration_issues: AcceptableMigrationIssues,
725 feature_migration: FeatureMigration,
726 _migration_target: DebputyIntegrationMode,
727) -> None:
728 feature_migration.tagline = "dh_install config files"
729 mutable_manifest = assume_not_none(manifest.mutable_manifest)
730 installations = mutable_manifest.installations(create_if_absent=False)
731 priority_lines = []
732 remaining_install_lines = []
733 warn_about_fixmes_in_dest_dir = False
735 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
737 for dctrl_bin in manifest.all_packages:
738 install_file, content = _dh_config_file(
739 debian_dir,
740 dctrl_bin,
741 "install",
742 "dh_install",
743 acceptable_migration_issues,
744 feature_migration,
745 manifest,
746 support_executable_files=True,
747 allow_dh_exec_rename=True,
748 )
749 if not install_file or not content:
750 continue
751 current_sources = []
752 sources_by_destdir: Dict[Tuple[str, Tuple[str, ...]], SourcesAndConditional] = (
753 {}
754 )
755 install_as_rules = []
756 multi_dest = collections.defaultdict(list)
757 seen_sources = set()
758 multi_dest_sources: Set[str] = set()
760 for dhe_line in content:
761 special_rule = None
762 if "=>" in dhe_line.tokens:
763 if dhe_line.tokens[0] == "=>" and len(dhe_line.tokens) == 2:
764 # This rule must be as early as possible to retain the semantics
765 path = _strip_d_tmp(
766 _normalize_path(dhe_line.tokens[1], with_prefix=False)
767 )
768 special_rule = AbstractMutableYAMLInstallRule.install_dest(
769 path,
770 dctrl_bin.name if not is_single_binary else None,
771 dest_dir=None,
772 when=dhe_line.conditional(),
773 )
774 elif len(dhe_line.tokens) != 3: 774 ↛ 775line 774 didn't jump to line 775 because the condition on line 774 was never true
775 _error(
776 f"Validation error in {install_file.path} on line {dhe_line.line_no}. Cannot migrate dh-exec"
777 ' renames that is not exactly "SOURCE => TARGET" or "=> TARGET".'
778 )
779 else:
780 install_rule = AbstractMutableYAMLInstallRule.install_as(
781 _strip_d_tmp(
782 _normalize_path(dhe_line.tokens[0], with_prefix=False)
783 ),
784 _normalize_path(dhe_line.tokens[2], with_prefix=False),
785 dctrl_bin.name if not is_single_binary else None,
786 when=dhe_line.conditional(),
787 )
788 install_as_rules.append(install_rule)
789 else:
790 if len(dhe_line.tokens) > 1:
791 sources = list(
792 _strip_d_tmp(_normalize_path(w, with_prefix=False))
793 for w in dhe_line.tokens[:-1]
794 )
795 dest_dir = _normalize_path(dhe_line.tokens[-1], with_prefix=False)
796 else:
797 sources = list(
798 _strip_d_tmp(_normalize_path(w, with_prefix=False))
799 for w in dhe_line.tokens
800 )
801 dest_dir = None
803 multi_dest_sources.update(s for s in sources if s in seen_sources)
804 seen_sources.update(sources)
806 if dest_dir is None and dhe_line.conditional() is None:
807 current_sources.extend(sources)
808 continue
809 key = (dest_dir, dhe_line.conditional_key())
810 ctor = functools.partial(
811 SourcesAndConditional,
812 dest_dir=dest_dir,
813 conditional=dhe_line.conditional(),
814 )
815 md = _fetch_or_create(
816 sources_by_destdir,
817 key,
818 ctor,
819 )
820 md.sources.extend(sources)
822 if special_rule:
823 priority_lines.append(special_rule)
825 remaining_install_lines.extend(install_as_rules)
827 for md in sources_by_destdir.values():
828 if multi_dest_sources:
829 sources = [s for s in md.sources if s not in multi_dest_sources]
830 already_installed = (s for s in md.sources if s in multi_dest_sources)
831 for s in already_installed:
832 # The sources are ignored, so we can reuse the object as-is
833 multi_dest[s].append(md)
834 if not sources:
835 continue
836 else:
837 sources = md.sources
838 install_rule = AbstractMutableYAMLInstallRule.install_dest(
839 sources[0] if len(sources) == 1 else sources,
840 dctrl_bin.name if not is_single_binary else None,
841 dest_dir=md.dest_dir,
842 when=md.conditional,
843 )
844 remaining_install_lines.append(install_rule)
846 if current_sources:
847 if multi_dest_sources:
848 sources = [s for s in current_sources if s not in multi_dest_sources]
849 already_installed = (
850 s for s in current_sources if s in multi_dest_sources
851 )
852 for s in already_installed:
853 # The sources are ignored, so we can reuse the object as-is
854 dest_dir = os.path.dirname(s)
855 if has_glob_magic(dest_dir):
856 warn_about_fixmes_in_dest_dir = True
857 dest_dir = f"FIXME: {dest_dir} (could not reliably compute the dest dir)"
858 multi_dest[s].append(
859 SourcesAndConditional(
860 dest_dir=dest_dir,
861 conditional=None,
862 )
863 )
864 else:
865 sources = current_sources
867 if sources:
868 install_rule = AbstractMutableYAMLInstallRule.install_dest(
869 sources[0] if len(sources) == 1 else sources,
870 dctrl_bin.name if not is_single_binary else None,
871 dest_dir=None,
872 )
873 remaining_install_lines.append(install_rule)
875 if multi_dest:
876 for source, dest_and_conditionals in multi_dest.items():
877 dest_dirs = [dac.dest_dir for dac in dest_and_conditionals]
878 # We assume the conditional is the same.
879 conditional = next(
880 iter(
881 dac.conditional
882 for dac in dest_and_conditionals
883 if dac.conditional is not None
884 ),
885 None,
886 )
887 remaining_install_lines.append(
888 AbstractMutableYAMLInstallRule.multi_dest_install(
889 source,
890 dest_dirs,
891 dctrl_bin.name if not is_single_binary else None,
892 when=conditional,
893 )
894 )
896 if priority_lines:
897 installations.extend(priority_lines)
899 if remaining_install_lines:
900 installations.extend(remaining_install_lines)
902 feature_migration.successful_manifest_changes += len(priority_lines) + len(
903 remaining_install_lines
904 )
905 if warn_about_fixmes_in_dest_dir:
906 feature_migration.warn(
907 "TODO: FIXME left in dest-dir(s) of some installation rules."
908 " Please review these and remove the FIXME (plus correct as necessary)"
909 )
912def migrate_installdocs_file(
913 debian_dir: VirtualPath,
914 manifest: HighLevelManifest,
915 acceptable_migration_issues: AcceptableMigrationIssues,
916 feature_migration: FeatureMigration,
917 _migration_target: DebputyIntegrationMode,
918) -> None:
919 feature_migration.tagline = "dh_installdocs config files"
920 mutable_manifest = assume_not_none(manifest.mutable_manifest)
921 installations = mutable_manifest.installations(create_if_absent=False)
923 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
925 for dctrl_bin in manifest.all_packages:
926 install_file, content = _dh_config_file(
927 debian_dir,
928 dctrl_bin,
929 "docs",
930 "dh_installdocs",
931 acceptable_migration_issues,
932 feature_migration,
933 manifest,
934 support_executable_files=True,
935 )
936 if not install_file:
937 continue
938 assert content is not None
939 docs: List[str] = []
940 for dhe_line in content:
941 if dhe_line.arch_filter or dhe_line.build_profile_filter: 941 ↛ 942line 941 didn't jump to line 942 because the condition on line 941 was never true
942 _error(
943 f"Unable to migrate line {dhe_line.line_no} of {install_file.path}."
944 " Missing support for conditions."
945 )
946 docs.extend(_normalize_path(w, with_prefix=False) for w in dhe_line.tokens)
948 if not docs: 948 ↛ 949line 948 didn't jump to line 949 because the condition on line 948 was never true
949 continue
950 feature_migration.successful_manifest_changes += 1
951 install_rule = AbstractMutableYAMLInstallRule.install_docs(
952 docs if len(docs) > 1 else docs[0],
953 dctrl_bin.name if not is_single_binary else None,
954 )
955 installations.create_definition_if_missing()
956 installations.append(install_rule)
959def migrate_installexamples_file(
960 debian_dir: VirtualPath,
961 manifest: HighLevelManifest,
962 acceptable_migration_issues: AcceptableMigrationIssues,
963 feature_migration: FeatureMigration,
964 _migration_target: DebputyIntegrationMode,
965) -> None:
966 feature_migration.tagline = "dh_installexamples config files"
967 mutable_manifest = assume_not_none(manifest.mutable_manifest)
968 installations = mutable_manifest.installations(create_if_absent=False)
969 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
971 for dctrl_bin in manifest.all_packages:
972 install_file, content = _dh_config_file(
973 debian_dir,
974 dctrl_bin,
975 "examples",
976 "dh_installexamples",
977 acceptable_migration_issues,
978 feature_migration,
979 manifest,
980 support_executable_files=True,
981 )
982 if not install_file:
983 continue
984 assert content is not None
985 examples: List[str] = []
986 for dhe_line in content:
987 if dhe_line.arch_filter or dhe_line.build_profile_filter: 987 ↛ 988line 987 didn't jump to line 988 because the condition on line 987 was never true
988 _error(
989 f"Unable to migrate line {dhe_line.line_no} of {install_file.path}."
990 " Missing support for conditions."
991 )
992 examples.extend(
993 _normalize_path(w, with_prefix=False) for w in dhe_line.tokens
994 )
996 if not examples: 996 ↛ 997line 996 didn't jump to line 997 because the condition on line 996 was never true
997 continue
998 feature_migration.successful_manifest_changes += 1
999 install_rule = AbstractMutableYAMLInstallRule.install_examples(
1000 examples if len(examples) > 1 else examples[0],
1001 dctrl_bin.name if not is_single_binary else None,
1002 )
1003 installations.create_definition_if_missing()
1004 installations.append(install_rule)
1007@dataclasses.dataclass(slots=True)
1008class InfoFilesDefinition:
1009 sources: List[str] = dataclasses.field(default_factory=list)
1010 conditional: Optional[Union[str, Mapping[str, Any]]] = None
1013def migrate_installinfo_file(
1014 debian_dir: VirtualPath,
1015 manifest: HighLevelManifest,
1016 acceptable_migration_issues: AcceptableMigrationIssues,
1017 feature_migration: FeatureMigration,
1018 _migration_target: DebputyIntegrationMode,
1019) -> None:
1020 feature_migration.tagline = "dh_installinfo config files"
1021 mutable_manifest = assume_not_none(manifest.mutable_manifest)
1022 installations = mutable_manifest.installations(create_if_absent=False)
1023 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
1025 for dctrl_bin in manifest.all_packages:
1026 info_file, content = _dh_config_file(
1027 debian_dir,
1028 dctrl_bin,
1029 "info",
1030 "dh_installinfo",
1031 acceptable_migration_issues,
1032 feature_migration,
1033 manifest,
1034 support_executable_files=True,
1035 )
1036 if not info_file:
1037 continue
1038 assert content is not None
1039 info_files_by_condition: Dict[Tuple[str, ...], InfoFilesDefinition] = {}
1040 for dhe_line in content:
1041 key = dhe_line.conditional_key()
1042 ctr = functools.partial(
1043 InfoFilesDefinition, conditional=dhe_line.conditional()
1044 )
1045 info_def = _fetch_or_create(
1046 info_files_by_condition,
1047 key,
1048 ctr,
1049 )
1050 info_def.sources.extend(
1051 _normalize_path(w, with_prefix=False) for w in dhe_line.tokens
1052 )
1054 if not info_files_by_condition: 1054 ↛ 1055line 1054 didn't jump to line 1055 because the condition on line 1054 was never true
1055 continue
1056 feature_migration.successful_manifest_changes += 1
1057 installations.create_definition_if_missing()
1058 for info_def in info_files_by_condition.values():
1059 info_files = info_def.sources
1060 install_rule = AbstractMutableYAMLInstallRule.install_docs(
1061 info_files if len(info_files) > 1 else info_files[0],
1062 dctrl_bin.name if not is_single_binary else None,
1063 dest_dir="{{path:GNU_INFO_DIR}}",
1064 when=info_def.conditional,
1065 )
1066 installations.append(install_rule)
1069@dataclasses.dataclass(slots=True)
1070class ManpageDefinition:
1071 sources: List[str] = dataclasses.field(default_factory=list)
1072 language: Optional[str] = None
1073 conditional: Optional[Union[str, Mapping[str, Any]]] = None
1076DK = TypeVar("DK")
1077DV = TypeVar("DV")
1080def _fetch_or_create(d: Dict[DK, DV], key: DK, factory: Callable[[], DV]) -> DV:
1081 v = d.get(key)
1082 if v is None:
1083 v = factory()
1084 d[key] = v
1085 return v
1088def migrate_installman_file(
1089 debian_dir: VirtualPath,
1090 manifest: HighLevelManifest,
1091 acceptable_migration_issues: AcceptableMigrationIssues,
1092 feature_migration: FeatureMigration,
1093 _migration_target: DebputyIntegrationMode,
1094) -> None:
1095 feature_migration.tagline = "dh_installman config files"
1096 mutable_manifest = assume_not_none(manifest.mutable_manifest)
1097 installations = mutable_manifest.installations(create_if_absent=False)
1098 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
1099 warn_about_basename = False
1101 for dctrl_bin in manifest.all_packages:
1102 manpages_file, content = _dh_config_file(
1103 debian_dir,
1104 dctrl_bin,
1105 "manpages",
1106 "dh_installman",
1107 acceptable_migration_issues,
1108 feature_migration,
1109 manifest,
1110 support_executable_files=True,
1111 allow_dh_exec_rename=True,
1112 )
1113 if not manpages_file:
1114 continue
1115 assert content is not None
1117 vanilla_definitions = []
1118 install_as_rules = []
1119 complex_definitions: Dict[
1120 Tuple[Optional[str], Tuple[str, ...]], ManpageDefinition
1121 ] = {}
1122 install_rule: AbstractMutableYAMLInstallRule
1123 for dhe_line in content:
1124 if "=>" in dhe_line.tokens: 1124 ↛ 1127line 1124 didn't jump to line 1127 because the condition on line 1124 was never true
1125 # dh-exec allows renaming features. For `debputy`, we degenerate it into an `install` (w. `as`) feature
1126 # without any of the `install-man` features.
1127 if dhe_line.tokens[0] == "=>" and len(dhe_line.tokens) == 2:
1128 _error(
1129 f'Unsupported "=> DEST" rule for error in {manpages_file.path} on line {dhe_line.line_no}."'
1130 f' Cannot migrate dh-exec renames that is not exactly "SOURCE => TARGET" for d/manpages files.'
1131 )
1132 elif len(dhe_line.tokens) != 3:
1133 _error(
1134 f"Validation error in {manpages_file.path} on line {dhe_line.line_no}. Cannot migrate dh-exec"
1135 ' renames that is not exactly "SOURCE => TARGET" or "=> TARGET".'
1136 )
1137 else:
1138 install_rule = AbstractMutableYAMLInstallRule.install_doc_as(
1139 _normalize_path(dhe_line.tokens[0], with_prefix=False),
1140 _normalize_path(dhe_line.tokens[2], with_prefix=False),
1141 dctrl_bin.name if not is_single_binary else None,
1142 when=dhe_line.conditional(),
1143 )
1144 install_as_rules.append(install_rule)
1145 continue
1147 sources = [_normalize_path(w, with_prefix=False) for w in dhe_line.tokens]
1148 needs_basename = any(
1149 MAN_GUESS_FROM_BASENAME.search(x)
1150 and not MAN_GUESS_LANG_FROM_PATH.search(x)
1151 for x in sources
1152 )
1153 if needs_basename or dhe_line.conditional() is not None:
1154 if needs_basename: 1154 ↛ 1158line 1154 didn't jump to line 1158 because the condition on line 1154 was always true
1155 warn_about_basename = True
1156 language = "derive-from-basename"
1157 else:
1158 language = None
1159 key = (language, dhe_line.conditional_key())
1160 ctor = functools.partial(
1161 ManpageDefinition,
1162 language=language,
1163 conditional=dhe_line.conditional(),
1164 )
1165 manpage_def = _fetch_or_create(
1166 complex_definitions,
1167 key,
1168 ctor,
1169 )
1170 manpage_def.sources.extend(sources)
1171 else:
1172 vanilla_definitions.extend(sources)
1174 if not install_as_rules and not vanilla_definitions and not complex_definitions: 1174 ↛ 1175line 1174 didn't jump to line 1175 because the condition on line 1174 was never true
1175 continue
1176 feature_migration.successful_manifest_changes += 1
1177 installations.create_definition_if_missing()
1178 installations.extend(install_as_rules)
1179 if vanilla_definitions: 1179 ↛ 1191line 1179 didn't jump to line 1191 because the condition on line 1179 was always true
1180 man_source = (
1181 vanilla_definitions
1182 if len(vanilla_definitions) > 1
1183 else vanilla_definitions[0]
1184 )
1185 install_rule = AbstractMutableYAMLInstallRule.install_man(
1186 man_source,
1187 dctrl_bin.name if not is_single_binary else None,
1188 None,
1189 )
1190 installations.append(install_rule)
1191 for manpage_def in complex_definitions.values():
1192 sources = manpage_def.sources
1193 install_rule = AbstractMutableYAMLInstallRule.install_man(
1194 sources if len(sources) > 1 else sources[0],
1195 dctrl_bin.name if not is_single_binary else None,
1196 manpage_def.language,
1197 when=manpage_def.conditional,
1198 )
1199 installations.append(install_rule)
1201 if warn_about_basename:
1202 feature_migration.warn(
1203 'Detected man pages that might rely on "derive-from-basename" logic. Please double check'
1204 " that the generated `install-man` rules are correct"
1205 )
1208def migrate_not_installed_file(
1209 debian_dir: VirtualPath,
1210 manifest: HighLevelManifest,
1211 acceptable_migration_issues: AcceptableMigrationIssues,
1212 feature_migration: FeatureMigration,
1213 _migration_target: DebputyIntegrationMode,
1214) -> None:
1215 feature_migration.tagline = "dh_missing's not-installed config file"
1216 mutable_manifest = assume_not_none(manifest.mutable_manifest)
1217 installations = mutable_manifest.installations(create_if_absent=False)
1218 main_binary = [p for p in manifest.all_packages if p.is_main_package][0]
1220 missing_file, content = _dh_config_file(
1221 debian_dir,
1222 main_binary,
1223 "not-installed",
1224 "dh_missing",
1225 acceptable_migration_issues,
1226 feature_migration,
1227 manifest,
1228 support_executable_files=False,
1229 pkgfile_lookup=False,
1230 )
1231 discard_rules: List[str] = []
1232 if missing_file:
1233 assert content is not None
1234 for dhe_line in content:
1235 discard_rules.extend(
1236 _normalize_path(w, with_prefix=False) for w in dhe_line.tokens
1237 )
1239 if discard_rules:
1240 feature_migration.successful_manifest_changes += 1
1241 install_rule = AbstractMutableYAMLInstallRule.discard(
1242 discard_rules if len(discard_rules) > 1 else discard_rules[0],
1243 )
1244 installations.create_definition_if_missing()
1245 installations.append(install_rule)
1248def detect_pam_files(
1249 debian_dir: VirtualPath,
1250 manifest: HighLevelManifest,
1251 _acceptable_migration_issues: AcceptableMigrationIssues,
1252 feature_migration: FeatureMigration,
1253 _migration_target: DebputyIntegrationMode,
1254) -> None:
1255 feature_migration.tagline = "detect dh_installpam files (min dh compat)"
1256 for dctrl_bin in manifest.all_packages:
1257 dh_config_file = dhe_pkgfile(debian_dir, dctrl_bin, "pam")
1258 if dh_config_file is not None:
1259 feature_migration.assumed_compat = 14
1260 break
1263def migrate_tmpfile(
1264 debian_dir: VirtualPath,
1265 manifest: HighLevelManifest,
1266 _acceptable_migration_issues: AcceptableMigrationIssues,
1267 feature_migration: FeatureMigration,
1268 _migration_target: DebputyIntegrationMode,
1269) -> None:
1270 feature_migration.tagline = "dh_installtmpfiles config files"
1271 for dctrl_bin in manifest.all_packages:
1272 dh_config_file = dhe_pkgfile(debian_dir, dctrl_bin, "tmpfile")
1273 if dh_config_file is not None:
1274 target = (
1275 dh_config_file.name.replace(".tmpfile", ".tmpfiles")
1276 if "." in dh_config_file.name
1277 else "tmpfiles"
1278 )
1279 _rename_file_if_exists(
1280 debian_dir,
1281 dh_config_file.name,
1282 target,
1283 feature_migration,
1284 )
1287def migrate_lintian_overrides_files(
1288 debian_dir: VirtualPath,
1289 manifest: HighLevelManifest,
1290 acceptable_migration_issues: AcceptableMigrationIssues,
1291 feature_migration: FeatureMigration,
1292 _migration_target: DebputyIntegrationMode,
1293) -> None:
1294 feature_migration.tagline = "dh_lintian config files"
1295 for dctrl_bin in manifest.all_packages:
1296 # We do not support executable lintian-overrides and `_dh_config_file` handles all of that.
1297 # Therefore, the return value is irrelevant to us.
1298 _dh_config_file(
1299 debian_dir,
1300 dctrl_bin,
1301 "lintian-overrides",
1302 "dh_lintian",
1303 acceptable_migration_issues,
1304 feature_migration,
1305 manifest,
1306 support_executable_files=False,
1307 remove_on_migration=False,
1308 )
1311def migrate_links_files(
1312 debian_dir: VirtualPath,
1313 manifest: HighLevelManifest,
1314 acceptable_migration_issues: AcceptableMigrationIssues,
1315 feature_migration: FeatureMigration,
1316 _migration_target: DebputyIntegrationMode,
1317) -> None:
1318 feature_migration.tagline = "dh_link files"
1319 mutable_manifest = assume_not_none(manifest.mutable_manifest)
1320 for dctrl_bin in manifest.all_packages:
1321 links_file, content = _dh_config_file(
1322 debian_dir,
1323 dctrl_bin,
1324 "links",
1325 "dh_link",
1326 acceptable_migration_issues,
1327 feature_migration,
1328 manifest,
1329 support_executable_files=True,
1330 )
1332 if links_file is None:
1333 continue
1334 assert content is not None
1336 package_definition = mutable_manifest.package(dctrl_bin.name)
1337 defined_symlink = {
1338 symlink.symlink_path: symlink.symlink_target
1339 for symlink in package_definition.symlinks()
1340 }
1342 seen_symlinks: Set[str] = set()
1344 for dhe_line in content:
1345 if len(dhe_line.tokens) != 2: 1345 ↛ 1346line 1345 didn't jump to line 1346 because the condition on line 1345 was never true
1346 raise UnsupportedFeature(
1347 f"The dh_link file {links_file.fs_path} did not have exactly two paths on line"
1348 f' {dhe_line.line_no} (line: "{dhe_line.original_line}"'
1349 )
1350 target, source = dhe_line.tokens
1351 if source in seen_symlinks: 1351 ↛ 1353line 1351 didn't jump to line 1353 because the condition on line 1351 was never true
1352 # According to #934499, this has happened in the wild already
1353 raise ConflictingChange(
1354 f"The {links_file.fs_path} file defines the link path {source} twice! Please ensure"
1355 " that it is defined at most once in that file"
1356 )
1357 seen_symlinks.add(source)
1358 # Symlinks in .links are always considered absolute, but you were not required to have a leading slash.
1359 # However, in the debputy manifest, you can have relative links, so we should ensure it is explicitly
1360 # absolute.
1361 if not target.startswith("/"): 1361 ↛ 1363line 1361 didn't jump to line 1363 because the condition on line 1361 was always true
1362 target = "/" + target
1363 existing_target = defined_symlink.get(source)
1364 if existing_target is not None: 1364 ↛ 1365line 1364 didn't jump to line 1365 because the condition on line 1364 was never true
1365 if existing_target != target:
1366 raise ConflictingChange(
1367 f'The symlink "{source}" points to "{target}" in {links_file}, but there is'
1368 f' another symlink with same path pointing to "{existing_target}" defined'
1369 " already (in the existing manifest or an migration e.g., inside"
1370 f" {links_file.fs_path})"
1371 )
1372 feature_migration.already_present += 1
1373 continue
1374 condition = dhe_line.conditional()
1375 package_definition.add_symlink(
1376 MutableYAMLSymlink.new_symlink(
1377 source,
1378 target,
1379 condition,
1380 )
1381 )
1382 feature_migration.successful_manifest_changes += 1
1385def migrate_misspelled_readme_debian_files(
1386 debian_dir: VirtualPath,
1387 manifest: HighLevelManifest,
1388 acceptable_migration_issues: AcceptableMigrationIssues,
1389 feature_migration: FeatureMigration,
1390 _migration_target: DebputyIntegrationMode,
1391) -> None:
1392 feature_migration.tagline = "misspelled README.Debian files"
1393 for dctrl_bin in manifest.all_packages:
1394 readme, _ = _dh_config_file(
1395 debian_dir,
1396 dctrl_bin,
1397 "README.debian",
1398 "dh_installdocs",
1399 acceptable_migration_issues,
1400 feature_migration,
1401 manifest,
1402 support_executable_files=False,
1403 remove_on_migration=False,
1404 )
1405 if readme is None:
1406 continue
1407 new_name = readme.name.replace("README.debian", "README.Debian")
1408 assert readme.name != new_name
1409 _rename_file_if_exists(
1410 debian_dir,
1411 readme.name,
1412 new_name,
1413 feature_migration,
1414 )
1417def migrate_doc_base_files(
1418 debian_dir: VirtualPath,
1419 manifest: HighLevelManifest,
1420 _: AcceptableMigrationIssues,
1421 feature_migration: FeatureMigration,
1422 _migration_target: DebputyIntegrationMode,
1423) -> None:
1424 feature_migration.tagline = "doc-base files"
1425 # ignore the dh_make ".EX" file if one should still be present. The dh_installdocs tool ignores it too.
1426 possible_effected_doc_base_files = [
1427 f
1428 for f in debian_dir.iterdir
1429 if (
1430 (".doc-base." in f.name or f.name.startswith("doc-base."))
1431 and not f.name.endswith("doc-base.EX")
1432 )
1433 ]
1434 known_packages = {d.name: d for d in manifest.all_packages}
1435 main_package = [d for d in manifest.all_packages if d.is_main_package][0]
1436 for doc_base_file in possible_effected_doc_base_files:
1437 parts = doc_base_file.name.split(".")
1438 owning_package = known_packages.get(parts[0])
1439 if owning_package is None: 1439 ↛ 1440line 1439 didn't jump to line 1440 because the condition on line 1439 was never true
1440 owning_package = main_package
1441 package_part = None
1442 else:
1443 package_part = parts[0]
1444 parts = parts[1:]
1446 if not parts or parts[0] != "doc-base": 1446 ↛ 1448line 1446 didn't jump to line 1448 because the condition on line 1446 was never true
1447 # Not a doc-base file after all
1448 continue
1450 if len(parts) > 1: 1450 ↛ 1457line 1450 didn't jump to line 1457 because the condition on line 1450 was always true
1451 name_part = ".".join(parts[1:])
1452 if package_part is None: 1452 ↛ 1454line 1452 didn't jump to line 1454 because the condition on line 1452 was never true
1453 # Named files must have a package prefix
1454 package_part = owning_package.name
1455 else:
1456 # No rename needed
1457 continue
1459 new_basename = ".".join(filter(None, (package_part, name_part, "doc-base")))
1460 _rename_file_if_exists(
1461 debian_dir,
1462 doc_base_file.name,
1463 new_basename,
1464 feature_migration,
1465 )
1468def migrate_dh_hook_targets(
1469 debian_dir: VirtualPath,
1470 _: HighLevelManifest,
1471 acceptable_migration_issues: AcceptableMigrationIssues,
1472 feature_migration: FeatureMigration,
1473 migration_target: DebputyIntegrationMode,
1474) -> None:
1475 feature_migration.tagline = "dh hook targets"
1476 source_root = os.path.dirname(debian_dir.fs_path)
1477 if source_root == "":
1478 source_root = "."
1479 detected_hook_targets = json.loads(
1480 subprocess.check_output(
1481 ["dh_assistant", "detect-hook-targets"],
1482 cwd=source_root,
1483 ).decode("utf-8")
1484 )
1485 sample_hook_target: Optional[str] = None
1486 replaced_commands = DH_COMMANDS_REPLACED[migration_target]
1488 for hook_target_def in detected_hook_targets["hook-targets"]:
1489 if hook_target_def["is-empty"]:
1490 continue
1491 command = hook_target_def["command"]
1492 if command not in replaced_commands:
1493 continue
1494 hook_target = hook_target_def["target-name"]
1495 advice = MIGRATION_AID_FOR_OVERRIDDEN_COMMANDS.get(command)
1496 if advice is None:
1497 if sample_hook_target is None:
1498 sample_hook_target = hook_target
1499 feature_migration.warn(
1500 f"TODO: MANUAL MIGRATION required for hook target {hook_target}"
1501 )
1502 else:
1503 feature_migration.warn(
1504 f"TODO: MANUAL MIGRATION required for hook target {hook_target}. Please see {advice}"
1505 f" for migration advice."
1506 )
1507 if (
1508 feature_migration.warnings
1509 and "dh-hook-targets" not in acceptable_migration_issues
1510 and sample_hook_target is not None
1511 ):
1512 raise UnsupportedFeature(
1513 f"The debian/rules file contains one or more non empty dh hook targets that will not"
1514 f" be run with the requested debputy dh sequence with no known migration advice. One of these would be"
1515 f" {sample_hook_target}.",
1516 ["dh-hook-targets"],
1517 )
1520def detect_unsupported_zz_debputy_features(
1521 debian_dir: VirtualPath,
1522 manifest: HighLevelManifest,
1523 acceptable_migration_issues: AcceptableMigrationIssues,
1524 feature_migration: FeatureMigration,
1525 _migration_target: DebputyIntegrationMode,
1526) -> None:
1527 feature_migration.tagline = "Known unsupported features"
1529 for unsupported_config in UNSUPPORTED_DH_CONFIGS_AND_TOOLS_FOR_ZZ_DEBPUTY:
1530 _unsupported_debhelper_config_file(
1531 debian_dir,
1532 manifest,
1533 unsupported_config,
1534 acceptable_migration_issues,
1535 feature_migration,
1536 )
1539def detect_obsolete_substvars(
1540 debian_dir: VirtualPath,
1541 _manifest: HighLevelManifest,
1542 _acceptable_migration_issues: AcceptableMigrationIssues,
1543 feature_migration: FeatureMigration,
1544 _migration_target: DebputyIntegrationMode,
1545) -> None:
1546 feature_migration.tagline = (
1547 "Check for obsolete ${foo:var} variables in debian/control"
1548 )
1549 ctrl_file = debian_dir.get("control")
1550 if not ctrl_file: 1550 ↛ 1551line 1550 didn't jump to line 1551 because the condition on line 1550 was never true
1551 feature_migration.warn(
1552 "Cannot find debian/control. Detection of obsolete substvars could not be performed."
1553 )
1554 return
1555 with ctrl_file.open() as fd:
1556 ctrl = list(Deb822.iter_paragraphs(fd))
1558 relationship_fields = dpkg_field_list_pkg_dep()
1559 relationship_fields_lc = frozenset(x.lower() for x in relationship_fields)
1561 for p in ctrl[1:]:
1562 seen_obsolete_relationship_substvars = set()
1563 obsolete_fields = set()
1564 is_essential = p.get("Essential") == "yes"
1565 for df in relationship_fields:
1566 field: Optional[str] = p.get(df)
1567 if field is None:
1568 continue
1569 df_lc = df.lower()
1570 number_of_relations = 0
1571 obsolete_substvars_in_field = set()
1572 for d in (d.strip() for d in field.strip().split(",")):
1573 if not d:
1574 continue
1575 number_of_relations += 1
1576 if not d.startswith("${"):
1577 continue
1578 try:
1579 end_idx = d.index("}")
1580 except ValueError:
1581 continue
1582 substvar_name = d[2:end_idx]
1583 if ":" not in substvar_name: 1583 ↛ 1584line 1583 didn't jump to line 1584 because the condition on line 1583 was never true
1584 continue
1585 _, field = substvar_name.rsplit(":", 1)
1586 field_lc = field.lower()
1587 if field_lc not in relationship_fields_lc: 1587 ↛ 1588line 1587 didn't jump to line 1588 because the condition on line 1587 was never true
1588 continue
1589 is_obsolete = field_lc == df_lc
1590 if (
1591 not is_obsolete
1592 and is_essential
1593 and substvar_name.lower() == "shlibs:depends"
1594 and df_lc == "pre-depends"
1595 ):
1596 is_obsolete = True
1598 if is_obsolete:
1599 obsolete_substvars_in_field.add(d)
1601 if number_of_relations == len(obsolete_substvars_in_field):
1602 obsolete_fields.add(df)
1603 else:
1604 seen_obsolete_relationship_substvars.update(obsolete_substvars_in_field)
1606 package = p.get("Package", "(Missing package name!?)")
1607 fo = feature_migration.fo
1608 if obsolete_fields:
1609 fields = ", ".join(obsolete_fields)
1610 feature_migration.warn(
1611 f"The following relationship fields can be removed from {package}: {fields}."
1612 f" (The content in them would be applied automatically. Note: {fo.bts('1067653')})"
1613 )
1614 if seen_obsolete_relationship_substvars:
1615 v = ", ".join(sorted(seen_obsolete_relationship_substvars))
1616 feature_migration.warn(
1617 f"The following relationship substitution variables can be removed from {package}: {v}"
1618 f" (Note: {fo.bts('1067653')})"
1619 )
1622def detect_dh_addons_zz_debputy_rrr(
1623 debian_dir: VirtualPath,
1624 _manifest: HighLevelManifest,
1625 _acceptable_migration_issues: AcceptableMigrationIssues,
1626 feature_migration: FeatureMigration,
1627 _migration_target: DebputyIntegrationMode,
1628) -> None:
1629 feature_migration.tagline = "Check for dh-sequence-addons"
1630 r = read_dh_addon_sequences(debian_dir)
1631 if r is None:
1632 feature_migration.warn(
1633 "Cannot find debian/control. Detection of unsupported/missing dh-sequence addon"
1634 " could not be performed. Please ensure the package will Build-Depend on dh-sequence-zz-debputy-rrr."
1635 )
1636 return
1638 bd_sequences, dr_sequences, _ = r
1640 remaining_sequences = bd_sequences | dr_sequences
1641 saw_dh_debputy = "zz-debputy-rrr" in remaining_sequences
1643 if not saw_dh_debputy:
1644 feature_migration.warn("Missing Build-Depends on dh-sequence-zz-debputy-rrr")
1647def detect_dh_addons_with_full_integration(
1648 _debian_dir: VirtualPath,
1649 _manifest: HighLevelManifest,
1650 _acceptable_migration_issues: AcceptableMigrationIssues,
1651 feature_migration: FeatureMigration,
1652 _migration_target: DebputyIntegrationMode,
1653) -> None:
1654 feature_migration.tagline = "Check for dh-sequence-addons and Build-Depends"
1655 feature_migration.warn(
1656 "TODO: Not implemented: Please remove any dh-sequence Build-Dependency"
1657 )
1658 feature_migration.warn(
1659 "TODO: Not implemented: Please ensure there is a Build-Dependency on `debputy (>= 0.1.45~)"
1660 )
1661 feature_migration.warn(
1662 "TODO: Not implemented: Please ensure there is a Build-Dependency on `dpkg-dev (>= 1.22.7~)"
1663 )
1666def detect_dh_addons_with_zz_integration(
1667 debian_dir: VirtualPath,
1668 _manifest: HighLevelManifest,
1669 acceptable_migration_issues: AcceptableMigrationIssues,
1670 feature_migration: FeatureMigration,
1671 _migration_target: DebputyIntegrationMode,
1672) -> None:
1673 feature_migration.tagline = "Check for dh-sequence-addons"
1674 r = read_dh_addon_sequences(debian_dir)
1675 if r is None:
1676 feature_migration.warn(
1677 "Cannot find debian/control. Detection of unsupported/missing dh-sequence addon"
1678 " could not be performed. Please ensure the package will Build-Depend on dh-sequence-zz-debputy"
1679 " and not rely on any other debhelper sequence addons except those debputy explicitly supports."
1680 )
1681 return
1683 assert _migration_target != INTEGRATION_MODE_FULL
1685 bd_sequences, dr_sequences, _ = r
1687 remaining_sequences = bd_sequences | dr_sequences
1688 saw_dh_debputy = (
1689 "debputy" in remaining_sequences or "zz-debputy" in remaining_sequences
1690 )
1691 saw_zz_debputy = "zz-debputy" in remaining_sequences
1692 must_use_zz_debputy = False
1693 remaining_sequences -= SUPPORTED_DH_ADDONS
1694 for sequence in remaining_sequences & DH_ADDONS_TO_PLUGINS.keys():
1695 migration = DH_ADDONS_TO_PLUGINS[sequence]
1696 feature_migration.require_plugin(migration.debputy_plugin)
1697 if migration.remove_dh_sequence: 1697 ↛ 1698line 1697 didn't jump to line 1698 because the condition on line 1697 was never true
1698 if migration.must_use_zz_debputy:
1699 must_use_zz_debputy = True
1700 if sequence in bd_sequences:
1701 feature_migration.warn(
1702 f"TODO: MANUAL MIGRATION - Remove build-dependency on dh-sequence-{sequence}"
1703 f" (replaced by debputy-plugin-{migration.debputy_plugin})"
1704 )
1705 else:
1706 feature_migration.warn(
1707 f"TODO: MANUAL MIGRATION - Remove --with {sequence} from dh in d/rules"
1708 f" (replaced by debputy-plugin-{migration.debputy_plugin})"
1709 )
1711 remaining_sequences -= DH_ADDONS_TO_PLUGINS.keys()
1713 alt_key = "unsupported-dh-sequences"
1714 for sequence in remaining_sequences & DH_ADDONS_TO_REMOVE: 1714 ↛ 1715line 1714 didn't jump to line 1715 because the loop on line 1714 never started
1715 if sequence in bd_sequences:
1716 feature_migration.warn(
1717 f"TODO: MANUAL MIGRATION - Remove build dependency on dh-sequence-{sequence}"
1718 )
1719 else:
1720 feature_migration.warn(
1721 f"TODO: MANUAL MIGRATION - Remove --with {sequence} from dh in d/rules"
1722 )
1724 remaining_sequences -= DH_ADDONS_TO_REMOVE
1726 for sequence in remaining_sequences:
1727 key = f"unsupported-dh-sequence-{sequence}"
1728 msg = f'The dh addon "{sequence}" is not known to work with dh-debputy and might malfunction'
1729 if (
1730 key not in acceptable_migration_issues
1731 and alt_key not in acceptable_migration_issues
1732 ):
1733 raise UnsupportedFeature(msg, [key, alt_key])
1734 feature_migration.warn(msg)
1736 if not saw_dh_debputy:
1737 feature_migration.warn("Missing Build-Depends on dh-sequence-zz-debputy")
1738 elif must_use_zz_debputy and not saw_zz_debputy: 1738 ↛ 1739line 1738 didn't jump to line 1739 because the condition on line 1738 was never true
1739 feature_migration.warn(
1740 "Please use the zz-debputy sequence rather than the debputy (needed due to dh add-on load order)"
1741 )
1744def _rename_file_if_exists(
1745 debian_dir: VirtualPath,
1746 source: str,
1747 dest: str,
1748 feature_migration: FeatureMigration,
1749) -> None:
1750 source_path = debian_dir.get(source)
1751 dest_path = debian_dir.get(dest)
1752 spath = (
1753 source_path.path
1754 if source_path is not None
1755 else os.path.join(debian_dir.path, source)
1756 )
1757 dpath = (
1758 dest_path.path if dest_path is not None else os.path.join(debian_dir.path, dest)
1759 )
1760 if source_path is not None and source_path.is_file:
1761 if dest_path is not None:
1762 if not dest_path.is_file:
1763 feature_migration.warnings.append(
1764 f'TODO: MANUAL MIGRATION - there is a "{spath}" (file) and "{dpath}" (not a file).'
1765 f' The migration wanted to replace "{spath}" with "{dpath}", but since "{dpath}" is not'
1766 " a file, this step is left as a manual migration."
1767 )
1768 return
1769 if (
1770 subprocess.call(["cmp", "-s", source_path.fs_path, dest_path.fs_path])
1771 != 0
1772 ):
1773 feature_migration.warnings.append(
1774 f'TODO: MANUAL MIGRATION - there is a "{source_path.path}" and "{dest_path.path}"'
1775 f" file. Normally these files are for the same package and there would only be one of"
1776 f" them. In this case, they both exist but their content differs. Be advised that"
1777 f' debputy tool will use the "{dest_path.path}".'
1778 )
1779 else:
1780 feature_migration.remove_on_success(dest_path.fs_path)
1781 else:
1782 feature_migration.rename_on_success(
1783 source_path.fs_path,
1784 os.path.join(debian_dir.fs_path, dest),
1785 )
1786 elif source_path is not None: 1786 ↛ exitline 1786 didn't return from function '_rename_file_if_exists' because the condition on line 1786 was always true
1787 feature_migration.warnings.append(
1788 f'TODO: MANUAL MIGRATION - The migration would normally have renamed "{spath}" to "{dpath}".'
1789 f' However, the migration assumed "{spath}" would be a file and it is not. Therefore, this step'
1790 " as a manual migration."
1791 )
1794def _find_dh_config_file_for_any_pkg(
1795 debian_dir: VirtualPath,
1796 manifest: HighLevelManifest,
1797 unsupported_config: UnsupportedDHConfig,
1798) -> Iterable[VirtualPath]:
1799 for dctrl_bin in manifest.all_packages:
1800 dh_config_file = dhe_pkgfile(
1801 debian_dir,
1802 dctrl_bin,
1803 unsupported_config.dh_config_basename,
1804 bug_950723_prefix_matching=unsupported_config.bug_950723_prefix_matching,
1805 )
1806 if dh_config_file is not None:
1807 yield dh_config_file
1810def _unsupported_debhelper_config_file(
1811 debian_dir: VirtualPath,
1812 manifest: HighLevelManifest,
1813 unsupported_config: UnsupportedDHConfig,
1814 acceptable_migration_issues: AcceptableMigrationIssues,
1815 feature_migration: FeatureMigration,
1816) -> None:
1817 dh_config_files = list(
1818 _find_dh_config_file_for_any_pkg(debian_dir, manifest, unsupported_config)
1819 )
1820 if not dh_config_files:
1821 return
1822 dh_tool = unsupported_config.dh_tool
1823 basename = unsupported_config.dh_config_basename
1824 file_stem = (
1825 f"@{basename}" if unsupported_config.bug_950723_prefix_matching else basename
1826 )
1827 dh_config_file = dh_config_files[0]
1828 if unsupported_config.is_missing_migration:
1829 feature_migration.warn(
1830 f'Missing migration support for the "{dh_config_file.path}" debhelper config file'
1831 f" (used by {dh_tool}). Manual migration may be feasible depending on the exact features"
1832 " required."
1833 )
1834 return
1835 primary_key = f"unsupported-dh-config-file-{file_stem}"
1836 secondary_key = "any-unsupported-dh-config-file"
1837 if (
1838 primary_key not in acceptable_migration_issues
1839 and secondary_key not in acceptable_migration_issues
1840 ):
1841 msg = (
1842 f'The "{dh_config_file.path}" debhelper config file (used by {dh_tool} is currently not'
1843 " supported by debputy."
1844 )
1845 raise UnsupportedFeature(
1846 msg,
1847 [primary_key, secondary_key],
1848 )
1849 for dh_config_file in dh_config_files:
1850 feature_migration.warn(
1851 f'TODO: MANUAL MIGRATION - Use of unsupported "{dh_config_file.path}" file (used by {dh_tool})'
1852 )