Coverage for src/debputy/dh_migration/migration.py: 7%
289 statements
« prev ^ index » next coverage.py v7.16.1, created at 2026-10-28 20:14 +0000
« prev ^ index » next coverage.py v7.16.1, created at 2026-10-28 20:14 +0000
1import json
2import os
3import re
4import subprocess
5import sys
6from itertools import chain
7from collections.abc import Callable, Mapping, Iterable, Container
9from debian.deb822 import Deb822
11from debputy.commands.debputy_cmd.context import CommandContext
12from debputy.commands.debputy_cmd.output import IOBasedOutputStyling
13from debputy.dh.debhelper_emulation import CannotEmulateExecutableDHConfigFile
14from debputy.dh.dh_assistant import read_dh_addon_sequences
15from debputy.dh_migration.migrators import Migrator, MigrationTarget
16from debputy.dh_migration.migrators_impl import (
17 INTEGRATION_MODE_DH_DEBPUTY,
18 INTEGRATION_MODE_DH_DEBPUTY_RRR,
19)
20from debputy.dh_migration.models import (
21 FeatureMigration,
22 AcceptableMigrationIssues,
23 UnsupportedFeature,
24 ConflictingChange,
25 MigrationRequest,
26)
27from debputy.highlevel_manifest import HighLevelManifest
28from debputy.integration_detection import determine_debputy_integration_mode
29from debputy.manifest_parser.exceptions import ManifestParseException
30from debputy.plugin.api import VirtualPath
31from debputy.plugin.api.spec import DebputyIntegrationMode, INTEGRATION_MODE_FULL
32from debputy.util import _error, _warn, _info, escape_shell, assume_not_none
34SUPPORTED_MIGRATIONS: Mapping[MigrationTarget, frozenset[MigrationTarget]] = {
35 INTEGRATION_MODE_FULL: frozenset([INTEGRATION_MODE_FULL]),
36 INTEGRATION_MODE_DH_DEBPUTY: frozenset(
37 [
38 "dh-single-to-multi-binary",
39 "dh-package-prefixed-config-files",
40 INTEGRATION_MODE_DH_DEBPUTY,
41 INTEGRATION_MODE_FULL,
42 ]
43 ),
44 INTEGRATION_MODE_DH_DEBPUTY_RRR: frozenset(
45 [
46 "dh-single-to-multi-binary",
47 "dh-package-prefixed-config-files",
48 INTEGRATION_MODE_DH_DEBPUTY_RRR,
49 INTEGRATION_MODE_DH_DEBPUTY,
50 INTEGRATION_MODE_FULL,
51 ]
52 ),
53}
56def _print_migration_summary(
57 fo: IOBasedOutputStyling,
58 migrations: list[FeatureMigration],
59 compat: int,
60 min_compat_level: int,
61 required_plugins: set[str],
62 requested_plugins: set[str] | None,
63) -> None:
64 warning_count = 0
66 for migration in migrations:
67 if not migration.anything_to_do:
68 continue
69 underline = "-" * len(migration.tagline)
70 if migration.warnings:
71 if warning_count:
72 _warn("")
73 _warn(f"Summary for migration: {migration.tagline}")
74 if not fo.optimize_for_screen_reader:
75 _warn(f"-----------------------{underline}")
76 warning_count += len(migration.warnings)
77 for warning in migration.warnings:
78 _warn(f" * {warning}")
80 if compat < min_compat_level:
81 if warning_count:
82 _warn("")
83 _warn("Supported debhelper compat check")
84 if not fo.optimize_for_screen_reader:
85 _warn("--------------------------------")
86 warning_count += 1
87 _warn(
88 f"The migration tool assumes debhelper compat {min_compat_level}+ semantics, but this package"
89 f" is using compat {compat}. Consider upgrading the package to compat {min_compat_level}"
90 " first."
91 )
93 if required_plugins:
94 if requested_plugins is None:
95 warning_count += 1
96 needed_plugins = ", ".join(f"debputy-plugin-{n}" for n in required_plugins)
97 if warning_count:
98 _warn("")
99 _warn("Missing debputy plugin check")
100 if not fo.optimize_for_screen_reader:
101 _warn("----------------------------")
102 _warn(
103 f"The migration tool could not read d/control and therefore cannot tell if all the required"
104 f" plugins have been requested. Please ensure that the package Build-Depends on: {needed_plugins}"
105 )
106 else:
107 missing_plugins = required_plugins - requested_plugins
108 if missing_plugins:
109 warning_count += 1
110 needed_plugins = ", ".join(
111 f"debputy-plugin-{n}" for n in missing_plugins
112 )
113 if warning_count:
114 _warn("")
115 _warn("Missing debputy plugin check")
116 if not fo.optimize_for_screen_reader:
117 _warn("----------------------------")
118 _warn(
119 f"The migration tool asserted that the following `debputy` plugins would be required, which"
120 f" are not explicitly requested. Please add the following to Build-Depends: {needed_plugins}"
121 )
123 if warning_count:
124 _warn("")
125 _warn(
126 f"/!\\ Total number of warnings or manual migrations required: {warning_count}"
127 )
130def _dh_compat_level() -> int | None:
131 try:
132 res = subprocess.check_output(
133 ["dh_assistant", "active-compat-level"], stderr=subprocess.DEVNULL
134 )
135 except subprocess.CalledProcessError:
136 compat = None
137 else:
138 try:
139 compat = json.loads(res)["declared-compat-level"]
140 except RuntimeError:
141 compat = None
142 else:
143 if not isinstance(compat, int):
144 compat = None
145 return compat
148def _requested_debputy_plugins(debian_dir: VirtualPath) -> set[str] | None:
149 ctrl_file = debian_dir.get("control")
150 if not ctrl_file:
151 return None
153 dep_regex = re.compile("^([a-z0-9][-+.a-z0-9]+)", re.ASCII)
154 plugins = set()
156 with ctrl_file.open() as fd:
157 ctrl = list(Deb822.iter_paragraphs(fd))
158 source_paragraph = ctrl[0] if ctrl else {}
160 for f in ("Build-Depends", "Build-Depends-Indep", "Build-Depends-Arch"):
161 field = source_paragraph.get(f)
162 if not field:
163 continue
165 for dep_clause in (d.strip() for d in field.split(",")):
166 match = dep_regex.match(dep_clause.strip())
167 if not match:
168 continue
169 dep = match.group(1)
170 if not dep.startswith("debputy-plugin-"):
171 continue
172 plugins.add(dep[15:])
173 return plugins
176def _check_migration_target(
177 context: CommandContext,
178 migration_target: DebputyIntegrationMode | None,
179) -> DebputyIntegrationMode:
180 r = read_dh_addon_sequences(context.debian_dir)
181 if r is not None:
182 bd_sequences, dr_sequences, _ = r
183 all_sequences = bd_sequences | dr_sequences
184 detected_migration_target = determine_debputy_integration_mode(
185 context.source_package().fields,
186 all_sequences,
187 )
188 else:
189 detected_migration_target = None
191 if migration_target is not None and detected_migration_target is not None:
192 supported_migrations = SUPPORTED_MIGRATIONS.get(
193 detected_migration_target,
194 frozenset([detected_migration_target]),
195 )
197 if (
198 migration_target != detected_migration_target
199 and migration_target not in supported_migrations
200 ):
201 _error(
202 f"Cannot migrate apply migration {migration_target} as it conflicts with the current state (which is {detected_migration_target})"
203 )
205 if migration_target is not None:
206 resolved_migration_target = migration_target
207 _info(f'Using "{resolved_migration_target}" as migration (requested)')
208 else:
209 if detected_migration_target is not None:
210 _info(
211 f'Using "{detected_migration_target}" as migration (based on the packaging)'
212 )
213 else:
214 detected_migration_target = INTEGRATION_MODE_DH_DEBPUTY
215 _info(
216 f'Using "{detected_migration_target}" as default migration target. Use --migration-target to choose!'
217 )
218 resolved_migration_target = detected_migration_target
220 return resolved_migration_target
223def _read_git_status_lines(
224 lines: Iterable[bytes],
225) -> Iterable[tuple[str, str, str | None]]:
226 line_iter = iter(lines)
227 while True:
228 try:
229 line = next(line_iter)
230 except StopIteration:
231 return
232 if not line:
233 # We get a final empty line
234 continue
235 if len(line) < 4:
236 _error(
237 f"Internal error: Got out of sync with the `git status --porcelain=v1 -z` output ({len(line)=})"
238 )
239 status_marker = line[0:2].decode("utf-8")
240 if line[2] != 32: # Space
241 _error(
242 "Internal error: Got out of sync with the `git status --porcelain=v1 -z` output (expected status + filename)"
243 )
244 if status_marker.startswith(("R", "C")) or status_marker.endswith(("R", "C")):
245 try:
246 second_filename = next(line_iter).decode("utf-8")
247 except StopIteration:
248 _error(
249 "Internal error: Expected one more line of output from `git status` but it was not there"
250 )
251 else:
252 second_filename = None
253 filename = line[3:].decode("utf-8")
254 yield status_marker, filename, second_filename
257def _git_status() -> tuple[bool, Container[str]]:
258 try:
259 top_level = (
260 subprocess.check_output(
261 ["git", "rev-parse", "--show-toplevel"],
262 stderr=subprocess.DEVNULL,
263 cwd="debian",
264 )
265 .strip()
266 .decode("utf-8")
267 )
268 except (subprocess.CalledProcessError, FileNotFoundError):
269 return False, frozenset()
271 if os.path.realpath(os.getcwd()) != os.path.realpath(top_level):
272 # Patches welcome. The primary problem is to have the status output match the `_check_vcs_clashes` method.
273 _error(
274 f"Unsupported git repo: The `debputy` command only supports `git` when cwd is the git root (git root is: {top_level}). Please use --ignore-vcs to continue."
275 )
277 try:
278 r = subprocess.check_output(
279 ["git", "status", "--porcelain=v1", "-z"],
280 stderr=subprocess.DEVNULL,
281 cwd="debian",
282 )
283 except subprocess.CalledProcessError:
284 _error(
285 "The `git status --porcelain=v1 -z` command returned non-zero (note: the command outputs binary). Please use `--ignore-vcs` to continue anyway."
286 )
287 except FileNotFoundError:
288 _error(
289 "Could not run `git status` and there is a `.git` directory. Please use `--ignore-vcs` to continue anyway."
290 )
292 untracked_files = set()
293 for status_marker, filename, second_filename in _read_git_status_lines(
294 r.split(b"\0")
295 ):
296 if status_marker not in ("??", "!!"):
297 _error(
298 "The current git tree is not clean, please commit or stash any pending changes or use `--ignore-vcs` to continue anyway."
299 )
300 # There is never a second file for `??` or `!!`
301 assert second_filename is None
302 untracked_files.add(filename)
303 return True, untracked_files
306def _check_vcs_clashes(
307 manifest: HighLevelManifest,
308 migrations: list[FeatureMigration],
309 untracked_files: Container[str],
310) -> None:
311 all_affected_files = []
312 if any(m.successful_manifest_changes for m in migrations):
313 all_affected_files.append(manifest.manifest_path)
315 for previous_path, new_path in (
316 p for m in migrations for p in m.rename_paths_on_success
317 ):
318 all_affected_files.append(previous_path)
319 all_affected_files.append(new_path)
321 all_affected_files.extend(p for m in migrations for p in m.remove_paths_on_success)
323 clashing_paths = [p for p in all_affected_files if p in untracked_files]
325 if clashing_paths:
326 print(file=sys.stderr)
327 _warn(
328 "The following untracked or ignored paths would be affected by this migration"
329 )
330 _warn("")
331 for path in clashing_paths:
332 _warn(f" * {path}")
333 _warn("")
334 _warn("Since they are not tracked, the VCS cannot undo the changes to them")
335 _warn("if the migration continued")
336 print(file=sys.stderr)
337 _error(
338 "Please stash, commit or remove the above files before continuing. Alternatively, use --ignore-vcs"
339 )
342def perform_migration(
343 fo: IOBasedOutputStyling,
344 manifest: HighLevelManifest,
345 acceptable_migration_issues: AcceptableMigrationIssues,
346 permit_destructive_changes: bool | None,
347 migration_target: DebputyIntegrationMode,
348 manifest_parser_factory: Callable[[str], HighLevelManifest],
349 migrators: list[Migrator],
350 *,
351 ignore_vcs: bool = False,
352) -> None:
353 migrations = []
354 compat = _dh_compat_level()
355 if compat is None:
356 _error(
357 'Cannot detect declared compat level (try running "dh_assistant active-compat-level")'
358 )
360 debian_dir = manifest.debian_dir
361 mutable_manifest = assume_not_none(manifest.mutable_manifest)
363 if not ignore_vcs:
364 uses_git, untracked_files = _git_status()
365 else:
366 uses_git = False
367 untracked_files = frozenset()
369 if permit_destructive_changes is None and uses_git:
370 permit_destructive_changes = True
372 migration_request = MigrationRequest(
373 debian_dir,
374 manifest,
375 acceptable_migration_issues,
376 migration_target,
377 )
379 try:
380 for migrator in migrators:
381 feature_migration = FeatureMigration(migrator.__name__, fo)
382 migrator(migration_request, feature_migration)
383 migrations.append(feature_migration)
384 except CannotEmulateExecutableDHConfigFile as e:
385 _error(
386 f"Unable to process the executable dh config file {e.config_file().fs_path}: {e.message()}"
387 )
388 except UnsupportedFeature as e:
389 msg = (
390 f"Unable to migrate automatically due to missing features in debputy. The feature is:"
391 f"\n\n * {e.message}"
392 )
393 keys = e.issue_keys
394 if keys:
395 primary_key = keys[0]
396 alt_keys = ""
397 if len(keys) > 1:
398 alt_keys = (
399 f' Alternatively you can also use one of: {", ".join(keys[1:])}. Please note that some'
400 " of these may cover more cases."
401 )
402 msg += (
403 f"\n\nUse --acceptable-migration-issues={primary_key} to convert this into a warning and try again."
404 " However, you should only do that if you believe you can replace the functionality manually"
405 f" or the usage is obsolete / can be removed. {alt_keys}"
406 )
407 _error(msg)
408 except ConflictingChange as e:
409 _error(
410 "The migration tool detected a conflict data being migrated and data already migrated / in the existing"
411 "manifest."
412 f"\n\n * {e.message}"
413 "\n\nPlease review the situation and resolve the conflict manually."
414 )
416 min_compat = max(
417 (m.assumed_compat for m in migrations if m.assumed_compat is not None),
418 default=0,
419 )
421 if compat < min_compat and "min-compat-level" not in acceptable_migration_issues:
422 # The migration summary special-cases the compat mismatch and warns for us.
423 _error(
424 f"The migration tool assumes debhelper compat {min_compat} or later but the package is only on"
425 f" compat {compat}. This may lead to incorrect result."
426 f"\n\nUse --acceptable-migration-issues=min-compat-level to convert this into a warning and"
427 f" try again, if you want to continue regardless."
428 )
430 requested_plugins = _requested_debputy_plugins(debian_dir)
431 required_plugins: set[str] = set()
432 required_plugins.update(
433 chain.from_iterable(
434 m.required_plugins for m in migrations if m.required_plugins
435 )
436 )
438 if uses_git:
439 _check_vcs_clashes(manifest, migrations, untracked_files)
441 _print_migration_summary(
442 fo,
443 migrations,
444 compat,
445 min_compat,
446 required_plugins,
447 requested_plugins,
448 )
449 migration_count = sum((m.performed_changes for m in migrations), 0)
451 if not migration_count:
452 _info(
453 "debputy was not able to find any (supported) migrations that it could perform for you."
454 )
455 return
457 if any(m.successful_manifest_changes for m in migrations):
458 new_manifest_path = manifest.manifest_path + ".new"
460 with open(new_manifest_path, "w") as fd:
461 mutable_manifest.write_to(fd)
463 try:
464 _info("Verifying the new manifest")
465 manifest_parser_factory(new_manifest_path)
466 except ManifestParseException as e:
467 raise AssertionError(
468 "Could not parse the manifest generated from the migrator"
469 ) from e
471 if permit_destructive_changes:
472 if os.path.isfile(manifest.manifest_path):
473 os.rename(manifest.manifest_path, manifest.manifest_path + ".orig")
474 os.rename(new_manifest_path, manifest.manifest_path)
475 if uses_git:
476 subprocess.check_call(["git", "add", manifest.manifest_path])
477 _info(f"Updated manifest {manifest.manifest_path}")
478 else:
479 _info(
480 f'Created draft manifest "{new_manifest_path}" (rename to "{manifest.manifest_path}"'
481 " to activate it)"
482 )
483 else:
484 _info("No manifest changes detected; skipping update of manifest.")
486 removals: int = sum((len(m.remove_paths_on_success) for m in migrations), 0)
487 renames: int = sum((len(m.rename_paths_on_success) for m in migrations), 0)
489 if renames:
490 if permit_destructive_changes:
491 _info("Paths being renamed:")
492 else:
493 _info("Migration *would* rename the following paths:")
494 prefix = "git " if uses_git else ""
495 for previous_path, new_path in (
496 p for m in migrations for p in m.rename_paths_on_success
497 ):
498 _info(f" {prefix}mv {escape_shell(previous_path, new_path)}")
500 if removals:
501 if permit_destructive_changes:
502 _info("Removals:")
503 else:
504 _info("Migration *would* remove the following files:")
505 cmd = "git rm" if uses_git else "rm -f"
506 for path in (p for m in migrations for p in m.remove_paths_on_success):
507 _info(f" {cmd} {escape_shell(path)}")
509 if permit_destructive_changes is None:
510 print()
511 _info(
512 "If you would like to perform the migration, please re-run with --apply-changes."
513 )
514 elif permit_destructive_changes:
515 _apply_renames_and_removals(migrations, uses_git)
517 print()
518 _info("Migrations performed successfully")
519 print()
520 _info(
521 "Remember to validate the resulting binary packages after rebuilding with debputy"
522 )
523 if uses_git:
524 _info(
525 "Use `git commit` to commit the changes or use `git reset` / `git restore` to undo them."
526 )
527 else:
528 print()
529 _info("No migrations performed as requested")
532def _apply_renames_and_removals(
533 migrations: list[FeatureMigration],
534 uses_git: bool,
535) -> None:
536 for previous_path, new_path in (
537 p for m in migrations for p in m.rename_paths_on_success
538 ):
539 if uses_git:
540 subprocess.check_call(["git", "mv", previous_path, new_path])
541 else:
542 os.rename(previous_path, new_path)
544 files_being_removed = [p for m in migrations for p in m.remove_paths_on_success]
545 if uses_git and files_being_removed:
546 command = ["git", "rm"]
547 command.extend(files_being_removed)
548 subprocess.check_call(command)
549 elif files_being_removed:
550 for path in (p for m in migrations for p in m.remove_paths_on_success):
551 os.unlink(path)