Coverage for src/debputy/dh_migration/migration.py: 8%
202 statements
« prev ^ index » next coverage.py v7.6.0, created at 2025-01-27 13:59 +0000
1import json
2import os
3import re
4import subprocess
5from itertools import chain
6from typing import Optional, List, Callable, Set, Container, Mapping, FrozenSet
8from debian.deb822 import Deb822
10from debputy.commands.debputy_cmd.context import CommandContext
11from debputy.commands.debputy_cmd.output import OutputStylingBase
12from debputy.dh.debhelper_emulation import CannotEmulateExecutableDHConfigFile
13from debputy.dh_migration.migrators import MIGRATORS
14from debputy.dh_migration.migrators_impl import (
15 INTEGRATION_MODE_DH_DEBPUTY,
16 INTEGRATION_MODE_DH_DEBPUTY_RRR,
17)
18from debputy.dh.dh_assistant import read_dh_addon_sequences
19from debputy.dh_migration.models import (
20 FeatureMigration,
21 AcceptableMigrationIssues,
22 UnsupportedFeature,
23 ConflictingChange,
24)
25from debputy.highlevel_manifest import HighLevelManifest
26from debputy.integration_detection import determine_debputy_integration_mode
27from debputy.manifest_parser.exceptions import ManifestParseException
28from debputy.plugin.api import VirtualPath
29from debputy.plugin.api.spec import DebputyIntegrationMode, INTEGRATION_MODE_FULL
30from debputy.util import _error, _warn, _info, escape_shell, assume_not_none
32SUPPORTED_MIGRATIONS: Mapping[
33 DebputyIntegrationMode, FrozenSet[DebputyIntegrationMode]
34] = {
35 INTEGRATION_MODE_FULL: frozenset([INTEGRATION_MODE_FULL]),
36 INTEGRATION_MODE_DH_DEBPUTY: frozenset(
37 [INTEGRATION_MODE_DH_DEBPUTY, INTEGRATION_MODE_FULL]
38 ),
39 INTEGRATION_MODE_DH_DEBPUTY_RRR: frozenset(
40 [
41 INTEGRATION_MODE_DH_DEBPUTY_RRR,
42 INTEGRATION_MODE_DH_DEBPUTY,
43 INTEGRATION_MODE_FULL,
44 ]
45 ),
46}
49def _print_migration_summary(
50 fo: OutputStylingBase,
51 migrations: List[FeatureMigration],
52 compat: int,
53 min_compat_level: int,
54 required_plugins: Set[str],
55 requested_plugins: Optional[Set[str]],
56) -> None:
57 warning_count = 0
59 for migration in migrations:
60 if not migration.anything_to_do:
61 continue
62 underline = "-" * len(migration.tagline)
63 if migration.warnings:
64 if warning_count:
65 _warn("")
66 _warn(f"Summary for migration: {migration.tagline}")
67 if not fo.optimize_for_screen_reader:
68 _warn(f"-----------------------{underline}")
69 warning_count += len(migration.warnings)
70 for warning in migration.warnings:
71 _warn(f" * {warning}")
73 if compat < min_compat_level:
74 if warning_count:
75 _warn("")
76 _warn("Supported debhelper compat check")
77 if not fo.optimize_for_screen_reader:
78 _warn("--------------------------------")
79 warning_count += 1
80 _warn(
81 f"The migration tool assumes debhelper compat {min_compat_level}+ semantics, but this package"
82 f" is using compat {compat}. Consider upgrading the package to compat {min_compat_level}"
83 " first."
84 )
86 if required_plugins:
87 if requested_plugins is None:
88 warning_count += 1
89 needed_plugins = ", ".join(f"debputy-plugin-{n}" for n in required_plugins)
90 if warning_count:
91 _warn("")
92 _warn("Missing debputy plugin check")
93 if not fo.optimize_for_screen_reader:
94 _warn("----------------------------")
95 _warn(
96 f"The migration tool could not read d/control and therefore cannot tell if all the required"
97 f" plugins have been requested. Please ensure that the package Build-Depends on: {needed_plugins}"
98 )
99 else:
100 missing_plugins = required_plugins - requested_plugins
101 if missing_plugins:
102 warning_count += 1
103 needed_plugins = ", ".join(
104 f"debputy-plugin-{n}" for n in missing_plugins
105 )
106 if warning_count:
107 _warn("")
108 _warn("Missing debputy plugin check")
109 if not fo.optimize_for_screen_reader:
110 _warn("----------------------------")
111 _warn(
112 f"The migration tool asserted that the following `debputy` plugins would be required, which"
113 f" are not explicitly requested. Please add the following to Build-Depends: {needed_plugins}"
114 )
116 if warning_count:
117 _warn("")
118 _warn(
119 f"/!\\ Total number of warnings or manual migrations required: {warning_count}"
120 )
123def _dh_compat_level() -> Optional[int]:
124 try:
125 res = subprocess.check_output(
126 ["dh_assistant", "active-compat-level"], stderr=subprocess.DEVNULL
127 )
128 except subprocess.CalledProcessError:
129 compat = None
130 else:
131 try:
132 compat = json.loads(res)["declared-compat-level"]
133 except RuntimeError:
134 compat = None
135 else:
136 if not isinstance(compat, int):
137 compat = None
138 return compat
141def _requested_debputy_plugins(debian_dir: VirtualPath) -> Optional[Set[str]]:
142 ctrl_file = debian_dir.get("control")
143 if not ctrl_file:
144 return None
146 dep_regex = re.compile("^([a-z0-9][-+.a-z0-9]+)", re.ASCII)
147 plugins = set()
149 with ctrl_file.open() as fd:
150 ctrl = list(Deb822.iter_paragraphs(fd))
151 source_paragraph = ctrl[0] if ctrl else {}
153 for f in ("Build-Depends", "Build-Depends-Indep", "Build-Depends-Arch"):
154 field = source_paragraph.get(f)
155 if not field:
156 continue
158 for dep_clause in (d.strip() for d in field.split(",")):
159 match = dep_regex.match(dep_clause.strip())
160 if not match:
161 continue
162 dep = match.group(1)
163 if not dep.startswith("debputy-plugin-"):
164 continue
165 plugins.add(dep[15:])
166 return plugins
169def _check_migration_target(
170 context: CommandContext,
171 migration_target: Optional[DebputyIntegrationMode],
172) -> DebputyIntegrationMode:
173 r = read_dh_addon_sequences(context.debian_dir)
174 if r is not None:
175 bd_sequences, dr_sequences, _ = r
176 all_sequences = bd_sequences | dr_sequences
177 detected_migration_target = determine_debputy_integration_mode(
178 context.source_package().fields,
179 all_sequences,
180 )
181 else:
182 detected_migration_target = None
184 if migration_target is not None and detected_migration_target is not None:
185 supported_migrations = SUPPORTED_MIGRATIONS.get(
186 detected_migration_target,
187 frozenset([detected_migration_target]),
188 )
190 if (
191 migration_target != detected_migration_target
192 and migration_target not in supported_migrations
193 ):
194 _error(
195 f"Cannot migrate from {detected_migration_target} to {migration_target}"
196 )
198 if migration_target is not None:
199 resolved_migration_target = migration_target
200 _info(f'Using "{resolved_migration_target}" as migration target as requested')
201 else:
202 if detected_migration_target is not None:
203 _info(
204 f'Using "{detected_migration_target}" as migration target based on the packaging'
205 )
206 else:
207 detected_migration_target = INTEGRATION_MODE_DH_DEBPUTY
208 _info(
209 f'Using "{detected_migration_target}" as default migration target. Use --migration-target to choose!'
210 )
211 resolved_migration_target = detected_migration_target
213 return resolved_migration_target
216def migrate_from_dh(
217 fo: OutputStylingBase,
218 manifest: HighLevelManifest,
219 acceptable_migration_issues: AcceptableMigrationIssues,
220 permit_destructive_changes: Optional[bool],
221 migration_target: DebputyIntegrationMode,
222 manifest_parser_factory: Callable[[str], HighLevelManifest],
223) -> None:
224 migrations = []
225 compat = _dh_compat_level()
226 if compat is None:
227 _error(
228 'Cannot detect declared compat level (try running "dh_assistant active-compat-level")'
229 )
231 debian_dir = manifest.debian_dir
232 mutable_manifest = assume_not_none(manifest.mutable_manifest)
234 try:
235 for migrator in MIGRATORS[migration_target]:
236 feature_migration = FeatureMigration(migrator.__name__, fo)
237 migrator(
238 debian_dir,
239 manifest,
240 acceptable_migration_issues,
241 feature_migration,
242 migration_target,
243 )
244 migrations.append(feature_migration)
245 except CannotEmulateExecutableDHConfigFile as e:
246 _error(
247 f"Unable to process the executable dh config file {e.config_file().fs_path}: {e.message()}"
248 )
249 except UnsupportedFeature as e:
250 msg = (
251 f"Unable to migrate automatically due to missing features in debputy. The feature is:"
252 f"\n\n * {e.message}"
253 )
254 keys = e.issue_keys
255 if keys:
256 primary_key = keys[0]
257 alt_keys = ""
258 if len(keys) > 1:
259 alt_keys = (
260 f' Alternatively you can also use one of: {", ".join(keys[1:])}. Please note that some'
261 " of these may cover more cases."
262 )
263 msg += (
264 f"\n\nUse --acceptable-migration-issues={primary_key} to convert this into a warning and try again."
265 " However, you should only do that if you believe you can replace the functionality manually"
266 f" or the usage is obsolete / can be removed. {alt_keys}"
267 )
268 _error(msg)
269 except ConflictingChange as e:
270 _error(
271 "The migration tool detected a conflict data being migrated and data already migrated / in the existing"
272 "manifest."
273 f"\n\n * {e.message}"
274 "\n\nPlease review the situation and resolve the conflict manually."
275 )
277 # We start on compat 12 for arch:any due to the new dh_makeshlibs and dh_installinit default
278 min_compat = 12
279 min_compat = max(
280 (m.assumed_compat for m in migrations if m.assumed_compat is not None),
281 default=min_compat,
282 )
284 if compat < min_compat and "min-compat-level" not in acceptable_migration_issues:
285 # The migration summary special-cases the compat mismatch and warns for us.
286 _error(
287 f"The migration tool assumes debhelper compat {min_compat} or later but the package is only on"
288 f" compat {compat}. This may lead to incorrect result."
289 f"\n\nUse --acceptable-migration-issues=min-compat-level to convert this into a warning and"
290 f" try again, if you want to continue regardless."
291 )
293 requested_plugins = _requested_debputy_plugins(debian_dir)
294 required_plugins: Set[str] = set()
295 required_plugins.update(
296 chain.from_iterable(
297 m.required_plugins for m in migrations if m.required_plugins
298 )
299 )
301 _print_migration_summary(
302 fo,
303 migrations,
304 compat,
305 min_compat,
306 required_plugins,
307 requested_plugins,
308 )
309 migration_count = sum((m.performed_changes for m in migrations), 0)
311 if not migration_count:
312 _info(
313 "debputy was not able to find any (supported) migrations that it could perform for you."
314 )
315 return
317 if any(m.successful_manifest_changes for m in migrations):
318 new_manifest_path = manifest.manifest_path + ".new"
320 with open(new_manifest_path, "w") as fd:
321 mutable_manifest.write_to(fd)
323 try:
324 _info("Verifying the generating manifest")
325 manifest_parser_factory(new_manifest_path)
326 except ManifestParseException as e:
327 raise AssertionError(
328 "Could not parse the manifest generated from the migrator"
329 ) from e
331 if permit_destructive_changes:
332 if os.path.isfile(manifest.manifest_path):
333 os.rename(manifest.manifest_path, manifest.manifest_path + ".orig")
334 os.rename(new_manifest_path, manifest.manifest_path)
335 _info(f"Updated manifest {manifest.manifest_path}")
336 else:
337 _info(
338 f'Created draft manifest "{new_manifest_path}" (rename to "{manifest.manifest_path}"'
339 " to activate it)"
340 )
341 else:
342 _info("No manifest changes detected; skipping update of manifest.")
344 removals: int = sum((len(m.remove_paths_on_success) for m in migrations), 0)
345 renames: int = sum((len(m.rename_paths_on_success) for m in migrations), 0)
347 if renames:
348 if permit_destructive_changes:
349 _info("Paths being renamed:")
350 else:
351 _info("Migration *would* rename the following paths:")
352 for previous_path, new_path in (
353 p for m in migrations for p in m.rename_paths_on_success
354 ):
355 _info(f" mv {escape_shell(previous_path, new_path)}")
357 if removals:
358 if permit_destructive_changes:
359 _info("Removals:")
360 else:
361 _info("Migration *would* remove the following files:")
362 for path in (p for m in migrations for p in m.remove_paths_on_success):
363 _info(f" rm -f {escape_shell(path)}")
365 if permit_destructive_changes is None:
366 print()
367 _info(
368 "If you would like to perform the migration, please re-run with --apply-changes."
369 )
370 elif permit_destructive_changes:
371 for previous_path, new_path in (
372 p for m in migrations for p in m.rename_paths_on_success
373 ):
374 os.rename(previous_path, new_path)
375 for path in (p for m in migrations for p in m.remove_paths_on_success):
376 os.unlink(path)
378 print()
379 _info("Migrations performed successfully")
380 print()
381 _info(
382 "Remember to validate the resulting binary packages after rebuilding with debputy"
383 )
384 else:
385 print()
386 _info("No migrations performed as requested")