Coverage for src/debputy/dh_migration/migration.py: 7%

289 statements  

« prev     ^ index     » next       coverage.py v7.16.1, created at 2026-10-28 20:14 +0000

1import json 

2import os 

3import re 

4import subprocess 

5import sys 

6from itertools import chain 

7from collections.abc import Callable, Mapping, Iterable, Container 

8 

9from debian.deb822 import Deb822 

10 

11from debputy.commands.debputy_cmd.context import CommandContext 

12from debputy.commands.debputy_cmd.output import IOBasedOutputStyling 

13from debputy.dh.debhelper_emulation import CannotEmulateExecutableDHConfigFile 

14from debputy.dh.dh_assistant import read_dh_addon_sequences 

15from debputy.dh_migration.migrators import Migrator, MigrationTarget 

16from debputy.dh_migration.migrators_impl import ( 

17 INTEGRATION_MODE_DH_DEBPUTY, 

18 INTEGRATION_MODE_DH_DEBPUTY_RRR, 

19) 

20from debputy.dh_migration.models import ( 

21 FeatureMigration, 

22 AcceptableMigrationIssues, 

23 UnsupportedFeature, 

24 ConflictingChange, 

25 MigrationRequest, 

26) 

27from debputy.highlevel_manifest import HighLevelManifest 

28from debputy.integration_detection import determine_debputy_integration_mode 

29from debputy.manifest_parser.exceptions import ManifestParseException 

30from debputy.plugin.api import VirtualPath 

31from debputy.plugin.api.spec import DebputyIntegrationMode, INTEGRATION_MODE_FULL 

32from debputy.util import _error, _warn, _info, escape_shell, assume_not_none 

33 

34SUPPORTED_MIGRATIONS: Mapping[MigrationTarget, frozenset[MigrationTarget]] = { 

35 INTEGRATION_MODE_FULL: frozenset([INTEGRATION_MODE_FULL]), 

36 INTEGRATION_MODE_DH_DEBPUTY: frozenset( 

37 [ 

38 "dh-single-to-multi-binary", 

39 "dh-package-prefixed-config-files", 

40 INTEGRATION_MODE_DH_DEBPUTY, 

41 INTEGRATION_MODE_FULL, 

42 ] 

43 ), 

44 INTEGRATION_MODE_DH_DEBPUTY_RRR: frozenset( 

45 [ 

46 "dh-single-to-multi-binary", 

47 "dh-package-prefixed-config-files", 

48 INTEGRATION_MODE_DH_DEBPUTY_RRR, 

49 INTEGRATION_MODE_DH_DEBPUTY, 

50 INTEGRATION_MODE_FULL, 

51 ] 

52 ), 

53} 

54 

55 

56def _print_migration_summary( 

57 fo: IOBasedOutputStyling, 

58 migrations: list[FeatureMigration], 

59 compat: int, 

60 min_compat_level: int, 

61 required_plugins: set[str], 

62 requested_plugins: set[str] | None, 

63) -> None: 

64 warning_count = 0 

65 

66 for migration in migrations: 

67 if not migration.anything_to_do: 

68 continue 

69 underline = "-" * len(migration.tagline) 

70 if migration.warnings: 

71 if warning_count: 

72 _warn("") 

73 _warn(f"Summary for migration: {migration.tagline}") 

74 if not fo.optimize_for_screen_reader: 

75 _warn(f"-----------------------{underline}") 

76 warning_count += len(migration.warnings) 

77 for warning in migration.warnings: 

78 _warn(f" * {warning}") 

79 

80 if compat < min_compat_level: 

81 if warning_count: 

82 _warn("") 

83 _warn("Supported debhelper compat check") 

84 if not fo.optimize_for_screen_reader: 

85 _warn("--------------------------------") 

86 warning_count += 1 

87 _warn( 

88 f"The migration tool assumes debhelper compat {min_compat_level}+ semantics, but this package" 

89 f" is using compat {compat}. Consider upgrading the package to compat {min_compat_level}" 

90 " first." 

91 ) 

92 

93 if required_plugins: 

94 if requested_plugins is None: 

95 warning_count += 1 

96 needed_plugins = ", ".join(f"debputy-plugin-{n}" for n in required_plugins) 

97 if warning_count: 

98 _warn("") 

99 _warn("Missing debputy plugin check") 

100 if not fo.optimize_for_screen_reader: 

101 _warn("----------------------------") 

102 _warn( 

103 f"The migration tool could not read d/control and therefore cannot tell if all the required" 

104 f" plugins have been requested. Please ensure that the package Build-Depends on: {needed_plugins}" 

105 ) 

106 else: 

107 missing_plugins = required_plugins - requested_plugins 

108 if missing_plugins: 

109 warning_count += 1 

110 needed_plugins = ", ".join( 

111 f"debputy-plugin-{n}" for n in missing_plugins 

112 ) 

113 if warning_count: 

114 _warn("") 

115 _warn("Missing debputy plugin check") 

116 if not fo.optimize_for_screen_reader: 

117 _warn("----------------------------") 

118 _warn( 

119 f"The migration tool asserted that the following `debputy` plugins would be required, which" 

120 f" are not explicitly requested. Please add the following to Build-Depends: {needed_plugins}" 

121 ) 

122 

123 if warning_count: 

124 _warn("") 

125 _warn( 

126 f"/!\\ Total number of warnings or manual migrations required: {warning_count}" 

127 ) 

128 

129 

130def _dh_compat_level() -> int | None: 

131 try: 

132 res = subprocess.check_output( 

133 ["dh_assistant", "active-compat-level"], stderr=subprocess.DEVNULL 

134 ) 

135 except subprocess.CalledProcessError: 

136 compat = None 

137 else: 

138 try: 

139 compat = json.loads(res)["declared-compat-level"] 

140 except RuntimeError: 

141 compat = None 

142 else: 

143 if not isinstance(compat, int): 

144 compat = None 

145 return compat 

146 

147 

148def _requested_debputy_plugins(debian_dir: VirtualPath) -> set[str] | None: 

149 ctrl_file = debian_dir.get("control") 

150 if not ctrl_file: 

151 return None 

152 

153 dep_regex = re.compile("^([a-z0-9][-+.a-z0-9]+)", re.ASCII) 

154 plugins = set() 

155 

156 with ctrl_file.open() as fd: 

157 ctrl = list(Deb822.iter_paragraphs(fd)) 

158 source_paragraph = ctrl[0] if ctrl else {} 

159 

160 for f in ("Build-Depends", "Build-Depends-Indep", "Build-Depends-Arch"): 

161 field = source_paragraph.get(f) 

162 if not field: 

163 continue 

164 

165 for dep_clause in (d.strip() for d in field.split(",")): 

166 match = dep_regex.match(dep_clause.strip()) 

167 if not match: 

168 continue 

169 dep = match.group(1) 

170 if not dep.startswith("debputy-plugin-"): 

171 continue 

172 plugins.add(dep[15:]) 

173 return plugins 

174 

175 

176def _check_migration_target( 

177 context: CommandContext, 

178 migration_target: DebputyIntegrationMode | None, 

179) -> DebputyIntegrationMode: 

180 r = read_dh_addon_sequences(context.debian_dir) 

181 if r is not None: 

182 bd_sequences, dr_sequences, _ = r 

183 all_sequences = bd_sequences | dr_sequences 

184 detected_migration_target = determine_debputy_integration_mode( 

185 context.source_package().fields, 

186 all_sequences, 

187 ) 

188 else: 

189 detected_migration_target = None 

190 

191 if migration_target is not None and detected_migration_target is not None: 

192 supported_migrations = SUPPORTED_MIGRATIONS.get( 

193 detected_migration_target, 

194 frozenset([detected_migration_target]), 

195 ) 

196 

197 if ( 

198 migration_target != detected_migration_target 

199 and migration_target not in supported_migrations 

200 ): 

201 _error( 

202 f"Cannot migrate apply migration {migration_target} as it conflicts with the current state (which is {detected_migration_target})" 

203 ) 

204 

205 if migration_target is not None: 

206 resolved_migration_target = migration_target 

207 _info(f'Using "{resolved_migration_target}" as migration (requested)') 

208 else: 

209 if detected_migration_target is not None: 

210 _info( 

211 f'Using "{detected_migration_target}" as migration (based on the packaging)' 

212 ) 

213 else: 

214 detected_migration_target = INTEGRATION_MODE_DH_DEBPUTY 

215 _info( 

216 f'Using "{detected_migration_target}" as default migration target. Use --migration-target to choose!' 

217 ) 

218 resolved_migration_target = detected_migration_target 

219 

220 return resolved_migration_target 

221 

222 

223def _read_git_status_lines( 

224 lines: Iterable[bytes], 

225) -> Iterable[tuple[str, str, str | None]]: 

226 line_iter = iter(lines) 

227 while True: 

228 try: 

229 line = next(line_iter) 

230 except StopIteration: 

231 return 

232 if not line: 

233 # We get a final empty line 

234 continue 

235 if len(line) < 4: 

236 _error( 

237 f"Internal error: Got out of sync with the `git status --porcelain=v1 -z` output ({len(line)=})" 

238 ) 

239 status_marker = line[0:2].decode("utf-8") 

240 if line[2] != 32: # Space 

241 _error( 

242 "Internal error: Got out of sync with the `git status --porcelain=v1 -z` output (expected status + filename)" 

243 ) 

244 if status_marker.startswith(("R", "C")) or status_marker.endswith(("R", "C")): 

245 try: 

246 second_filename = next(line_iter).decode("utf-8") 

247 except StopIteration: 

248 _error( 

249 "Internal error: Expected one more line of output from `git status` but it was not there" 

250 ) 

251 else: 

252 second_filename = None 

253 filename = line[3:].decode("utf-8") 

254 yield status_marker, filename, second_filename 

255 

256 

257def _git_status() -> tuple[bool, Container[str]]: 

258 try: 

259 top_level = ( 

260 subprocess.check_output( 

261 ["git", "rev-parse", "--show-toplevel"], 

262 stderr=subprocess.DEVNULL, 

263 cwd="debian", 

264 ) 

265 .strip() 

266 .decode("utf-8") 

267 ) 

268 except (subprocess.CalledProcessError, FileNotFoundError): 

269 return False, frozenset() 

270 

271 if os.path.realpath(os.getcwd()) != os.path.realpath(top_level): 

272 # Patches welcome. The primary problem is to have the status output match the `_check_vcs_clashes` method. 

273 _error( 

274 f"Unsupported git repo: The `debputy` command only supports `git` when cwd is the git root (git root is: {top_level}). Please use --ignore-vcs to continue." 

275 ) 

276 

277 try: 

278 r = subprocess.check_output( 

279 ["git", "status", "--porcelain=v1", "-z"], 

280 stderr=subprocess.DEVNULL, 

281 cwd="debian", 

282 ) 

283 except subprocess.CalledProcessError: 

284 _error( 

285 "The `git status --porcelain=v1 -z` command returned non-zero (note: the command outputs binary). Please use `--ignore-vcs` to continue anyway." 

286 ) 

287 except FileNotFoundError: 

288 _error( 

289 "Could not run `git status` and there is a `.git` directory. Please use `--ignore-vcs` to continue anyway." 

290 ) 

291 

292 untracked_files = set() 

293 for status_marker, filename, second_filename in _read_git_status_lines( 

294 r.split(b"\0") 

295 ): 

296 if status_marker not in ("??", "!!"): 

297 _error( 

298 "The current git tree is not clean, please commit or stash any pending changes or use `--ignore-vcs` to continue anyway." 

299 ) 

300 # There is never a second file for `??` or `!!` 

301 assert second_filename is None 

302 untracked_files.add(filename) 

303 return True, untracked_files 

304 

305 

306def _check_vcs_clashes( 

307 manifest: HighLevelManifest, 

308 migrations: list[FeatureMigration], 

309 untracked_files: Container[str], 

310) -> None: 

311 all_affected_files = [] 

312 if any(m.successful_manifest_changes for m in migrations): 

313 all_affected_files.append(manifest.manifest_path) 

314 

315 for previous_path, new_path in ( 

316 p for m in migrations for p in m.rename_paths_on_success 

317 ): 

318 all_affected_files.append(previous_path) 

319 all_affected_files.append(new_path) 

320 

321 all_affected_files.extend(p for m in migrations for p in m.remove_paths_on_success) 

322 

323 clashing_paths = [p for p in all_affected_files if p in untracked_files] 

324 

325 if clashing_paths: 

326 print(file=sys.stderr) 

327 _warn( 

328 "The following untracked or ignored paths would be affected by this migration" 

329 ) 

330 _warn("") 

331 for path in clashing_paths: 

332 _warn(f" * {path}") 

333 _warn("") 

334 _warn("Since they are not tracked, the VCS cannot undo the changes to them") 

335 _warn("if the migration continued") 

336 print(file=sys.stderr) 

337 _error( 

338 "Please stash, commit or remove the above files before continuing. Alternatively, use --ignore-vcs" 

339 ) 

340 

341 

342def perform_migration( 

343 fo: IOBasedOutputStyling, 

344 manifest: HighLevelManifest, 

345 acceptable_migration_issues: AcceptableMigrationIssues, 

346 permit_destructive_changes: bool | None, 

347 migration_target: DebputyIntegrationMode, 

348 manifest_parser_factory: Callable[[str], HighLevelManifest], 

349 migrators: list[Migrator], 

350 *, 

351 ignore_vcs: bool = False, 

352) -> None: 

353 migrations = [] 

354 compat = _dh_compat_level() 

355 if compat is None: 

356 _error( 

357 'Cannot detect declared compat level (try running "dh_assistant active-compat-level")' 

358 ) 

359 

360 debian_dir = manifest.debian_dir 

361 mutable_manifest = assume_not_none(manifest.mutable_manifest) 

362 

363 if not ignore_vcs: 

364 uses_git, untracked_files = _git_status() 

365 else: 

366 uses_git = False 

367 untracked_files = frozenset() 

368 

369 if permit_destructive_changes is None and uses_git: 

370 permit_destructive_changes = True 

371 

372 migration_request = MigrationRequest( 

373 debian_dir, 

374 manifest, 

375 acceptable_migration_issues, 

376 migration_target, 

377 ) 

378 

379 try: 

380 for migrator in migrators: 

381 feature_migration = FeatureMigration(migrator.__name__, fo) 

382 migrator(migration_request, feature_migration) 

383 migrations.append(feature_migration) 

384 except CannotEmulateExecutableDHConfigFile as e: 

385 _error( 

386 f"Unable to process the executable dh config file {e.config_file().fs_path}: {e.message()}" 

387 ) 

388 except UnsupportedFeature as e: 

389 msg = ( 

390 f"Unable to migrate automatically due to missing features in debputy. The feature is:" 

391 f"\n\n * {e.message}" 

392 ) 

393 keys = e.issue_keys 

394 if keys: 

395 primary_key = keys[0] 

396 alt_keys = "" 

397 if len(keys) > 1: 

398 alt_keys = ( 

399 f' Alternatively you can also use one of: {", ".join(keys[1:])}. Please note that some' 

400 " of these may cover more cases." 

401 ) 

402 msg += ( 

403 f"\n\nUse --acceptable-migration-issues={primary_key} to convert this into a warning and try again." 

404 " However, you should only do that if you believe you can replace the functionality manually" 

405 f" or the usage is obsolete / can be removed. {alt_keys}" 

406 ) 

407 _error(msg) 

408 except ConflictingChange as e: 

409 _error( 

410 "The migration tool detected a conflict data being migrated and data already migrated / in the existing" 

411 "manifest." 

412 f"\n\n * {e.message}" 

413 "\n\nPlease review the situation and resolve the conflict manually." 

414 ) 

415 

416 min_compat = max( 

417 (m.assumed_compat for m in migrations if m.assumed_compat is not None), 

418 default=0, 

419 ) 

420 

421 if compat < min_compat and "min-compat-level" not in acceptable_migration_issues: 

422 # The migration summary special-cases the compat mismatch and warns for us. 

423 _error( 

424 f"The migration tool assumes debhelper compat {min_compat} or later but the package is only on" 

425 f" compat {compat}. This may lead to incorrect result." 

426 f"\n\nUse --acceptable-migration-issues=min-compat-level to convert this into a warning and" 

427 f" try again, if you want to continue regardless." 

428 ) 

429 

430 requested_plugins = _requested_debputy_plugins(debian_dir) 

431 required_plugins: set[str] = set() 

432 required_plugins.update( 

433 chain.from_iterable( 

434 m.required_plugins for m in migrations if m.required_plugins 

435 ) 

436 ) 

437 

438 if uses_git: 

439 _check_vcs_clashes(manifest, migrations, untracked_files) 

440 

441 _print_migration_summary( 

442 fo, 

443 migrations, 

444 compat, 

445 min_compat, 

446 required_plugins, 

447 requested_plugins, 

448 ) 

449 migration_count = sum((m.performed_changes for m in migrations), 0) 

450 

451 if not migration_count: 

452 _info( 

453 "debputy was not able to find any (supported) migrations that it could perform for you." 

454 ) 

455 return 

456 

457 if any(m.successful_manifest_changes for m in migrations): 

458 new_manifest_path = manifest.manifest_path + ".new" 

459 

460 with open(new_manifest_path, "w") as fd: 

461 mutable_manifest.write_to(fd) 

462 

463 try: 

464 _info("Verifying the new manifest") 

465 manifest_parser_factory(new_manifest_path) 

466 except ManifestParseException as e: 

467 raise AssertionError( 

468 "Could not parse the manifest generated from the migrator" 

469 ) from e 

470 

471 if permit_destructive_changes: 

472 if os.path.isfile(manifest.manifest_path): 

473 os.rename(manifest.manifest_path, manifest.manifest_path + ".orig") 

474 os.rename(new_manifest_path, manifest.manifest_path) 

475 if uses_git: 

476 subprocess.check_call(["git", "add", manifest.manifest_path]) 

477 _info(f"Updated manifest {manifest.manifest_path}") 

478 else: 

479 _info( 

480 f'Created draft manifest "{new_manifest_path}" (rename to "{manifest.manifest_path}"' 

481 " to activate it)" 

482 ) 

483 else: 

484 _info("No manifest changes detected; skipping update of manifest.") 

485 

486 removals: int = sum((len(m.remove_paths_on_success) for m in migrations), 0) 

487 renames: int = sum((len(m.rename_paths_on_success) for m in migrations), 0) 

488 

489 if renames: 

490 if permit_destructive_changes: 

491 _info("Paths being renamed:") 

492 else: 

493 _info("Migration *would* rename the following paths:") 

494 prefix = "git " if uses_git else "" 

495 for previous_path, new_path in ( 

496 p for m in migrations for p in m.rename_paths_on_success 

497 ): 

498 _info(f" {prefix}mv {escape_shell(previous_path, new_path)}") 

499 

500 if removals: 

501 if permit_destructive_changes: 

502 _info("Removals:") 

503 else: 

504 _info("Migration *would* remove the following files:") 

505 cmd = "git rm" if uses_git else "rm -f" 

506 for path in (p for m in migrations for p in m.remove_paths_on_success): 

507 _info(f" {cmd} {escape_shell(path)}") 

508 

509 if permit_destructive_changes is None: 

510 print() 

511 _info( 

512 "If you would like to perform the migration, please re-run with --apply-changes." 

513 ) 

514 elif permit_destructive_changes: 

515 _apply_renames_and_removals(migrations, uses_git) 

516 

517 print() 

518 _info("Migrations performed successfully") 

519 print() 

520 _info( 

521 "Remember to validate the resulting binary packages after rebuilding with debputy" 

522 ) 

523 if uses_git: 

524 _info( 

525 "Use `git commit` to commit the changes or use `git reset` / `git restore` to undo them." 

526 ) 

527 else: 

528 print() 

529 _info("No migrations performed as requested") 

530 

531 

532def _apply_renames_and_removals( 

533 migrations: list[FeatureMigration], 

534 uses_git: bool, 

535) -> None: 

536 for previous_path, new_path in ( 

537 p for m in migrations for p in m.rename_paths_on_success 

538 ): 

539 if uses_git: 

540 subprocess.check_call(["git", "mv", previous_path, new_path]) 

541 else: 

542 os.rename(previous_path, new_path) 

543 

544 files_being_removed = [p for m in migrations for p in m.remove_paths_on_success] 

545 if uses_git and files_being_removed: 

546 command = ["git", "rm"] 

547 command.extend(files_being_removed) 

548 subprocess.check_call(command) 

549 elif files_being_removed: 

550 for path in (p for m in migrations for p in m.remove_paths_on_success): 

551 os.unlink(path)