Coverage for src/debputy/linting/lint_impl.py: 13%
346 statements
« prev ^ index » next coverage.py v7.6.0, created at 2025-01-27 13:59 +0000
1import dataclasses
2import os
3import stat
4import subprocess
5import sys
6import textwrap
7from typing import Optional, List, Union, NoReturn, Mapping
9from debputy.commands.debputy_cmd.context import CommandContext
10from debputy.commands.debputy_cmd.output import _output_styling, OutputStylingBase
11from debputy.filesystem_scan import FSROOverlay
12from debputy.linting.lint_util import (
13 LinterImpl,
14 LintReport,
15 LintStateImpl,
16 FormatterImpl,
17 TermLintReport,
18 LintDiagnosticResultState,
19)
20from debputy.lsp.diagnostics import DiagnosticData
21from debputy.lsp.lsp_debian_changelog import _lint_debian_changelog
22from debputy.lsp.lsp_debian_control import (
23 _lint_debian_control,
24 _reformat_debian_control,
25)
26from debputy.lsp.lsp_debian_copyright import (
27 _lint_debian_copyright,
28 _reformat_debian_copyright,
29)
30from debputy.lsp.lsp_debian_debputy_manifest import _lint_debian_debputy_manifest
31from debputy.lsp.lsp_debian_patches_series import _lint_debian_patches_series
32from debputy.lsp.lsp_debian_rules import _lint_debian_rules
33from debputy.lsp.lsp_debian_tests_control import (
34 _lint_debian_tests_control,
35 _reformat_debian_tests_control,
36)
37from debputy.lsp.lsp_debian_upstream_metadata import _lint_debian_upstream_metadata
38from debputy.lsp.maint_prefs import (
39 MaintainerPreferenceTable,
40 EffectiveFormattingPreference,
41 determine_effective_preference,
42)
43from debputy.lsp.quickfixes import provide_standard_quickfixes_from_diagnostics
44from debputy.lsp.spellchecking import disable_spellchecking
45from debputy.lsp.text_edit import (
46 get_well_formatted_edit,
47 merge_sort_text_edits,
48 apply_text_edits,
49 OverLappingTextEditException,
50)
51from debputy.lsp.vendoring._deb822_repro import Deb822FileElement
52from debputy.lsprotocol.types import (
53 CodeAction,
54 Command,
55 CodeActionParams,
56 CodeActionContext,
57 TextDocumentIdentifier,
58 TextEdit,
59 Position,
60 Range,
61 DiagnosticSeverity,
62 Diagnostic,
63)
64from debputy.packages import SourcePackage, BinaryPackage
65from debputy.plugin.api import VirtualPath
66from debputy.plugin.api.feature_set import PluginProvidedFeatureSet
67from debputy.util import _warn, _error, _info
68from debputy.yaml import MANIFEST_YAML, YAMLError
69from debputy.yaml.compat import CommentedMap
71LINTER_FORMATS = {
72 "debian/changelog": _lint_debian_changelog,
73 "debian/control": _lint_debian_control,
74 "debian/copyright": _lint_debian_copyright,
75 "debian/debputy.manifest": _lint_debian_debputy_manifest,
76 "debian/upstream/metadata": _lint_debian_upstream_metadata,
77 "debian/rules": _lint_debian_rules,
78 "debian/patches/series": _lint_debian_patches_series,
79 "debian/tests/control": _lint_debian_tests_control,
80}
83REFORMAT_FORMATS = {
84 "debian/control": _reformat_debian_control,
85 "debian/copyright": _reformat_debian_copyright,
86 "debian/tests/control": _reformat_debian_tests_control,
87}
90@dataclasses.dataclass(slots=True)
91class LintContext:
92 plugin_feature_set: PluginProvidedFeatureSet
93 maint_preference_table: MaintainerPreferenceTable
94 source_root: Optional[VirtualPath]
95 debian_dir: Optional[VirtualPath]
96 parsed_deb822_file_content: Optional[Deb822FileElement] = None
97 source_package: Optional[SourcePackage] = None
98 binary_packages: Optional[Mapping[str, BinaryPackage]] = None
99 effective_preference: Optional[EffectiveFormattingPreference] = None
100 style_tool: Optional[str] = None
101 unsupported_preference_reason: Optional[str] = None
102 salsa_ci: Optional[CommentedMap] = None
104 def state_for(
105 self,
106 path: str,
107 content: str,
108 lines: List[str],
109 lint_implementation: Optional[LinterImpl],
110 ) -> LintStateImpl:
111 return LintStateImpl(
112 self.plugin_feature_set,
113 self.maint_preference_table,
114 self.source_root,
115 self.debian_dir,
116 path,
117 content,
118 lines,
119 self.source_package,
120 self.binary_packages,
121 self.effective_preference,
122 lint_implementation,
123 )
126def gather_lint_info(context: CommandContext) -> LintContext:
127 source_root = FSROOverlay.create_root_dir(".", ".")
128 debian_dir = source_root.get("debian")
129 if debian_dir is not None and not debian_dir.is_dir:
130 debian_dir = None
131 lint_context = LintContext(
132 context.load_plugins(),
133 MaintainerPreferenceTable.load_preferences(),
134 source_root,
135 debian_dir,
136 )
137 try:
138 with open("debian/control") as fd:
139 deb822_file, source_package, binary_packages = (
140 context.dctrl_parser.parse_source_debian_control(fd, ignore_errors=True)
141 )
142 except FileNotFoundError:
143 source_package = None
144 else:
145 lint_context.parsed_deb822_file_content = deb822_file
146 lint_context.source_package = source_package
147 lint_context.binary_packages = binary_packages
148 salsa_ci_map: Optional[CommentedMap] = None
149 for ci_file in ("debian/salsa-ci.yml", ".gitlab-ci.yml"):
150 try:
151 with open(ci_file) as fd:
152 salsa_ci_map = MANIFEST_YAML.load(fd)
153 if not isinstance(salsa_ci_map, CommentedMap):
154 salsa_ci_map = None
155 break
156 except FileNotFoundError:
157 pass
158 except YAMLError:
159 break
160 if source_package is not None or salsa_ci_map is not None:
161 pref, tool, pref_reason = determine_effective_preference(
162 lint_context.maint_preference_table,
163 source_package,
164 salsa_ci_map,
165 )
166 lint_context.effective_preference = pref
167 lint_context.style_tool = tool
168 lint_context.unsupported_preference_reason = pref_reason
170 return lint_context
173def initialize_lint_report(context: CommandContext) -> LintReport:
174 lint_report_format = context.parsed_args.lint_report_format
175 report_output = context.parsed_args.report_output
177 if lint_report_format == "term":
178 fo = _output_styling(context.parsed_args, sys.stdout)
179 if report_output is not None:
180 _warn("--report-output is redundant for the `term` report")
181 return TermLintReport(fo)
182 if lint_report_format == "junit4-xml":
183 try:
184 import junit_xml
185 except ImportError:
186 _error(
187 "The `junit4-xml` report format requires `python3-junit.xml` to be installed"
188 )
190 from debputy.linting.lint_report_junit import JunitLintReport
192 if report_output is None:
193 report_output = "debputy-lint-junit.xml"
195 return JunitLintReport(report_output)
197 raise AssertionError(f"Missing case for lint_report_format: {lint_report_format}")
200def perform_linting(context: CommandContext) -> None:
201 parsed_args = context.parsed_args
202 if not parsed_args.spellcheck:
203 disable_spellchecking()
204 linter_exit_code = parsed_args.linter_exit_code
205 lint_report = initialize_lint_report(context)
206 lint_context = gather_lint_info(context)
208 for name_stem in LINTER_FORMATS:
209 filename = f"./{name_stem}"
210 if not os.path.isfile(filename):
211 continue
212 perform_linting_of_file(
213 lint_context,
214 filename,
215 name_stem,
216 context.parsed_args.auto_fix,
217 lint_report,
218 )
219 if lint_report.number_of_invalid_diagnostics:
220 _warn(
221 "Some diagnostics did not explicitly set severity. Please report the bug and include the output"
222 )
223 if lint_report.number_of_broken_diagnostics:
224 _error(
225 "Some sub-linters reported issues. Please report the bug and include the output"
226 )
228 if parsed_args.warn_about_check_manifest and os.path.isfile(
229 "debian/debputy.manifest"
230 ):
231 _info("Note: Due to a limitation in the linter, debian/debputy.manifest is")
232 _info("only **partially** checked by this command at the time of writing.")
233 _info("Please use `debputy check-manifest` to fully check the manifest.")
235 lint_report.finish_report()
237 if linter_exit_code:
238 _exit_with_lint_code(lint_report)
241def perform_reformat(
242 context: CommandContext,
243 *,
244 named_style: Optional[str] = None,
245) -> None:
246 parsed_args = context.parsed_args
247 fo = _output_styling(context.parsed_args, sys.stdout)
248 lint_context = gather_lint_info(context)
249 if named_style is not None:
250 style = lint_context.maint_preference_table.named_styles.get(named_style)
251 if style is None:
252 styles = ", ".join(lint_context.maint_preference_table.named_styles)
253 _error(f'There is no style named "{style}". Options include: {styles}')
254 if (
255 lint_context.effective_preference is not None
256 and lint_context.effective_preference != style
257 ):
258 _info(
259 f'Note that the style "{named_style}" does not match the style that `debputy` was configured to use.'
260 )
261 _info("This may be a non-issue (if the configuration is out of date).")
262 lint_context.effective_preference = style
264 if lint_context.effective_preference is None:
265 if lint_context.unsupported_preference_reason is not None:
266 _warn(
267 "While `debputy` could identify a formatting for this package, it does not support it."
268 )
269 _warn(f"{lint_context.unsupported_preference_reason}")
270 if lint_context.style_tool is not None:
271 _info(
272 f"The following tool might be able to apply the style: {lint_context.style_tool}"
273 )
274 if parsed_args.supported_style_required:
275 _error(
276 "Sorry; `debputy` does not support the style. Use --unknown-or-unsupported-style-is-ok to make"
277 " this a non-error (note that `debputy` will not reformat the packaging in this case; just not"
278 " exit with an error code)."
279 )
280 else:
281 print(
282 textwrap.dedent(
283 """\
284 You can enable set a style by doing either of:
286 * You can set `X-Style: black` in the source stanza of `debian/control` to pick
287 `black` as the preferred style for this package.
288 - Note: `black` is an opinionated style that follows the spirit of the `black` code formatter
289 for Python.
290 - If you use `pre-commit`, then there is a formatting hook at
291 https://salsa.debian.org/debian/debputy-pre-commit-hooks
293 * If you use the Debian Salsa CI pipeline, then you can set SALSA_CI_DISABLE_WRAP_AND_SORT (`no`)
294 or SALSA_CI_ENABLE_WRAP_AND_SORT (`yes`) to the relevant value and `debputy` will pick up the
295 configuration from there.
296 - Note: The option must be in `.gitlab-ci.yml` or `debian/salsa-ci.yml` to work. The Salsa CI
297 pipeline will use `wrap-and-sort` while `debputy` uses its own emulation of `wrap-and-sort`
298 (`debputy` also needs to apply the style via `debputy lsp server`).
300 * The `debputy` code also comes with a built-in style database. This may be interesting for
301 packaging teams, so set a default team style that applies to all packages maintained by
302 that packaging team.
303 - Individuals can also add their style, which can useful for ad-hoc packaging teams, where
304 `debputy` will automatically apply a style if *all* co-maintainers agree to it.
306 Note the above list is an ordered list of how `debputy` determines which style to use in case
307 multiple options are available.
308 """
309 )
310 )
311 if parsed_args.supported_style_required:
312 if lint_context.style_tool is not None:
313 _error(
314 "Sorry, `debputy reformat` does not support the packaging style. However, the"
315 f" formatting is supposedly handled by: {lint_context.style_tool}"
316 )
317 _error(
318 "Sorry; `debputy` does not know which style to use for this package. Please either set a"
319 "style or use --unknown-or-unsupported-style-is-ok to make this a non-error"
320 )
321 _info("")
322 _info(
323 "Doing nothing since no supported style could be identified as requested."
324 " See above how to set a style."
325 )
326 _info("Use --supported-style-is-required if this should be an error instead.")
327 sys.exit(0)
329 changes = False
330 auto_fix = context.parsed_args.auto_fix
331 for name_stem in REFORMAT_FORMATS:
332 formatter = REFORMAT_FORMATS.get(name_stem)
333 filename = f"./{name_stem}"
334 if formatter is None or not os.path.isfile(filename):
335 continue
337 reformatted = perform_reformat_of_file(
338 fo,
339 lint_context,
340 filename,
341 formatter,
342 auto_fix,
343 )
344 if reformatted:
345 changes = True
347 if changes and parsed_args.linter_exit_code:
348 sys.exit(2)
351def perform_reformat_of_file(
352 fo: OutputStylingBase,
353 lint_context: LintContext,
354 filename: str,
355 formatter: FormatterImpl,
356 auto_fix: bool,
357) -> bool:
358 with open(filename, "rt", encoding="utf-8") as fd:
359 text = fd.read()
361 lines = text.splitlines(keepends=True)
362 lint_state = lint_context.state_for(
363 filename,
364 text,
365 lines,
366 None,
367 )
368 edits = formatter(lint_state)
369 if not edits:
370 return False
372 try:
373 replacement = apply_text_edits(text, lines, edits)
374 except OverLappingTextEditException:
375 _error(
376 f"The reformater for {filename} produced overlapping edits (which is broken and will not work)"
377 )
379 output_filename = f"{filename}.tmp"
380 with open(output_filename, "wt", encoding="utf-8") as fd:
381 fd.write(replacement)
383 r = subprocess.run(["diff", "-u", filename, output_filename]).returncode
384 if r != 0 and r != 1:
385 _warn(f"diff -u {filename} {output_filename} failed!?")
386 if auto_fix:
387 orig_mode = stat.S_IMODE(os.stat(filename).st_mode)
388 os.chmod(output_filename, orig_mode)
389 os.rename(output_filename, filename)
390 print(
391 fo.colored(
392 f"Reformatted {filename}.",
393 fg="green",
394 style="bold",
395 )
396 )
397 else:
398 os.unlink(output_filename)
400 return True
403def _exit_with_lint_code(lint_report: LintReport) -> NoReturn:
404 diagnostics_count = lint_report.diagnostics_count
405 if (
406 diagnostics_count[DiagnosticSeverity.Error]
407 or diagnostics_count[DiagnosticSeverity.Warning]
408 ):
409 sys.exit(2)
410 sys.exit(0)
413def perform_linting_of_file(
414 lint_context: LintContext,
415 filename: str,
416 file_format: str,
417 auto_fixing_enabled: bool,
418 lint_report: LintReport,
419) -> None:
420 handler = LINTER_FORMATS.get(file_format)
421 if handler is None:
422 return
423 with open(filename, "rt", encoding="utf-8") as fd:
424 text = fd.read()
426 if auto_fixing_enabled:
427 _auto_fix_run(
428 lint_context,
429 filename,
430 text,
431 handler,
432 lint_report,
433 )
434 else:
435 _diagnostics_run(
436 lint_context,
437 filename,
438 text,
439 handler,
440 lint_report,
441 )
444def _overlapping_edit(
445 last_edit_range: Range,
446 last_fix_range: Range,
447) -> bool:
448 last_edit_start_pos = last_edit_range.start
449 last_fix_start_pos = last_fix_range.start
450 if last_edit_start_pos.line < last_fix_start_pos.line:
451 return True
452 if (
453 last_edit_start_pos.line == last_fix_start_pos.character
454 and last_edit_start_pos.character < last_fix_start_pos.character
455 ):
456 return True
458 if last_fix_range.end == last_fix_start_pos:
459 return False
461 if last_fix_range.end < last_edit_start_pos:
462 return False
463 return True
466def _max_range(
467 r1: Range,
468 r2: Range,
469) -> Range:
470 if r2.end > r1.end:
471 return r2
472 if r2.end < r1.end:
473 return r1
474 if r2.start > r1.start:
475 return r2
476 return r1
479_INITIAL_FIX_RANGE = Range(
480 Position(0, 0),
481 Position(0, 0),
482)
485def _is_non_interactive_auto_fix_allowed(diagnostic: Diagnostic) -> bool:
486 diag_data: Optional[DiagnosticData] = diagnostic.data
487 return diag_data is None or diag_data.get("enable_non_interactive_auto_fix", True)
490def _auto_fix_run(
491 lint_context: LintContext,
492 filename: str,
493 text: str,
494 linter: LinterImpl,
495 lint_report: LintReport,
496) -> None:
497 another_round = True
498 unfixed_diagnostics: List[Diagnostic] = []
499 remaining_rounds = 10
500 fixed_count = 0
501 too_many_rounds = False
502 lines = text.splitlines(keepends=True)
503 lint_state = lint_context.state_for(
504 filename,
505 text,
506 lines,
507 linter,
508 )
509 current_issues = lint_state.gather_diagnostics()
510 issue_count_start = len(current_issues) if current_issues else 0
511 while another_round and current_issues:
512 another_round = False
513 last_fix_range = _INITIAL_FIX_RANGE
514 unfixed_diagnostics.clear()
515 edits = []
516 fixed_diagnostics = []
517 for diagnostic in current_issues:
518 if not _is_non_interactive_auto_fix_allowed(diagnostic):
519 unfixed_diagnostics.append(diagnostic)
520 continue
521 actions = provide_standard_quickfixes_from_diagnostics(
522 CodeActionParams(
523 TextDocumentIdentifier(filename),
524 diagnostic.range,
525 CodeActionContext(
526 [diagnostic],
527 ),
528 ),
529 )
530 auto_fixing_edits = resolve_auto_fixer(filename, actions)
532 if not auto_fixing_edits:
533 unfixed_diagnostics.append(diagnostic)
534 continue
536 sorted_edits = merge_sort_text_edits(
537 [get_well_formatted_edit(e) for e in auto_fixing_edits],
538 )
539 last_edit = sorted_edits[-1]
540 last_edit_range = last_edit.range
541 if _overlapping_edit(last_edit_range, last_fix_range):
542 if not another_round:
543 if remaining_rounds > 0:
544 remaining_rounds -= 1
545 print(
546 "Detected overlapping edit; scheduling another edit round."
547 )
548 another_round = True
549 else:
550 _warn(
551 "Too many overlapping edits; stopping after this round (circuit breaker)."
552 )
553 too_many_rounds = True
554 continue
555 edits.extend(sorted_edits)
556 fixed_diagnostics.append(diagnostic)
557 last_fix_range = _max_range(last_fix_range, sorted_edits[-1].range)
559 if another_round and not edits:
560 _error(
561 "Internal error: Detected an overlapping edit and yet had no edits to perform..."
562 )
564 fixed_count += len(fixed_diagnostics)
566 try:
567 text = apply_text_edits(
568 text,
569 lines,
570 edits,
571 )
572 except OverLappingTextEditException:
573 _error(
574 f"Failed to apply edits for f{filename} due to over lapping edits. Please file a bug"
575 f" against `debputy` with a contents of the `debian` directory or a minimal reproducer"
576 )
577 lines = text.splitlines(keepends=True)
579 with lint_report.line_state(lint_state):
580 for diagnostic in fixed_diagnostics:
581 lint_report.report_diagnostic(
582 diagnostic,
583 result_state=LintDiagnosticResultState.FIXED,
584 )
585 lint_state.content = text
586 lint_state.lines = lines
587 lint_state.clear_cache()
588 current_issues = lint_state.gather_diagnostics()
590 if fixed_count:
591 output_filename = f"{filename}.tmp"
592 with open(output_filename, "wt", encoding="utf-8") as fd:
593 fd.write(text)
594 orig_mode = stat.S_IMODE(os.stat(filename).st_mode)
595 os.chmod(output_filename, orig_mode)
596 os.rename(output_filename, filename)
597 lines = text.splitlines(keepends=True)
598 lint_state.content = text
599 lint_state.lines = lines
600 remaining_issues = lint_state.gather_diagnostics() or []
601 else:
602 remaining_issues = current_issues or []
604 with lint_report.line_state(lint_state):
605 for diagnostic in remaining_issues:
606 lint_report.report_diagnostic(diagnostic)
608 if isinstance(lint_report, TermLintReport):
609 # TODO: Not optimal, but will do for now.
610 fo = lint_report.fo
611 print()
612 if fixed_count:
613 remaining_issues_count = len(remaining_issues)
614 print(
615 fo.colored(
616 f"Fixes applied to {filename}: {fixed_count}."
617 f" Number of issues went from {issue_count_start} to {remaining_issues_count}",
618 fg="green",
619 style="bold",
620 )
621 )
622 elif remaining_issues:
623 print(
624 fo.colored(
625 f"None of the issues in {filename} could be fixed automatically. Sorry!",
626 fg="yellow",
627 bg="black",
628 style="bold",
629 )
630 )
631 else:
632 assert not current_issues
633 print(
634 fo.colored(
635 f"No issues detected in {filename}",
636 fg="green",
637 style="bold",
638 )
639 )
640 if too_many_rounds:
641 print(
642 fo.colored(
643 f"Not all fixes for issues in {filename} could be applied due to overlapping edits.",
644 fg="yellow",
645 bg="black",
646 style="bold",
647 )
648 )
649 print(
650 "Running once more may cause more fixes to be applied. However, you may be facing"
651 " pathological performance."
652 )
655def _diagnostics_run(
656 lint_context: LintContext,
657 filename: str,
658 text: str,
659 linter: LinterImpl,
660 lint_report: LintReport,
661) -> None:
662 lines = text.splitlines(keepends=True)
663 lint_state = lint_context.state_for(filename, text, lines, linter)
664 with lint_report.line_state(lint_state):
665 issues = lint_state.gather_diagnostics()
666 for diagnostic in issues:
667 actions = provide_standard_quickfixes_from_diagnostics(
668 CodeActionParams(
669 TextDocumentIdentifier(filename),
670 diagnostic.range,
671 CodeActionContext(
672 [diagnostic],
673 ),
674 ),
675 )
676 auto_fixer = resolve_auto_fixer(filename, actions)
677 has_auto_fixer = bool(auto_fixer) and _is_non_interactive_auto_fix_allowed(
678 diagnostic
679 )
681 result_state = LintDiagnosticResultState.REPORTED
682 if has_auto_fixer:
683 result_state = LintDiagnosticResultState.AUTO_FIXABLE
684 elif has_at_least_lsp_quickfix(actions):
685 result_state = LintDiagnosticResultState.MANUAL_FIXABLE
687 lint_report.report_diagnostic(diagnostic, result_state=result_state)
690def has_at_least_lsp_quickfix(
691 actions: Optional[List[Union[Command, CodeAction]]],
692) -> bool:
693 if actions is None:
694 return False
695 for action in actions:
696 if not isinstance(action, CodeAction):
697 continue
698 if action.edit is None and action.command is None:
699 continue
700 return True
701 return False
704def resolve_auto_fixer(
705 document_ref: str,
706 actions: Optional[List[Union[Command, CodeAction]]],
707) -> Optional[List[TextEdit]]:
708 if actions is None or len(actions) != 1:
709 return None
710 action = actions[0]
711 if not isinstance(action, CodeAction):
712 return None
713 workspace_edit = action.edit
714 if workspace_edit is None or action.command is not None:
715 return None
716 if (
717 not workspace_edit.changes
718 or len(workspace_edit.changes) != 1
719 or document_ref not in workspace_edit.changes
720 ):
721 return None
722 return workspace_edit.changes[document_ref]