From ca6f387d8a0cb981002264d946ffd631af0dff46 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:08:54 -0700 Subject: [PATCH 01/10] Move linemap module into resyntax/grimoire Co-Authored-By: Claude Fable 5 --- {private => grimoire}/linemap.rkt | 0 private/analysis.rkt | 2 +- private/line-replacement.rkt | 2 +- private/refactoring-result.rkt | 2 +- private/syntax-replacement.rkt | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename {private => grimoire}/linemap.rkt (100%) diff --git a/private/linemap.rkt b/grimoire/linemap.rkt similarity index 100% rename from private/linemap.rkt rename to grimoire/linemap.rkt diff --git a/private/analysis.rkt b/private/analysis.rkt index e723a5b..9e21409 100644 --- a/private/analysis.rkt +++ b/private/analysis.rkt @@ -42,7 +42,7 @@ resyntax/default-recommendations/analyzers/ignored-result-values resyntax/default-recommendations/analyzers/variable-mutability resyntax/private/analyzer - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/private/logger resyntax/grimoire/source resyntax/private/string-indent diff --git a/private/line-replacement.rkt b/private/line-replacement.rkt index 639f74f..5e52a29 100644 --- a/private/line-replacement.rkt +++ b/private/line-replacement.rkt @@ -27,7 +27,7 @@ rebellion/streaming/reducer rebellion/streaming/transducer rebellion/type/record - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/grimoire/string-replacement) diff --git a/private/refactoring-result.rkt b/private/refactoring-result.rkt index 7b8aa47..734591f 100644 --- a/private/refactoring-result.rkt +++ b/private/refactoring-result.rkt @@ -50,7 +50,7 @@ resyntax/private/code-snippet resyntax/private/commit resyntax/private/line-replacement - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/private/logger resyntax/grimoire/source resyntax/grimoire/string-replacement diff --git a/private/syntax-replacement.rkt b/private/syntax-replacement.rkt index 0789a78..8dd87c1 100644 --- a/private/syntax-replacement.rkt +++ b/private/syntax-replacement.rkt @@ -42,7 +42,7 @@ rebellion/collection/range-set rebellion/private/static-name rebellion/type/record - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/private/logger resyntax/grimoire/source resyntax/private/string-indent From 172371834940454cb51fa0d871e6c66f55b8633f Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:08:54 -0700 Subject: [PATCH 02/10] Make string-linemap produce immutable line strings The linemap-lines contract has always promised immutable strings, but the implementation built lines with substring, which returns mutable ones. Any use of linemap-lines through the contract boundary raised a broke-its-own- contract error. It has no callers outside the module, so nothing noticed. Co-Authored-By: Claude Fable 5 --- grimoire/linemap.rkt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grimoire/linemap.rkt b/grimoire/linemap.rkt index f499ce6..536c43e 100644 --- a/grimoire/linemap.rkt +++ b/grimoire/linemap.rkt @@ -45,12 +45,12 @@ (let loop ([line-number 1] [line-start-index 0] [index 0]) (cond [(= index char-count) - (define last-line (substring str line-start-index index)) + (define last-line (string->immutable-string (substring str line-start-index index))) (vector-builder-add lines last-line) (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number) (vector-builder-add start-positions-by-line-number (add1 line-start-index))] [(equal? (string-ref str index) #\newline) - (define next-line (substring str line-start-index index)) + (define next-line (string->immutable-string (substring str line-start-index index))) (vector-builder-add lines next-line) (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number) (vector-builder-add start-positions-by-line-number (add1 line-start-index)) From 2a3855d97a2e75471e944c0b955f10a81651f8dd Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:08:54 -0700 Subject: [PATCH 03/10] Draft API reference for linemaps Co-Authored-By: Claude Fable 5 --- grimoire.scrbl | 1 + grimoire/linemap.scrbl | 96 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 grimoire/linemap.scrbl diff --git a/grimoire.scrbl b/grimoire.scrbl index 02bf46d..5f058ee 100644 --- a/grimoire.scrbl +++ b/grimoire.scrbl @@ -18,3 +18,4 @@ programmatically on anything found here. @include-section[(lib "resyntax/grimoire/syntax-property-bundle.scrbl")] @include-section[(lib "resyntax/grimoire/expansion-analyzers.scrbl")] @include-section[(lib "resyntax/grimoire/string-replacement.scrbl")] +@include-section[(lib "resyntax/grimoire/linemap.scrbl")] diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl new file mode 100644 index 0000000..6c64fad --- /dev/null +++ b/grimoire/linemap.scrbl @@ -0,0 +1,96 @@ +#lang scribble/manual + + +@(require (for-label racket/base + racket/contract/base + rebellion/base/comparator + rebellion/base/range + resyntax/grimoire/linemap)) + + +@title[#:tag "linemap"]{Linemaps} +@defmodule[resyntax/grimoire/linemap] + +A @deftech{linemap} is a precomputed index of a string's line structure that supports converting +between character positions and line numbers. Resyntax straddles two views of source code: the +line-oriented view, used by @tech{source groups} and the command-line interface (see +@secref["cli"]) to select which code to analyze, and the character-oriented view, used by +@tech{string replacements} and syntax object source locations. Linemaps are the bridge between them --- Resyntax uses linemaps to compute which +lines a refactoring suggestion modifies, to render string replacements as line-based diffs, and to +restrict analysis to the requested lines. + +@bold{All positions and line numbers in a linemap are one-based}, matching the conventions of +@racket[syntax-position] and @racket[syntax-line]: the first character of a string is at position +@racket[1], on line @racket[1]. Beware that this is the @emph{opposite} of the convention used by +@tech{string replacements}, which address characters with @emph{zero-based} indices. Converting +between the two worlds requires adding or subtracting one, as discussed in +@secref["string-replacement"]. + +The lines of a string are the segments separated by newline characters. The terminating newline is +not part of a line's contents, but positions of newline characters belong to the lines they +terminate. A string that ends with a newline has a final empty line after it, and the empty string +consists of a single empty line. + + +@defproc[(linemap? [v any/c]) boolean?]{ + A predicate that recognizes @tech{linemaps}.} + + +@defproc[(string-linemap [str string?]) linemap?]{ + Constructs a @tech{linemap} of the lines in @racket[str]. Only @racket[#\newline] characters are + treated as line separators.} + + +@defproc[(linemap-lines [map linemap?]) + (vectorof (and/c string? immutable?) #:immutable #true)]{ + Returns a vector of the lines of the string that @racket[map] was built from, without their + terminating newlines. Note that because line numbers are one-based and vector indices are + zero-based, line @racket[_n] is at index @racket[_n] minus one in the returned vector.} + + +@defproc[(linemap-position-to-line [map linemap?] [position exact-positive-integer?]) + exact-positive-integer?]{ + Returns the line number of the line containing @racket[position]. The position of a newline + character is considered contained by the line that the newline terminates. Positions beyond the + end of the string do not raise an error; they are all treated as belonging to the last line.} + + +@defproc[(linemap-line-start-position [map linemap?] [line exact-positive-integer?]) + exact-positive-integer?]{ + Returns the position of the first character of @racket[line]. If the string ends with a newline, + the start position of its final, empty line is one past the end of the string. Raises an error if + @racket[line] is greater than the number of lines in the string.} + + +@defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-positive-integer?]) + exact-positive-integer?]{ + Returns the position of the first character of the line containing @racket[position]. Equivalent + to composing @racket[linemap-position-to-line] with @racket[linemap-line-start-position].} + + +@defproc[(linemap-position-to-end-of-line [map linemap?] [position exact-positive-integer?]) + exact-positive-integer?]{ + Returns the position just past the last character of the contents of the line containing + @racket[position] --- that is, the position of the line's terminating newline, or one past the + end of the string if the line is the last one.} + + +@defproc[(syntax-start-line-position [stx syntax?] [#:linemap map linemap?]) + exact-positive-integer?]{ + Returns the position of the start of the line on which @racket[stx] begins. The + @tech[#:doc '(lib "scribblings/reference/reference.scrbl")]{source location} of @racket[stx] must + refer to positions within the string that @racket[map] was built from.} + + +@defproc[(syntax-end-line-position [stx syntax?] [#:linemap map linemap?]) + exact-positive-integer?]{ + Returns the position of the end of the line on which @racket[stx] ends, in the same sense as + @racket[linemap-position-to-end-of-line]. The source location of @racket[stx] must refer to + positions within the string that @racket[map] was built from.} + + +@defproc[(syntax-line-range [stx syntax?] [#:linemap map linemap?]) range?]{ + Returns a closed range (with @racket[natural<=>] as its comparator) containing the line numbers + of every line that @racket[stx] spans, from the line on which it begins to the line on which it + ends. The source location of @racket[stx] must refer to positions within the string that + @racket[map] was built from.} From 39d126d2afb5e9ca188635aa5cbda70bc62ae009 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:18:19 -0700 Subject: [PATCH 04/10] Delete unused linemap bindings Co-Authored-By: Claude Fable 5 --- grimoire/linemap.rkt | 12 ------------ grimoire/linemap.scrbl | 37 +++++-------------------------------- 2 files changed, 5 insertions(+), 44 deletions(-) diff --git a/grimoire/linemap.rkt b/grimoire/linemap.rkt index 536c43e..4b05243 100644 --- a/grimoire/linemap.rkt +++ b/grimoire/linemap.rkt @@ -8,13 +8,9 @@ (contract-out [string-linemap (-> string? linemap?)] [linemap? (-> any/c boolean?)] - [linemap-lines (-> linemap? (vectorof (and/c string? immutable?) #:immutable #true #:flat? #true))] [linemap-position-to-line (-> linemap? exact-positive-integer? exact-positive-integer?)] - [linemap-line-start-position (-> linemap? exact-positive-integer? exact-positive-integer?)] [linemap-position-to-start-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)] [linemap-position-to-end-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)] - [syntax-start-line-position (-> syntax? #:linemap linemap? exact-positive-integer?)] - [syntax-end-line-position (-> syntax? #:linemap linemap? exact-positive-integer?)] [syntax-line-range (-> syntax? #:linemap linemap? range?)])) @@ -88,14 +84,6 @@ (linemap-line-end-position map (linemap-position-to-line map position))) -(define (syntax-start-line-position stx #:linemap map) - (linemap-position-to-start-of-line map (syntax-position stx))) - - -(define (syntax-end-line-position stx #:linemap map) - (linemap-position-to-end-of-line map (+ (syntax-position stx) (syntax-span stx)))) - - (define (syntax-line-range stx #:linemap map) (define first-line (syntax-line stx)) (define last-line (linemap-position-to-line map (+ (syntax-position stx) (syntax-span stx)))) diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl index 6c64fad..de76b97 100644 --- a/grimoire/linemap.scrbl +++ b/grimoire/linemap.scrbl @@ -41,13 +41,6 @@ consists of a single empty line. treated as line separators.} -@defproc[(linemap-lines [map linemap?]) - (vectorof (and/c string? immutable?) #:immutable #true)]{ - Returns a vector of the lines of the string that @racket[map] was built from, without their - terminating newlines. Note that because line numbers are one-based and vector indices are - zero-based, line @racket[_n] is at index @racket[_n] minus one in the returned vector.} - - @defproc[(linemap-position-to-line [map linemap?] [position exact-positive-integer?]) exact-positive-integer?]{ Returns the line number of the line containing @racket[position]. The position of a newline @@ -55,17 +48,11 @@ consists of a single empty line. end of the string do not raise an error; they are all treated as belonging to the last line.} -@defproc[(linemap-line-start-position [map linemap?] [line exact-positive-integer?]) - exact-positive-integer?]{ - Returns the position of the first character of @racket[line]. If the string ends with a newline, - the start position of its final, empty line is one past the end of the string. Raises an error if - @racket[line] is greater than the number of lines in the string.} - - @defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-positive-integer?]) exact-positive-integer?]{ - Returns the position of the first character of the line containing @racket[position]. Equivalent - to composing @racket[linemap-position-to-line] with @racket[linemap-line-start-position].} + Returns the position of the first character of the line containing @racket[position]. If the + string ends with a newline and @racket[position] is on the final, empty line after it, that + line's start position is one past the end of the string.} @defproc[(linemap-position-to-end-of-line [map linemap?] [position exact-positive-integer?]) @@ -75,22 +62,8 @@ consists of a single empty line. end of the string if the line is the last one.} -@defproc[(syntax-start-line-position [stx syntax?] [#:linemap map linemap?]) - exact-positive-integer?]{ - Returns the position of the start of the line on which @racket[stx] begins. The - @tech[#:doc '(lib "scribblings/reference/reference.scrbl")]{source location} of @racket[stx] must - refer to positions within the string that @racket[map] was built from.} - - -@defproc[(syntax-end-line-position [stx syntax?] [#:linemap map linemap?]) - exact-positive-integer?]{ - Returns the position of the end of the line on which @racket[stx] ends, in the same sense as - @racket[linemap-position-to-end-of-line]. The source location of @racket[stx] must refer to - positions within the string that @racket[map] was built from.} - - @defproc[(syntax-line-range [stx syntax?] [#:linemap map linemap?]) range?]{ Returns a closed range (with @racket[natural<=>] as its comparator) containing the line numbers of every line that @racket[stx] spans, from the line on which it begins to the line on which it - ends. The source location of @racket[stx] must refer to positions within the string that - @racket[map] was built from.} + ends. The @tech[#:doc '(lib "scribblings/reference/reference.scrbl")]{source location} of + @racket[stx] must refer to positions within the string that @racket[map] was built from.} From 8ef47bd3de6a69fcfcfb57246c57baf4ca7bc450 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:35:35 -0700 Subject: [PATCH 05/10] Make linemap positions zero-indexed Linemaps are built from strings, and Racket string APIs (and Resyntax's string replacements) index characters from zero. One-indexed positions are a file port and IDE convention that doesn't belong here. Line numbers stay one-indexed, since they're primarily useful in UI contexts. Every caller was converting with add1/sub1 at the boundary; those shims are now gone, and syntax-line-range converts from one-indexed syntax positions itself. Co-Authored-By: Claude Fable 5 --- grimoire/linemap.rkt | 139 +++++++++++++++++---------------- grimoire/linemap.scrbl | 31 ++++---- private/line-replacement.rkt | 11 +-- private/refactoring-result.rkt | 8 +- private/syntax-replacement.rkt | 7 +- 5 files changed, 100 insertions(+), 96 deletions(-) diff --git a/grimoire/linemap.rkt b/grimoire/linemap.rkt index 4b05243..c88634b 100644 --- a/grimoire/linemap.rkt +++ b/grimoire/linemap.rkt @@ -8,9 +8,11 @@ (contract-out [string-linemap (-> string? linemap?)] [linemap? (-> any/c boolean?)] - [linemap-position-to-line (-> linemap? exact-positive-integer? exact-positive-integer?)] - [linemap-position-to-start-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)] - [linemap-position-to-end-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)] + [linemap-position-to-line (-> linemap? exact-nonnegative-integer? exact-positive-integer?)] + [linemap-position-to-start-of-line + (-> linemap? exact-nonnegative-integer? exact-nonnegative-integer?)] + [linemap-position-to-end-of-line + (-> linemap? exact-nonnegative-integer? exact-nonnegative-integer?)] [syntax-line-range (-> syntax? #:linemap linemap? range?)])) @@ -43,13 +45,13 @@ [(= index char-count) (define last-line (string->immutable-string (substring str line-start-index index))) (vector-builder-add lines last-line) - (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number) - (vector-builder-add start-positions-by-line-number (add1 line-start-index))] + (sorted-map-builder-put line-numbers-by-start-position line-start-index line-number) + (vector-builder-add start-positions-by-line-number line-start-index)] [(equal? (string-ref str index) #\newline) (define next-line (string->immutable-string (substring str line-start-index index))) (vector-builder-add lines next-line) - (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number) - (vector-builder-add start-positions-by-line-number (add1 line-start-index)) + (sorted-map-builder-put line-numbers-by-start-position line-start-index line-number) + (vector-builder-add start-positions-by-line-number line-start-index) (define next-index (add1 index)) (loop (add1 line-number) next-index next-index)] [else (loop line-number line-start-index (add1 index))])) @@ -86,7 +88,9 @@ (define (syntax-line-range stx #:linemap map) (define first-line (syntax-line stx)) - (define last-line (linemap-position-to-line map (+ (syntax-position stx) (syntax-span stx)))) + ;; Syntax object positions are one-indexed, unlike linemap positions. + (define last-line + (linemap-position-to-line map (+ (sub1 (syntax-position stx)) (syntax-span stx)))) (unless (<= first-line last-line) (raise-arguments-error 'syntax-line-range "syntax object's last line number is before its first line number" @@ -105,118 +109,119 @@ (define (nat-map . args) (apply sorted-map #:key-comparator natural<=> args)) - (check-equal? (string-linemap "") (linemap #("") (nat-map 1 1) #(1))) - (check-equal? (string-linemap "a") (linemap #("a") (nat-map 1 1) #(1))) - (check-equal? (string-linemap "λ") (linemap #("λ") (nat-map 1 1) #(1))) - (check-equal? (string-linemap "a\n") (linemap #("a" "") (nat-map 1 1 3 2) #(1 3))) - (check-equal? (string-linemap "λ\n") (linemap #("λ" "") (nat-map 1 1 3 2) #(1 3))) - (check-equal? (string-linemap "aaa\n") (linemap #("aaa" "") (nat-map 1 1 5 2) #(1 5))) - (check-equal? (string-linemap "aaa\nbbb") (linemap #("aaa" "bbb") (nat-map 1 1 5 2) #(1 5))) + (check-equal? (string-linemap "") (linemap #("") (nat-map 0 1) #(0))) + (check-equal? (string-linemap "a") (linemap #("a") (nat-map 0 1) #(0))) + (check-equal? (string-linemap "λ") (linemap #("λ") (nat-map 0 1) #(0))) + (check-equal? (string-linemap "a\n") (linemap #("a" "") (nat-map 0 1 2 2) #(0 2))) + (check-equal? (string-linemap "λ\n") (linemap #("λ" "") (nat-map 0 1 2 2) #(0 2))) + (check-equal? (string-linemap "aaa\n") (linemap #("aaa" "") (nat-map 0 1 4 2) #(0 4))) + (check-equal? (string-linemap "aaa\nbbb") (linemap #("aaa" "bbb") (nat-map 0 1 4 2) #(0 4))) (check-equal? (string-linemap "aaa\nbbb\n") - (linemap #("aaa" "bbb" "") (nat-map 1 1 5 2 9 3) #(1 5 9))) + (linemap #("aaa" "bbb" "") (nat-map 0 1 4 2 8 3) #(0 4 8))) (check-equal? (string-linemap "a\n\n\nb") - (linemap #("a" "" "" "b") (nat-map 1 1 3 2 4 3 5 4) #(1 3 4 5)))) + (linemap #("a" "" "" "b") (nat-map 0 1 2 2 3 3 4 4) #(0 2 3 4)))) (test-case (name-string linemap-position-to-line) (test-case "two-line string with ending newline" (define map (string-linemap "hello\nworld\n")) + (check-equal? (linemap-position-to-line map 0) 1) (check-equal? (linemap-position-to-line map 1) 1) (check-equal? (linemap-position-to-line map 2) 1) (check-equal? (linemap-position-to-line map 3) 1) (check-equal? (linemap-position-to-line map 4) 1) (check-equal? (linemap-position-to-line map 5) 1) - (check-equal? (linemap-position-to-line map 6) 1) + (check-equal? (linemap-position-to-line map 6) 2) (check-equal? (linemap-position-to-line map 7) 2) (check-equal? (linemap-position-to-line map 8) 2) (check-equal? (linemap-position-to-line map 9) 2) (check-equal? (linemap-position-to-line map 10) 2) (check-equal? (linemap-position-to-line map 11) 2) - (check-equal? (linemap-position-to-line map 12) 2) - (check-equal? (linemap-position-to-line map 13) 3)) + (check-equal? (linemap-position-to-line map 12) 3)) (test-case "multiple blank lines" (define map (string-linemap "a\n\nb")) + (check-equal? (linemap-position-to-line map 0) 1) (check-equal? (linemap-position-to-line map 1) 1) - (check-equal? (linemap-position-to-line map 2) 1) - (check-equal? (linemap-position-to-line map 3) 2) - (check-equal? (linemap-position-to-line map 4) 3) - (check-equal? (linemap-position-to-line map 5) 3))) + (check-equal? (linemap-position-to-line map 2) 2) + (check-equal? (linemap-position-to-line map 3) 3) + (check-equal? (linemap-position-to-line map 4) 3))) (test-case (name-string linemap-line-start-position) (test-case "two-line string with ending newline" (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-line-start-position map 1) 1) - (check-equal? (linemap-line-start-position map 2) 7) - (check-equal? (linemap-line-start-position map 3) 13)) + (check-equal? (linemap-line-start-position map 1) 0) + (check-equal? (linemap-line-start-position map 2) 6) + (check-equal? (linemap-line-start-position map 3) 12)) (test-case "multiple blank lines" (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-line-start-position map 1) 1) - (check-equal? (linemap-line-start-position map 2) 3) - (check-equal? (linemap-line-start-position map 3) 4))) + (check-equal? (linemap-line-start-position map 1) 0) + (check-equal? (linemap-line-start-position map 2) 2) + (check-equal? (linemap-line-start-position map 3) 3))) (test-case (name-string linemap-line-end-position) (test-case "two-line string with ending newline" (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-line-end-position map 1) 6) - (check-equal? (linemap-line-end-position map 2) 12)) + (check-equal? (linemap-line-end-position map 1) 5) + (check-equal? (linemap-line-end-position map 2) 11)) (test-case "multiple blank lines" (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-line-end-position map 1) 2) - (check-equal? (linemap-line-end-position map 2) 3) - (check-equal? (linemap-line-end-position map 3) 5))) + (check-equal? (linemap-line-end-position map 1) 1) + (check-equal? (linemap-line-end-position map 2) 2) + (check-equal? (linemap-line-end-position map 3) 4))) (test-case (name-string linemap-position-to-start-of-line) (test-case "two-line string with ending newline" (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-position-to-start-of-line map 1) 1) - (check-equal? (linemap-position-to-start-of-line map 2) 1) - (check-equal? (linemap-position-to-start-of-line map 3) 1) - (check-equal? (linemap-position-to-start-of-line map 4) 1) - (check-equal? (linemap-position-to-start-of-line map 5) 1) - (check-equal? (linemap-position-to-start-of-line map 6) 1) - (check-equal? (linemap-position-to-start-of-line map 7) 7) - (check-equal? (linemap-position-to-start-of-line map 8) 7) - (check-equal? (linemap-position-to-start-of-line map 9) 7) - (check-equal? (linemap-position-to-start-of-line map 10) 7) - (check-equal? (linemap-position-to-start-of-line map 11) 7) - (check-equal? (linemap-position-to-start-of-line map 12) 7) - (check-equal? (linemap-position-to-start-of-line map 13) 13)) + (check-equal? (linemap-position-to-start-of-line map 0) 0) + (check-equal? (linemap-position-to-start-of-line map 1) 0) + (check-equal? (linemap-position-to-start-of-line map 2) 0) + (check-equal? (linemap-position-to-start-of-line map 3) 0) + (check-equal? (linemap-position-to-start-of-line map 4) 0) + (check-equal? (linemap-position-to-start-of-line map 5) 0) + (check-equal? (linemap-position-to-start-of-line map 6) 6) + (check-equal? (linemap-position-to-start-of-line map 7) 6) + (check-equal? (linemap-position-to-start-of-line map 8) 6) + (check-equal? (linemap-position-to-start-of-line map 9) 6) + (check-equal? (linemap-position-to-start-of-line map 10) 6) + (check-equal? (linemap-position-to-start-of-line map 11) 6) + (check-equal? (linemap-position-to-start-of-line map 12) 12)) (test-case "multiple blank lines" (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-position-to-start-of-line map 1) 1) - (check-equal? (linemap-position-to-start-of-line map 2) 1) + (check-equal? (linemap-position-to-start-of-line map 0) 0) + (check-equal? (linemap-position-to-start-of-line map 1) 0) + (check-equal? (linemap-position-to-start-of-line map 2) 2) (check-equal? (linemap-position-to-start-of-line map 3) 3) - (check-equal? (linemap-position-to-start-of-line map 4) 4) - (check-equal? (linemap-position-to-start-of-line map 5) 4))) + (check-equal? (linemap-position-to-start-of-line map 4) 3))) (test-case (name-string linemap-position-to-end-of-line) (test-case "two-line string with ending newline" (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-position-to-end-of-line map 1) 6) - (check-equal? (linemap-position-to-end-of-line map 2) 6) - (check-equal? (linemap-position-to-end-of-line map 3) 6) - (check-equal? (linemap-position-to-end-of-line map 4) 6) - (check-equal? (linemap-position-to-end-of-line map 5) 6) - (check-equal? (linemap-position-to-end-of-line map 6) 6) - (check-equal? (linemap-position-to-end-of-line map 7) 12) - (check-equal? (linemap-position-to-end-of-line map 8) 12) - (check-equal? (linemap-position-to-end-of-line map 9) 12) - (check-equal? (linemap-position-to-end-of-line map 10) 12) - (check-equal? (linemap-position-to-end-of-line map 11) 12) + (check-equal? (linemap-position-to-end-of-line map 0) 5) + (check-equal? (linemap-position-to-end-of-line map 1) 5) + (check-equal? (linemap-position-to-end-of-line map 2) 5) + (check-equal? (linemap-position-to-end-of-line map 3) 5) + (check-equal? (linemap-position-to-end-of-line map 4) 5) + (check-equal? (linemap-position-to-end-of-line map 5) 5) + (check-equal? (linemap-position-to-end-of-line map 6) 11) + (check-equal? (linemap-position-to-end-of-line map 7) 11) + (check-equal? (linemap-position-to-end-of-line map 8) 11) + (check-equal? (linemap-position-to-end-of-line map 9) 11) + (check-equal? (linemap-position-to-end-of-line map 10) 11) + (check-equal? (linemap-position-to-end-of-line map 11) 11) (check-equal? (linemap-position-to-end-of-line map 12) 12)) (test-case "multiple blank lines" (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-position-to-end-of-line map 1) 2) + (check-equal? (linemap-position-to-end-of-line map 0) 1) + (check-equal? (linemap-position-to-end-of-line map 1) 1) (check-equal? (linemap-position-to-end-of-line map 2) 2) - (check-equal? (linemap-position-to-end-of-line map 3) 3) - (check-equal? (linemap-position-to-end-of-line map 4) 5) - (check-equal? (linemap-position-to-end-of-line map 5) 5)))) + (check-equal? (linemap-position-to-end-of-line map 3) 4) + (check-equal? (linemap-position-to-end-of-line map 4) 4)))) diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl index de76b97..0b6cc91 100644 --- a/grimoire/linemap.scrbl +++ b/grimoire/linemap.scrbl @@ -19,12 +19,15 @@ line-oriented view, used by @tech{source groups} and the command-line interface lines a refactoring suggestion modifies, to render string replacements as line-based diffs, and to restrict analysis to the requested lines. -@bold{All positions and line numbers in a linemap are one-based}, matching the conventions of -@racket[syntax-position] and @racket[syntax-line]: the first character of a string is at position -@racket[1], on line @racket[1]. Beware that this is the @emph{opposite} of the convention used by -@tech{string replacements}, which address characters with @emph{zero-based} indices. Converting -between the two worlds requires adding or subtracting one, as discussed in -@secref["string-replacement"]. +@bold{Positions in a linemap are zero-based, but line numbers are one-based.} Positions are +character indices into the string, following the same convention as Racket's string operations and +as @tech{string replacements}: the first character of a string is at position @racket[0]. Line +numbers instead begin at line @racket[1], matching @racket[syntax-line] and the conventions of +code editors --- line numbers are almost exclusively useful in user interfaces, where one-based +numbering is expected. Beware that @racket[syntax-position] and file port positions are +@emph{one-based}, unlike linemap positions. The @racket[syntax-line-range] operation performs that +conversion itself, but positions obtained from syntax objects by other means must be converted +before use with a linemap. The lines of a string are the segments separated by newline characters. The terminating newline is not part of a line's contents, but positions of newline characters belong to the lines they @@ -41,25 +44,25 @@ consists of a single empty line. treated as line separators.} -@defproc[(linemap-position-to-line [map linemap?] [position exact-positive-integer?]) +@defproc[(linemap-position-to-line [map linemap?] [position exact-nonnegative-integer?]) exact-positive-integer?]{ Returns the line number of the line containing @racket[position]. The position of a newline character is considered contained by the line that the newline terminates. Positions beyond the end of the string do not raise an error; they are all treated as belonging to the last line.} -@defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-positive-integer?]) - exact-positive-integer?]{ +@defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-nonnegative-integer?]) + exact-nonnegative-integer?]{ Returns the position of the first character of the line containing @racket[position]. If the string ends with a newline and @racket[position] is on the final, empty line after it, that - line's start position is one past the end of the string.} + line's start position is equal to the length of the string.} -@defproc[(linemap-position-to-end-of-line [map linemap?] [position exact-positive-integer?]) - exact-positive-integer?]{ +@defproc[(linemap-position-to-end-of-line [map linemap?] [position exact-nonnegative-integer?]) + exact-nonnegative-integer?]{ Returns the position just past the last character of the contents of the line containing - @racket[position] --- that is, the position of the line's terminating newline, or one past the - end of the string if the line is the last one.} + @racket[position] --- that is, the position of the line's terminating newline, or the length of + the string if the line is the last one.} @defproc[(syntax-line-range [stx syntax?] [#:linemap map linemap?]) range?]{ diff --git a/private/line-replacement.rkt b/private/line-replacement.rkt index 5e52a29..37d8df2 100644 --- a/private/line-replacement.rkt +++ b/private/line-replacement.rkt @@ -77,16 +77,13 @@ (define new-lmap (string-linemap new-string)) (define start-line - (linemap-position-to-line orig-lmap (add1 (string-replacement-start replacement)))) + (linemap-position-to-line orig-lmap (string-replacement-start replacement))) (define start-pos - (sub1 - (linemap-position-to-start-of-line orig-lmap (add1 (string-replacement-start replacement))))) + (linemap-position-to-start-of-line orig-lmap (string-replacement-start replacement))) (define original-end-pos - (sub1 - (linemap-position-to-end-of-line orig-lmap - (add1 (string-replacement-original-end replacement))))) + (linemap-position-to-end-of-line orig-lmap (string-replacement-original-end replacement))) (define new-end-pos - (sub1 (linemap-position-to-end-of-line new-lmap (add1 (string-replacement-new-end replacement))))) + (linemap-position-to-end-of-line new-lmap (string-replacement-new-end replacement))) (define original-substr (substring original-string start-pos original-end-pos)) (define new-substr (substring new-string start-pos new-end-pos)) diff --git a/private/refactoring-result.rkt b/private/refactoring-result.rkt index 734591f..90ed951 100644 --- a/private/refactoring-result.rkt +++ b/private/refactoring-result.rkt @@ -192,9 +192,9 @@ (define lmap (string-linemap full-orig-code)) (define start (string-replacement-start replacement)) (define end (string-replacement-original-end replacement)) - (define start-column (- (add1 start) (linemap-position-to-start-of-line lmap (add1 start)))) + (define start-column (- start (linemap-position-to-start-of-line lmap start))) (define raw-text (string->immutable-string (substring full-orig-code start end))) - (code-snippet raw-text start-column (linemap-position-to-line lmap (add1 start)))) + (code-snippet raw-text start-column (linemap-position-to-line lmap start))) (define (refactoring-result-new-code result) @@ -203,8 +203,8 @@ (source->string (syntax-replacement-source (refactoring-result-syntax-replacement result)))) (define lmap (string-linemap full-orig-code)) (define start (string-replacement-start replacement)) - (define original-line (linemap-position-to-line lmap (add1 start))) - (define original-column (- (add1 start) (linemap-position-to-start-of-line lmap (add1 start)))) + (define original-line (linemap-position-to-line lmap start)) + (define original-column (- start (linemap-position-to-start-of-line lmap start))) (define refactored-source-code (string-replacement-apply replacement full-orig-code)) (define new-code-string (substring refactored-source-code diff --git a/private/syntax-replacement.rkt b/private/syntax-replacement.rkt index 8dd87c1..3e130ae 100644 --- a/private/syntax-replacement.rkt +++ b/private/syntax-replacement.rkt @@ -267,10 +267,9 @@ (define trailing-text-length (let ([linemap (string-linemap original)] [orig-end (string-replacement-original-end replacement)]) - ;; Convert from 0-based string indices to 1-based positions for linemap - (if (= (linemap-position-to-line linemap (add1 start)) - (linemap-position-to-line linemap (add1 orig-end))) - (- (linemap-position-to-end-of-line linemap (add1 orig-end)) (add1 orig-end)) + (if (= (linemap-position-to-line linemap start) + (linemap-position-to-line linemap orig-end)) + (- (linemap-position-to-end-of-line linemap orig-end) orig-end) 0))) (define allowed-width (- base-allowed-width trailing-text-length)) From cd533afc8a6bc5d03f4d1610cb4b9baa32540eb5 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:40:29 -0700 Subject: [PATCH 06/10] Document newline normalization in source reading The fix for #272 (Windows \r\n newlines breaking analysis, PR #274) lives in with-input-from-source, which reencodes every source port with newline conversion. That behavior and the invariant it protects were undocumented. Co-Authored-By: Claude Fable 5 --- grimoire/linemap.scrbl | 8 ++++++-- grimoire/source.scrbl | 18 ++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl index 0b6cc91..ae60597 100644 --- a/grimoire/linemap.scrbl +++ b/grimoire/linemap.scrbl @@ -5,7 +5,8 @@ racket/contract/base rebellion/base/comparator rebellion/base/range - resyntax/grimoire/linemap)) + resyntax/grimoire/linemap + resyntax/grimoire/source)) @title[#:tag "linemap"]{Linemaps} @@ -41,7 +42,10 @@ consists of a single empty line. @defproc[(string-linemap [str string?]) linemap?]{ Constructs a @tech{linemap} of the lines in @racket[str]. Only @racket[#\newline] characters are - treated as line separators.} + treated as line separators. In particular, Windows-style @racket["\r\n"] line endings are not + understood. This never arises in practice, because Resyntax normalizes all newlines to + @racket[#\newline] when reading @tech{source code} --- see @racket[with-input-from-source] for + details on that normalization and why it matters.} @defproc[(linemap-position-to-line [map linemap?] [position exact-nonnegative-integer?]) diff --git a/grimoire/source.scrbl b/grimoire/source.scrbl index 6984a38..9682ce1 100644 --- a/grimoire/source.scrbl +++ b/grimoire/source.scrbl @@ -4,6 +4,7 @@ @(require (for-label racket/base racket/contract/base racket/path + racket/port rebellion/base/immutable-string rebellion/collection/range-set resyntax/grimoire/source @@ -104,13 +105,26 @@ stack of dependent changes to commit in series without actually mutating the fil @defproc[(source->string [code source?]) immutable-string?]{ Returns the full text of @racket[code], reading it from the filesystem if necessary. For @racket[modified-source?] values, this returns the new, updated text rather than the original - unmodified text.} + unmodified text. The returned text has its newlines normalized, as described in + @racket[with-input-from-source].} @defproc[(with-input-from-source [code source?] [proc (-> any)]) any]{ Calls @racket[proc] with @racket[current-input-port] set to a freshly opened input port reading the contents of @racket[code]. For unmodified file sources, this opens a file port. For modified - sources and string sources, this opens a string port without interacting with the filesystem.} + sources and string sources, this opens a string port without interacting with the filesystem. + + The opened port decodes the source as UTF-8 and @bold{normalizes newlines}, as in + @racket[reencode-input-port]: Windows-style @racket["\r\n"] sequences (and other newline + conventions) are converted into single @racket[#\newline] characters. Every operation that reads + a source goes through this normalization, including @racket[source->string] and + @racket[source-read-syntax]. Normalizing consistently is load-bearing: when line counting is + enabled on a port, Racket counts a @racket["\r\n"] sequence as a @emph{single} position, so + without normalization the source locations of syntax objects read from a source would disagree + with the character indices of that source's text. Resyntax relies on the assumption that a syntax + object's position and span identify exactly the range of characters it was read from. Analyzing + code with Windows-style newlines used to violate that assumption and break Resyntax in + hard-to-diagnose ways.} @section{Parsing, Expanding, and Compiling Sources} From fc7f3b7b68cc07e7261a564f11de00a4a234e66c Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:56:36 -0700 Subject: [PATCH 07/10] Normalize newlines when constructing string and modified sources Previously only the port-reading path normalized newlines, so two string sources could be non-equal? yet denote the same text via source->string. Normalizing eagerly in the struct guards removes that corner case as early in the data flow as possible. The guard conversion matches the full set of sequences that reencode-input-port converts, verified empirically: \r\n, \r NEL, \r, NEL, and LS all become \n. Co-Authored-By: Claude Fable 5 --- grimoire/source.rkt | 32 ++++++++++++++++++++++++++++++-- grimoire/source.scrbl | 13 ++++++++++--- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/grimoire/source.rkt b/grimoire/source.rkt index 8350f50..112cbe4 100644 --- a/grimoire/source.rkt +++ b/grimoire/source.rkt @@ -53,6 +53,19 @@ ;@---------------------------------------------------------------------------------------------------- +;; Matches every newline sequence that reencode-input-port's newline conversion recognizes. +(define newline-sequence-pattern + (let ([cr (string #\return)] + [lf (string #\newline)] + [nel (string (integer->char #x85))] + [ls (string (integer->char #x2028))]) + (regexp (string-append cr lf "|" cr nel "|" cr "|" nel "|" ls)))) + + +(define (string-normalize-newlines str) + (regexp-replace* newline-sequence-pattern str (string #\newline))) + + (struct source () #:transparent) @@ -66,12 +79,13 @@ (struct string-source unmodified-source (contents) #:transparent - #:guard (λ (contents _) (string->immutable-string contents))) + #:guard (λ (contents _) (string->immutable-string (string-normalize-newlines contents)))) (struct modified-source source (original contents) #:transparent - #:guard (λ (original contents _) (values original (string->immutable-string contents)))) + #:guard (λ (original contents _) + (values original (string->immutable-string (string-normalize-newlines contents))))) (define (source-name src) @@ -137,6 +151,20 @@ (module+ test + (test-case "string and modified sources normalize newlines upon construction" + (define cr (string #\return)) + (define lf (string #\newline)) + (define nel (string (integer->char #x85))) + (define ls (string (integer->char #x2028))) + (define normalized (string-source (string-append "a" lf "b" lf "c"))) + (check-equal? (string-source (string-append "a" cr lf "b" cr lf "c")) normalized) + (check-equal? (string-source (string-append "a" cr "b" cr "c")) normalized) + (check-equal? (string-source (string-append "a" nel "b" cr nel "c")) normalized) + (check-equal? (string-source (string-append "a" ls "b" ls "c")) normalized) + (define base (string-source "base")) + (check-equal? (modified-source base (string-append "a" cr lf "b")) + (modified-source base (string-append "a" lf "b")))) + (test-case "source-read-language" (check-equal? (source-read-language (string-source "#lang racket")) 'racket) (check-equal? (source-read-language (string-source "#lang at-exp racket")) 'at-exp) diff --git a/grimoire/source.scrbl b/grimoire/source.scrbl index 9682ce1..30db79c 100644 --- a/grimoire/source.scrbl +++ b/grimoire/source.scrbl @@ -64,7 +64,10 @@ stack of dependent changes to commit in series without actually mutating the fil @defproc[(string-source [contents string?]) string-source?]{ - Constructs a source string containing @racket[contents] directly.} + Constructs a source string containing @racket[contents] directly. The newlines of + @racket[contents] are normalized at construction time, in the same manner described in + @racket[with-input-from-source]. Normalizing eagerly ensures that two string sources denoting the + same text are @racket[equal?] even if they were constructed with different newline conventions.} @defproc[(modified-source? [v any/c]) boolean?]{ @@ -77,7 +80,8 @@ stack of dependent changes to commit in series without actually mutating the fil Constructs a modified source that replaces the contents of @racket[original] with @racket[new-contents]. This represents a whole-file replacement --- the @emph{complete} contents of @racket[original] are @emph{entirely} swapped out with @racket[new-contents]. Modified sources cannot - represent partial edits on their own.} + represent partial edits on their own. Like @racket[string-source], the newlines of + @racket[new-contents] are normalized at construction time.} @defproc[(source-name [code source?]) (or/c path? symbol?)]{ @@ -124,7 +128,10 @@ stack of dependent changes to commit in series without actually mutating the fil with the character indices of that source's text. Resyntax relies on the assumption that a syntax object's position and span identify exactly the range of characters it was read from. Analyzing code with Windows-style newlines used to violate that assumption and break Resyntax in - hard-to-diagnose ways.} + hard-to-diagnose ways. + + String sources and modified sources also apply this normalization eagerly, when the source value + is constructed, so the port-level conversion only has a visible effect for file sources.} @section{Parsing, Expanding, and Compiling Sources} From fc2bf196747e55b38a3d4cfc890dbc8337ea60c9 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 01:02:38 -0700 Subject: [PATCH 08/10] Normalize newlines through reencode-input-port itself Rather than a regexp that reproduces the port's conversion behavior, round- trip the string through an actual newline-converting reencoded port. Slightly more copying, but it can never fall out of sync with what reading a source does. Co-Authored-By: Claude Fable 5 --- grimoire/source.rkt | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/grimoire/source.rkt b/grimoire/source.rkt index 112cbe4..9cb4f2d 100644 --- a/grimoire/source.rkt +++ b/grimoire/source.rkt @@ -53,17 +53,13 @@ ;@---------------------------------------------------------------------------------------------------- -;; Matches every newline sequence that reencode-input-port's newline conversion recognizes. -(define newline-sequence-pattern - (let ([cr (string #\return)] - [lf (string #\newline)] - [nel (string (integer->char #x85))] - [ls (string (integer->char #x2028))]) - (regexp (string-append cr lf "|" cr nel "|" cr "|" nel "|" ls)))) - - +;; Reads str back out through a newline-converting reencoded port, guaranteeing that this +;; normalization can never disagree with the one performed by with-input-from-source. (define (string-normalize-newlines str) - (regexp-replace* newline-sequence-pattern str (string #\newline))) + (define reencoded-in + (reencode-input-port + (open-input-string str) "UTF-8" #false #false 'string-normalize-newlines #true)) + (port->string reencoded-in)) (struct source () #:transparent) From 68dd1673951dadc5b410b0c2b8d6b444631028c9 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 01:29:09 -0700 Subject: [PATCH 09/10] Share one reencode-input-port wrapper across all source reading Co-Authored-By: Claude Fable 5 --- grimoire/source.rkt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/grimoire/source.rkt b/grimoire/source.rkt index 9cb4f2d..a7bc150 100644 --- a/grimoire/source.rkt +++ b/grimoire/source.rkt @@ -53,13 +53,14 @@ ;@---------------------------------------------------------------------------------------------------- -;; Reads str back out through a newline-converting reencoded port, guaranteeing that this -;; normalization can never disagree with the one performed by with-input-from-source. +;; All source text flows through this port wrapper, which decodes it as UTF-8 and converts +;; newline sequences like \r\n into single \n characters. +(define (reencoded-source-input-port in) + (reencode-input-port in "UTF-8" #false #false (object-name in) #true)) + + (define (string-normalize-newlines str) - (define reencoded-in - (reencode-input-port - (open-input-string str) "UTF-8" #false #false 'string-normalize-newlines #true)) - (port->string reencoded-in)) + (port->string (reencoded-source-input-port (open-input-string str)))) (struct source () #:transparent) @@ -93,8 +94,7 @@ (define (with-input-from-source code proc) (define (call-proc-with-reencoded-input in) - (define reencoded-in (reencode-input-port in "UTF-8" #false #false (object-name in) #true)) - (parameterize ([current-input-port reencoded-in]) + (parameterize ([current-input-port (reencoded-source-input-port in)]) (proc))) (match code From 00e44332184dadc435d5730f3d1118c6fb61e985 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 01:51:57 -0700 Subject: [PATCH 10/10] Link line-counting docs from newline normalization prose Co-Authored-By: Claude Fable 5 --- grimoire/source.scrbl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/grimoire/source.scrbl b/grimoire/source.scrbl index 30db79c..0f41351 100644 --- a/grimoire/source.scrbl +++ b/grimoire/source.scrbl @@ -122,13 +122,15 @@ stack of dependent changes to commit in series without actually mutating the fil @racket[reencode-input-port]: Windows-style @racket["\r\n"] sequences (and other newline conventions) are converted into single @racket[#\newline] characters. Every operation that reads a source goes through this normalization, including @racket[source->string] and - @racket[source-read-syntax]. Normalizing consistently is load-bearing: when line counting is - enabled on a port, Racket counts a @racket["\r\n"] sequence as a @emph{single} position, so - without normalization the source locations of syntax objects read from a source would disagree - with the character indices of that source's text. Resyntax relies on the assumption that a syntax - object's position and span identify exactly the range of characters it was read from. Analyzing - code with Windows-style newlines used to violate that assumption and break Resyntax in - hard-to-diagnose ways. + @racket[source-read-syntax]. Normalizing consistently is load-bearing: as described in + @secref["linecol" #:doc '(lib "scribblings/reference/reference.scrbl")], Racket performs this + same conversion whenever it counts line and column numbers, treating a @racket["\r\n"] sequence + as a @emph{single} position --- and line counting must be enabled for syntax objects to receive + line numbers in their source locations. Without normalization, then, the source locations of + syntax objects read from a source would disagree with the character indices of that source's + text. Resyntax relies on the assumption that a syntax object's position and span identify exactly + the range of characters it was read from. Analyzing code with Windows-style newlines used to + violate that assumption and break Resyntax in hard-to-diagnose ways. String sources and modified sources also apply this normalization eagerly, when the source value is constructed, so the port-level conversion only has a visible effect for file sources.}