fix JuliaLang#28990, improve top-level location info (JuliaLang#30641)

This gives better location info inside `:toplevel` expressions and
for REPL inputs that aren't block constructs.
The low-level parser entry points are simplified and renamed to parse-one,
parse-all, and parse-file.

Author: JeffBezanson

base/client.jl

@@ -146,15 +146,30 @@ function eval_user_input(@nospecialize(ast), show_value::Bool)
     nothing
 end
 
+function _parse_input_line_core(s::String, filename::String)
+    ex = ccall(:jl_parse_all, Any, (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t),
+               s, sizeof(s), filename, sizeof(filename))
+    if ex isa Expr && ex.head === :toplevel
+        if isempty(ex.args)
+            return nothing
+        end
+        last = ex.args[end]
+        if last isa Expr && (last.head === :error || last.head === :incomplete)
+            # if a parse error happens in the middle of a multi-line input
+            # return only the error, so that none of the input is evaluated.
+            return last
+        end
+    end
+    return ex
+end
+
 function parse_input_line(s::String; filename::String="none", depwarn=true)
     # For now, assume all parser warnings are depwarns
     ex = if depwarn
-        ccall(:jl_parse_input_line, Any, (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t),
-              s, sizeof(s), filename, sizeof(filename))
+        _parse_input_line_core(s, filename)
     else
         with_logger(NullLogger()) do
-            ccall(:jl_parse_input_line, Any, (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t),
-                  s, sizeof(s), filename, sizeof(filename))
+            _parse_input_line_core(s, filename)
         end
     end
     return ex

base/meta.jl

@@ -128,10 +128,6 @@ function parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool
     if raise && isa(ex,Expr) && ex.head === :error
         throw(ParseError(ex.args[1]))
     end
-    if ex === ()
-        raise && throw(ParseError("end of input"))
-        ex = Expr(:error, "end of input")
-    end
     return ex, pos+1 # C is zero-based, Julia is 1-based
 end
 

src/ast.c

@@ -763,20 +763,26 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v)
     return julia_to_scm_noalloc2(fl_ctx, v);
 }
 
-// this is used to parse a line of repl input
-JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len, const char *filename, size_t filename_len)
+// parse an entire string like a file, reading multiple expressions
+JL_DLLEXPORT jl_value_t *jl_parse_all(const char *str, size_t len, const char *filename, size_t filename_len)
 {
     JL_TIMING(PARSING);
     jl_ast_context_t *ctx = jl_ast_ctx_enter();
     fl_context_t *fl_ctx = &ctx->fl;
     value_t s = cvalue_static_cstrn(fl_ctx, str, len);
     value_t files = cvalue_static_cstrn(fl_ctx, filename, filename_len);
-    value_t e = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-string")), s, files);
+    value_t e = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-all")), s, files);
     jl_value_t *res = e == fl_ctx->FL_EOF ? jl_nothing : scm_to_julia(fl_ctx, e, NULL);
     jl_ast_ctx_leave(ctx);
     return res;
 }
 
+// for backwards compat
+JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len, const char *filename, size_t filename_len)
+{
+    return jl_parse_all(str, len, filename, filename_len);
+}
+
 // this is for parsing one expression out of a string, keeping track of
 // the current position.
 JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
@@ -792,7 +798,7 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
     jl_ast_context_t *ctx = jl_ast_ctx_enter();
     fl_context_t *fl_ctx = &ctx->fl;
     value_t s = cvalue_static_cstrn(fl_ctx, str, len);
-    value_t p = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-parse-one-string")),
+    value_t p = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-parse-one")),
                           s, fixnum(pos0), greedy?fl_ctx->T:fl_ctx->F);
     jl_value_t *expr=NULL, *pos1=NULL;
     JL_GC_PUSH2(&expr, &pos1);
@@ -828,7 +834,7 @@ jl_value_t *jl_parse_eval_all(const char *fname,
         JL_TIMING(PARSING);
         value_t t = cvalue_static_cstrn(fl_ctx, content, contentlen);
         fl_gc_handle(fl_ctx, &t);
-        ast = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-string-stream")), t, f);
+        ast = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-all")), t, f);
         fl_free_gc_handles(fl_ctx, 1);
     }
     else {
@@ -869,9 +875,9 @@ jl_value_t *jl_parse_eval_all(const char *fname,
                 }
                 // expand non-final expressions in statement position (value unused)
                 expression =
-                    fl_applyn(fl_ctx, 1,
+                    fl_applyn(fl_ctx, 3,
                               symbol_value(symbol(fl_ctx, iscons(cdr_(ast)) ? "jl-expand-to-thunk-stmt" : "jl-expand-to-thunk")),
-                              expression);
+                              expression, symbol(fl_ctx, jl_filename), fixnum(jl_lineno));
             }
             jl_get_ptls_states()->world_age = jl_world_counter;
             form = scm_to_julia(fl_ctx, expression, inmodule);
@@ -929,6 +935,21 @@ jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr, jl_module
     return result;
 }
 
+jl_value_t *jl_call_scm_on_ast_and_loc(const char *funcname, jl_value_t *expr, jl_module_t *inmodule,
+                                       const char *file, int line)
+{
+    jl_ast_context_t *ctx = jl_ast_ctx_enter();
+    fl_context_t *fl_ctx = &ctx->fl;
+    JL_AST_PRESERVE_PUSH(ctx, old_roots, inmodule);
+    value_t arg = julia_to_scm(fl_ctx, expr);
+    value_t e = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, funcname)), arg,
+                          symbol(fl_ctx, file), fixnum(line));
+    jl_value_t *result = scm_to_julia(fl_ctx, e, inmodule);
+    JL_AST_PRESERVE_POP(ctx, old_roots);
+    jl_ast_ctx_leave(ctx);
+    return result;
+}
+
 // syntax tree accessors
 
 JL_DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr)
@@ -1165,29 +1186,40 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand1(jl_value_t *expr, jl_module_t *inmodule
     return expr;
 }
 
-JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule)
+JL_DLLEXPORT jl_value_t *jl_expand_with_loc(jl_value_t *expr, jl_module_t *inmodule,
+                                            const char *file, int line)
 {
     JL_TIMING(LOWERING);
     JL_GC_PUSH1(&expr);
     expr = jl_copy_ast(expr);
     expr = jl_expand_macros(expr, inmodule, NULL, 0);
-    expr = jl_call_scm_on_ast("jl-expand-to-thunk", expr, inmodule);
+    expr = jl_call_scm_on_ast_and_loc("jl-expand-to-thunk", expr, inmodule, file, line);
     JL_GC_POP();
     return expr;
 }
 
+JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule)
+{
+    return jl_expand_with_loc(expr, inmodule, "none", 0);
+}
+
 // expand in a context where the expression value is unused
-JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule)
+JL_DLLEXPORT jl_value_t *jl_expand_stmt_with_loc(jl_value_t *expr, jl_module_t *inmodule,
+                                                 const char *file, int line)
 {
     JL_TIMING(LOWERING);
     JL_GC_PUSH1(&expr);
     expr = jl_copy_ast(expr);
     expr = jl_expand_macros(expr, inmodule, NULL, 0);
-    expr = jl_call_scm_on_ast("jl-expand-to-thunk-stmt", expr, inmodule);
+    expr = jl_call_scm_on_ast_and_loc("jl-expand-to-thunk-stmt", expr, inmodule, file, line);
     JL_GC_POP();
     return expr;
 }
 
+JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule)
+{
+    return jl_expand_stmt_with_loc(expr, inmodule, "none", 0);
+}
 
 #ifdef __cplusplus
 }

src/jlapi.c

@@ -88,7 +88,7 @@ JL_DLLEXPORT jl_value_t *jl_eval_string(const char *str)
     jl_value_t *r;
     JL_TRY {
         const char filename[] = "none";
-        jl_value_t *ast = jl_parse_input_line(str, strlen(str),
+        jl_value_t *ast = jl_parse_all(str, strlen(str),
                 filename, strlen(filename));
         JL_GC_PUSH1(&ast);
         r = jl_toplevel_eval_in(jl_main_module, ast);

src/jlfrontend.scm

@@ -73,7 +73,7 @@
 ;; return a lambda expression representing a thunk for a top-level expression
 ;; note: expansion of stuff inside module is delayed, so the contents obey
 ;; toplevel expansion order (don't expand until stuff before is evaluated).
-(define (expand-toplevel-expr-- e)
+(define (expand-toplevel-expr-- e file line)
   (let ((ex0 (julia-expand-macroscope e)))
     (if (and (pair? ex0) (eq? (car ex0) 'toplevel))
         ex0
@@ -94,7 +94,8 @@
                              (scope-block
                               (block ,@(map (lambda (v) `(implicit-global ,v)) existing-gv)
                                      ,@(map (lambda (v) `(implicit-global ,v)) gv)
-                                     ,ex))))))
+                                     ,ex)))
+                    file line)))
           (if (and (null? (cdadr (caddr th)))
                    (and (length= (lam:body th) 2)
                         (let ((retval (cadadr (lam:body th))))
@@ -114,7 +115,7 @@
            (and (eq? (car e) 'global) (every symbol? (cdr e))
                 (every (lambda (x) (not (memq x '(true false)))) (cdr e))))))
 
-(define (expand-toplevel-expr e)
+(define (expand-toplevel-expr e file line)
   (cond ((or (atom? e) (toplevel-only-expr? e))
          (if (underscore-symbol? e)
              (error "all-underscore identifier used as rvalue"))
@@ -124,7 +125,7 @@
            (if (not last)
                (begin (reset-gensyms)
                       (set! *in-expand* #t)))
-           (begin0 (expand-toplevel-expr-- e)
+           (begin0 (expand-toplevel-expr-- e file line)
                    (set! *in-expand* last))))))
 
 ;; construct default definitions of `eval` for non-bare modules
@@ -146,10 +147,11 @@
        (= (call include ,x)
           (block
            ,@loc
-           (call (top include) ,name ,x)))))))
+           (call (top include) ,name ,x)))))
+   'none 0))
 
-;; parse only, returning end position, no expansion.
-(define (jl-parse-one-string s pos0 greedy)
+;; parse one expression (if greedy) or atom, returning end position
+(define (jl-parse-one s pos0 greedy)
   (let ((inp (open-input-string s)))
     (io.seek inp pos0)
     (let ((expr (error-wrap (lambda ()
@@ -158,25 +160,7 @@
                                   (julia-parse inp parse-atom))))))
       (cons expr (io.pos inp)))))
 
-(define (jl-parse-string s filename)
-  (with-bindings ((current-filename (symbol filename)))
-    (error-wrap (lambda ()
-                  (let ((inp  (make-token-stream (open-input-string s))))
-                    ;; parse all exprs into a (toplevel ...) form
-                    (let loop ((exprs '()))
-                      ;; delay expansion so macros run in the Task executing
-                      ;; the input, not the task parsing it (issue #2378)
-                      ;; used to be (expand-toplevel-expr expr)
-                      (let ((expr (julia-parse inp)))
-                        (if (eof-object? expr)
-                            (cond ((null? exprs)     expr)
-                                  ((length= exprs 1) (car exprs))
-                                  (else (cons 'toplevel (reverse! exprs))))
-                            (if (and (pair? expr) (eq? (car expr) 'toplevel))
-                                (loop (nreconc (cdr expr) exprs))
-                                (loop (cons expr exprs)))))))))))
-
-(define (jl-parse-all io filename)
+(define (parse-all- io filename)
   (unwind-protect
    (with-bindings ((current-filename (symbol filename)))
     (let ((stream (make-token-stream io)))
@@ -192,44 +176,47 @@
                              (julia-parse stream)))))
                 (if (eof-object? expr)
                     (cons 'toplevel (reverse! exprs))
-                    (let* ((iserr (and (pair? expr) (eq? (car expr) 'error)))
-			   (next (list* expr
-					;; for error, get most recent line number (#16720)
-					(if iserr
-					    `(line ,(input-port-line io))
-					    `(line ,lineno))
-					exprs)))
+                    (let* ((iserr  (and (pair? expr) (eq? (car expr) 'error)))
+                           ;; for error, get most recent line number (#16720)
+                           (lineno (if iserr (input-port-line io) lineno))
+                           (next   (list* expr
+                                          ;; include filename in first line node
+                                          (if (null? exprs)
+                                              `(line ,lineno ,(symbol filename))
+                                              `(line ,lineno))
+                                          exprs)))
                       (if iserr
                           (cons 'toplevel (reverse! next))
                           (loop next))))))))))
    (io.close io)))
 
-;; parse file-in-a-string
-(define (jl-parse-string-stream str filename)
-  (jl-parse-all (open-input-string str) filename))
+;; parse all expressions in a string, the same way files are parsed
+(define (jl-parse-all str filename)
+  (parse-all- (open-input-string str) filename))
 
 (define (jl-parse-file filename)
   (trycatch
-   (jl-parse-all (open-input-file filename) filename)
+   (parse-all- (open-input-file filename) filename)
    (lambda (e) #f)))
 
 ; expand a piece of raw surface syntax to an executable thunk
-(define (jl-expand-to-thunk expr)
+(define (jl-expand-to-thunk expr file line)
   (error-wrap (lambda ()
-                (expand-toplevel-expr expr))))
+                (expand-toplevel-expr expr file line))))
 
-(define (jl-expand-to-thunk-stmt expr)
+(define (jl-expand-to-thunk-stmt expr file line)
   (jl-expand-to-thunk (if (toplevel-only-expr? expr)
                           expr
-                          `(block ,expr (null)))))
+                          `(block ,expr (null)))
+                      file line))
 
 (define (jl-expand-macroscope expr)
   (error-wrap (lambda ()
                 (julia-expand-macroscope expr))))
 
 ; run whole frontend on a string. useful for testing.
 (define (fe str)
-  (expand-toplevel-expr (julia-parse str)))
+  (expand-toplevel-expr (julia-parse str) 'none 0))
 
 (define (profile-e s)
   (with-exception-catcher