diff --git a/crates/erg_common/error.rs b/crates/erg_common/error.rs index 878cc28bf..48a8e8889 100644 --- a/crates/erg_common/error.rs +++ b/crates/erg_common/error.rs @@ -360,6 +360,23 @@ impl Location { } } + pub fn left_main_concat(l: &L, r: &R) -> Self { + let l_loc = l.loc(); + let r_loc = r.loc(); + match ( + l_loc.ln_begin(), + l_loc.col_begin(), + r_loc.ln_end(), + r_loc.col_end(), + ) { + (Some(lb), Some(cb), Some(le), Some(ce)) => Self::range(lb, cb, le, ce), + (Some(_), _, None, None) => l_loc, + (Some(lb), _, Some(le), _) => Self::LineRange(lb, le), + (Some(l), _, _, _) | (_, _, Some(l), _) => Self::Line(l), + _ => Self::Unknown, + } + } + pub fn stream(ls: &[L]) -> Self { if ls.is_empty() { return Self::Unknown; diff --git a/crates/erg_compiler/codegen.rs b/crates/erg_compiler/codegen.rs index 4e26eef0a..fa400023f 100644 --- a/crates/erg_compiler/codegen.rs +++ b/crates/erg_compiler/codegen.rs @@ -3156,7 +3156,7 @@ impl PyCodeGenerator { } None => {} } - let none = Token::new(TokenKind::NoneLit, "None", line, 0); + let none = Token::new_fake(TokenKind::NoneLit, "None", line, 0, 0); attrs.push(Expr::Lit(Literal::new(ValueObj::None, none))); let block = Block::new(attrs); let body = DefBody::new(EQUAL, block, DefId(0)); diff --git a/crates/erg_compiler/context/instantiate_spec.rs b/crates/erg_compiler/context/instantiate_spec.rs index ad858e8b0..3ebaa7fd1 100644 --- a/crates/erg_compiler/context/instantiate_spec.rs +++ b/crates/erg_compiler/context/instantiate_spec.rs @@ -1483,7 +1483,7 @@ impl Context { } TypeSpec::Dict(dict) => { let mut inst_tys = dict! {}; - for (k, v) in dict { + for (k, v) in dict.kvs.iter() { inst_tys.insert( self.instantiate_typespec_full( k, @@ -1505,7 +1505,7 @@ impl Context { } TypeSpec::Record(rec) => { let mut inst_tys = dict! {}; - for (k, v) in rec { + for (k, v) in rec.attrs.iter() { inst_tys.insert( self.instantiate_field(k)?, self.instantiate_typespec_full( diff --git a/crates/erg_compiler/link_hir.rs b/crates/erg_compiler/link_hir.rs index 6b04006b7..aff633ce0 100644 --- a/crates/erg_compiler/link_hir.rs +++ b/crates/erg_compiler/link_hir.rs @@ -475,11 +475,12 @@ impl<'a> HIRLinker<'a> { let mut comps = dir.components(); let _first = comps.next().unwrap(); let path = dir.to_string_lossy().replace(['/', '\\'], "."); - let token = Token::new( + let token = Token::new_fake( TokenKind::StrLit, path, mod_name_lit.ln_begin().unwrap(), mod_name_lit.col_begin().unwrap(), + mod_name_lit.col_end().unwrap(), ); let mod_name = Expr::Lit(Literal::try_from(token).unwrap()); args.insert_pos(0, PosArg::new(mod_name)); diff --git a/crates/erg_parser/ast.rs b/crates/erg_parser/ast.rs index db14d9516..00180fce0 100644 --- a/crates/erg_parser/ast.rs +++ b/crates/erg_parser/ast.rs @@ -90,7 +90,7 @@ impl Literal { } pub fn nat(n: usize, line: u32) -> Self { - let token = Token::new(TokenKind::NatLit, Str::from(n.to_string()), line, 0); + let token = Token::new_fake(TokenKind::NatLit, Str::from(n.to_string()), line, 0, 0); Self { token } } @@ -814,6 +814,15 @@ impl_nested_display_for_enum!(Dict; Normal, Comprehension); impl_display_for_enum!(Dict; Normal, Comprehension); impl_locational_for_enum!(Dict; Normal, Comprehension); +impl Dict { + pub fn braces(&self) -> (&Token, &Token) { + match self { + Self::Normal(dict) => (&dict.l_brace, &dict.r_brace), + Self::Comprehension(dict) => (&dict.l_brace, &dict.r_brace), + } + } +} + #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum ClassAttr { Def(Def), @@ -961,6 +970,13 @@ impl Record { attrs: RecordAttrs::new(Vec::with_capacity(0)), }) } + + pub fn braces(&self) -> (&Token, &Token) { + match self { + Self::Normal(record) => (&record.l_brace, &record.r_brace), + Self::Mixed(record) => (&record.l_brace, &record.r_brace), + } + } } /// Record can be defined with shorthend/normal mixed style, i.e. {x; y=expr; z; ...} @@ -2608,6 +2624,78 @@ impl TupleTypeSpec { } } +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct DictTypeSpec { + pub braces: Option<(Token, Token)>, + pub kvs: Vec<(TypeSpec, TypeSpec)>, +} + +impl fmt::Display for DictTypeSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{{")?; + for (k, v) in self.kvs.iter() { + write!(f, "{k}: {v}, ")?; + } + write!(f, "}}") + } +} + +impl Locational for DictTypeSpec { + fn loc(&self) -> Location { + if let Some((lparen, rparen)) = &self.braces { + Location::concat(lparen, rparen) + } else if !self.kvs.is_empty() { + let (first, _) = self.kvs.first().unwrap(); + let (_, last) = self.kvs.last().unwrap(); + Location::concat(first, last) + } else { + Location::Unknown + } + } +} + +impl DictTypeSpec { + pub const fn new(braces: Option<(Token, Token)>, kvs: Vec<(TypeSpec, TypeSpec)>) -> Self { + Self { braces, kvs } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct RecordTypeSpec { + pub braces: Option<(Token, Token)>, + pub attrs: Vec<(Identifier, TypeSpec)>, +} + +impl fmt::Display for RecordTypeSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{{")?; + for (k, v) in self.attrs.iter() { + write!(f, "{k} = {v}; ")?; + } + write!(f, "}}") + } +} + +impl Locational for RecordTypeSpec { + fn loc(&self) -> Location { + if let Some((lparen, rparen)) = &self.braces { + Location::concat(lparen, rparen) + } else if !self.attrs.is_empty() { + let (first, _) = self.attrs.first().unwrap(); + let (_, last) = self.attrs.last().unwrap(); + Location::concat(first, last) + } else { + Location::Unknown + } + } +} + +impl RecordTypeSpec { + pub const fn new(braces: Option<(Token, Token)>, attrs: Vec<(Identifier, TypeSpec)>) -> Self { + Self { braces, attrs } + } +} + #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct RefinementTypeSpec { pub var: Token, @@ -2656,8 +2744,8 @@ pub enum TypeSpec { Array(ArrayTypeSpec), SetWithLen(SetWithLenTypeSpec), Tuple(TupleTypeSpec), - Dict(Vec<(TypeSpec, TypeSpec)>), - Record(Vec<(Identifier, TypeSpec)>), + Dict(DictTypeSpec), + Record(RecordTypeSpec), // Option(), And(Box, Box), Not(Box), @@ -2687,20 +2775,8 @@ impl fmt::Display for TypeSpec { Self::Array(arr) => write!(f, "{arr}"), Self::SetWithLen(set) => write!(f, "{set}"), Self::Tuple(tup) => write!(f, "{tup}"), - Self::Dict(dict) => { - write!(f, "{{")?; - for (k, v) in dict.iter() { - write!(f, "{k}: {v}, ")?; - } - write!(f, "}}") - } - Self::Record(rec) => { - write!(f, "{{")?; - for (k, v) in rec.iter() { - write!(f, "{k} = {v}; ")?; - } - write!(f, "}}") - } + Self::Dict(dict) => dict.fmt(f), + Self::Record(rec) => rec.fmt(f), Self::Enum(elems) => { write!(f, "{{")?; for elem in elems.pos_args() { @@ -2728,20 +2804,8 @@ impl Locational for TypeSpec { Self::Array(arr) => arr.loc(), Self::SetWithLen(set) => set.loc(), Self::Tuple(tup) => tup.loc(), - Self::Dict(dict) => { - if dict.is_empty() { - Location::Unknown - } else { - Location::concat(&dict.first().unwrap().0, &dict.last().unwrap().1) - } - } - Self::Record(rec) => { - if rec.is_empty() { - Location::Unknown - } else { - Location::concat(&rec.first().unwrap().0, &rec.last().unwrap().1) - } - } + Self::Dict(dict) => dict.loc(), + Self::Record(rec) => rec.loc(), Self::Enum(set) => set.loc(), Self::Interval { lhs, rhs, .. } => Location::concat(lhs, rhs), Self::Subr(s) => s.loc(), @@ -2986,7 +3050,11 @@ impl VarName { } pub fn from_str_and_line(symbol: Str, line: u32) -> Self { - Self(Token::new(TokenKind::Symbol, symbol, line, 0)) + Self(Token::new_fake(TokenKind::Symbol, symbol, line, 0, 0)) + } + + pub fn from_str_and_loc(symbol: Str, loc: Location) -> Self { + Self(Token::new_with_loc(TokenKind::Symbol, symbol, loc)) } #[inline] @@ -3210,6 +3278,13 @@ impl Identifier { ) } + pub fn private_with_loc(name: Str, loc: Location) -> Self { + Self::new( + VisModifierSpec::Private, + VarName::from_str_and_loc(name, loc), + ) + } + pub fn public_with_line(dot: Token, name: Str, line: u32) -> Self { Self::new( VisModifierSpec::Public(dot), @@ -3801,7 +3876,7 @@ impl_display_from_nested!(NonDefaultParamSignature); impl Locational for NonDefaultParamSignature { fn loc(&self) -> Location { if let Some(t_spec) = &self.t_spec { - Location::concat(&self.pat, t_spec) + Location::left_main_concat(&self.pat, t_spec) } else { self.pat.loc() } @@ -4611,12 +4686,13 @@ impl Expr { } } - pub fn local(name: &str, lineno: u32, col_begin: u32) -> Self { - Self::Accessor(Accessor::local(Token::new( + pub fn local(name: &str, lineno: u32, col_begin: u32, col_end: u32) -> Self { + Self::Accessor(Accessor::local(Token::new_fake( TokenKind::Symbol, Str::rc(name), lineno, col_begin, + col_end, ))) } diff --git a/crates/erg_parser/desugar.rs b/crates/erg_parser/desugar.rs index 8d286ec77..83328ea23 100644 --- a/crates/erg_parser/desugar.rs +++ b/crates/erg_parser/desugar.rs @@ -4,6 +4,7 @@ //! e.g. Literal parameters, Multi assignment //! 型チェックなどによる検証は行わない +use erg_common::error::Location; use erg_common::fresh::FreshNameGenerator; use erg_common::traits::{Locational, Stream}; use erg_common::Str; @@ -15,7 +16,7 @@ use crate::ast::{ DefaultParamSignature, Dict, Dummy, Expr, Identifier, KeyValue, KwArg, Lambda, LambdaSignature, Literal, Methods, MixedRecord, Module, NonDefaultParamSignature, NormalArray, NormalDict, NormalRecord, NormalSet, NormalTuple, ParamPattern, ParamRecordAttr, ParamTuplePattern, Params, - PatchDef, PosArg, ReDef, Record, RecordAttrOrIdent, RecordAttrs, Set as astSet, + PatchDef, PosArg, ReDef, Record, RecordAttrOrIdent, RecordAttrs, RecordTypeSpec, Set as astSet, SetComprehension, SetWithLength, Signature, SubrSignature, Tuple, TupleTypeSpec, TypeAppArgs, TypeAppArgsKind, TypeBoundSpecs, TypeSpec, TypeSpecWithOp, UnaryOp, VarName, VarPattern, VarRecordAttr, VarSignature, VisModifierSpec, @@ -427,11 +428,15 @@ impl Desugarer { match &arg.expr { Expr::Accessor(Accessor::Ident(ident)) => { let param_name = ident.inspect(); - let param = VarName::new(Token::new( + let col_begin = name.col_end().unwrap_or(0) + 1; + let col_end = + col_begin + param_name.chars().count() as u32; + let param = VarName::new(Token::new_fake( TokenKind::Symbol, param_name, name.ln_begin().unwrap_or(1), - name.col_end().unwrap_or(0) + 1, + col_begin, + col_end, )); let param = NonDefaultParamSignature::new( ParamPattern::VarName(param), @@ -446,11 +451,14 @@ impl Desugarer { } Expr::Accessor(Accessor::Ident(ident)) => { let param_name = ident.inspect(); - let param = VarName::new(Token::new( + let col_begin = name.col_end().unwrap_or(0) + 1; // HACK: `(name) %x = ...`という形を想定 + let col_end = col_begin + param_name.chars().count() as u32; + let param = VarName::new(Token::new_fake( TokenKind::Symbol, param_name, name.ln_begin().unwrap_or(1), - name.col_end().unwrap_or(0) + 1, // HACK: `(name) %x = ...`という形を想定 + col_begin, + col_end, )); let param = NonDefaultParamSignature::new( ParamPattern::VarName(param), @@ -502,6 +510,7 @@ impl Desugarer { name, first_arg.ln_begin().unwrap_or(1), first_arg.col_begin().unwrap_or(0), + first_arg.col_end().unwrap_or(0), )); call.args.insert_pos(0, arg); } @@ -575,20 +584,20 @@ impl Desugarer { fn gen_buf_name_and_sig( &mut self, - line: u32, + loc: Location, t_spec: Option, ) -> (Str, Signature) { let buf_name = self.var_gen.fresh_varname(); let buf_sig = Signature::Var(VarSignature::new( - VarPattern::Ident(Identifier::private_with_line(Str::rc(&buf_name), line)), + VarPattern::Ident(Identifier::private_with_loc(Str::rc(&buf_name), loc)), t_spec, )); (buf_name, buf_sig) } - fn gen_buf_nd_param(&mut self, line: u32) -> (Str, ParamPattern) { + fn gen_buf_nd_param(&mut self, loc: Location) -> (Str, ParamPattern) { let buf_name = self.var_gen.fresh_varname(); - let pat = ParamPattern::VarName(VarName::from_str_and_line(Str::rc(&buf_name), line)); + let pat = ParamPattern::VarName(VarName::from_str_and_loc(buf_name.clone(), loc)); (buf_name, pat) } @@ -632,8 +641,7 @@ impl Desugarer { body, }) => match &v.pat { VarPattern::Tuple(tup) => { - let (buf_name, buf_sig) = - self.gen_buf_name_and_sig(v.ln_begin().unwrap_or(1), v.t_spec); + let (buf_name, buf_sig) = self.gen_buf_name_and_sig(v.loc(), v.t_spec); let block = body .block .into_iter() @@ -652,8 +660,7 @@ impl Desugarer { } } VarPattern::Array(arr) => { - let (buf_name, buf_sig) = - self.gen_buf_name_and_sig(v.ln_begin().unwrap_or(1), v.t_spec); + let (buf_name, buf_sig) = self.gen_buf_name_and_sig(v.loc(), v.t_spec); let block = body .block .into_iter() @@ -672,8 +679,7 @@ impl Desugarer { } } VarPattern::Record(rec) => { - let (buf_name, buf_sig) = - self.gen_buf_name_and_sig(v.ln_begin().unwrap_or(1), v.t_spec); + let (buf_name, buf_sig) = self.gen_buf_name_and_sig(v.loc(), v.t_spec); let block = body .block .into_iter() @@ -698,7 +704,7 @@ impl Desugarer { *pack.class_as_expr.clone(), ); let (buf_name, buf_sig) = self.gen_buf_name_and_sig( - v.ln_begin().unwrap_or(1), + v.loc(), Some(t_spec), // TODO: これだとvの型指定の意味がなくなる ); let block = body @@ -781,6 +787,7 @@ impl Desugarer { buf_name, sig.ln_begin().unwrap_or(1), sig.col_begin().unwrap_or(0), + sig.col_end().unwrap_or(0), ); let acc = match buf_index { BufIndex::Tuple(n) => obj.tuple_attr(Literal::nat(n, sig.ln_begin().unwrap_or(1))), @@ -807,8 +814,7 @@ impl Desugarer { let body = DefBody::new(op, block, id); match &sig.pat { VarPattern::Tuple(tup) => { - let (buf_name, buf_sig) = - self.gen_buf_name_and_sig(sig.ln_begin().unwrap_or(1), None); + let (buf_name, buf_sig) = self.gen_buf_name_and_sig(sig.loc(), None); let buf_def = Def::new(buf_sig, body); new_module.push(Expr::Def(buf_def)); for (n, elem) in tup.elems.iter().enumerate() { @@ -821,8 +827,7 @@ impl Desugarer { } } VarPattern::Array(arr) => { - let (buf_name, buf_sig) = - self.gen_buf_name_and_sig(sig.ln_begin().unwrap_or(1), None); + let (buf_name, buf_sig) = self.gen_buf_name_and_sig(sig.loc(), None); let buf_def = Def::new(buf_sig, body); new_module.push(Expr::Def(buf_def)); for (n, elem) in arr.elems.iter().enumerate() { @@ -835,8 +840,7 @@ impl Desugarer { } } VarPattern::Record(rec) => { - let (buf_name, buf_sig) = - self.gen_buf_name_and_sig(sig.ln_begin().unwrap_or(1), None); + let (buf_name, buf_sig) = self.gen_buf_name_and_sig(sig.loc(), None); let buf_def = Def::new(buf_sig, body); new_module.push(Expr::Def(buf_def)); for VarRecordAttr { lhs, rhs } in rec.attrs.iter() { @@ -851,8 +855,7 @@ impl Desugarer { VarPattern::DataPack(pack) => { let t_spec = TypeSpecWithOp::new(COLON, pack.class.clone(), *pack.class_as_expr.clone()); - let (buf_name, buf_sig) = - self.gen_buf_name_and_sig(sig.ln_begin().unwrap_or(1), Some(t_spec)); + let (buf_name, buf_sig) = self.gen_buf_name_and_sig(sig.loc(), Some(t_spec)); let buf_def = Def::new(buf_sig, body); new_module.push(Expr::Def(buf_def)); for VarRecordAttr { lhs, rhs } in pack.args.attrs.iter() { @@ -911,6 +914,7 @@ impl Desugarer { ident.inspect(), ident.ln_begin().unwrap_or(1), ident.col_begin().unwrap_or(0), + ident.col_end().unwrap_or(0), )]), DefId(get_hash(&(&sig, ident.inspect()))), ); @@ -933,7 +937,7 @@ impl Desugarer { kind: TokenKind::RSqBr, ..len.token }; - let elem = Expr::local("Obj", l_sqbr.lineno, l_sqbr.col_begin); + let elem = Expr::local("Obj", l_sqbr.lineno, l_sqbr.col_begin, l_sqbr.col_end); let array = Array::WithLength(ArrayWithLength::new( l_sqbr, r_sqbr, @@ -997,11 +1001,12 @@ impl Desugarer { ParamPattern::VarName(_v) => {} ParamPattern::Lit(l) => { let lit = l.clone(); - param.pat = ParamPattern::Discard(Token::new( + param.pat = ParamPattern::Discard(Token::new_fake( TokenKind::UBar, "_", l.ln_begin().unwrap_or(1), l.col_begin().unwrap_or(0), + l.col_end().unwrap_or(0), )); let l_brace = Token { content: "{".into(), @@ -1020,7 +1025,7 @@ impl Desugarer { param.t_spec = Some(t_spec); } ParamPattern::Tuple(tup) => { - let (buf_name, buf_param) = self.gen_buf_nd_param(line); + let (buf_name, buf_param) = self.gen_buf_nd_param(tup.loc()); let mut ty_specs = vec![]; let mut ty_exprs = vec![]; for (n, elem) in tup.elems.non_defaults.iter_mut().enumerate() { @@ -1031,12 +1036,17 @@ impl Desugarer { BufIndex::Tuple(n), insertion_idx, ); - let infer = Token::new(TokenKind::Try, "?", line, 0); + let infer = Token::new_fake(TokenKind::Try, "?", line, 0, 0); let ty_expr = elem .t_spec .as_ref() .map(|ts| *ts.t_spec_as_expr.clone()) - .unwrap_or(Expr::local("Obj", infer.lineno, infer.col_begin)); + .unwrap_or(Expr::local( + "Obj", + infer.lineno, + infer.col_begin, + infer.col_end, + )); ty_exprs.push(PosArg::new(ty_expr)); ty_specs.push( elem.t_spec @@ -1058,7 +1068,7 @@ impl Desugarer { param.pat = buf_param; } ParamPattern::Array(arr) => { - let (buf_name, buf_param) = self.gen_buf_nd_param(line); + let (buf_name, buf_param) = self.gen_buf_nd_param(arr.loc()); for (n, elem) in arr.elems.non_defaults.iter_mut().enumerate() { insertion_idx = self.desugar_nested_param_pattern( body, @@ -1070,8 +1080,14 @@ impl Desugarer { } if param.t_spec.is_none() { let len = arr.elems.non_defaults.len(); - let len = Literal::new(Token::new(TokenKind::NatLit, len.to_string(), line, 0)); - let infer = Token::new(TokenKind::Try, "?", line, 0); + let len = Literal::new(Token::new_fake( + TokenKind::NatLit, + len.to_string(), + line, + 0, + 0, + )); + let infer = Token::new_fake(TokenKind::Try, "?", line, 0, 0); let t_spec = ArrayTypeSpec::new(TypeSpec::Infer(infer), ConstExpr::Lit(len.clone())); let t_spec_as_expr = Self::dummy_array_expr(len); @@ -1084,7 +1100,7 @@ impl Desugarer { param.pat = buf_param; } ParamPattern::Record(rec) => { - let (buf_name, buf_param) = self.gen_buf_nd_param(line); + let (buf_name, buf_param) = self.gen_buf_nd_param(rec.loc()); for ParamRecordAttr { lhs, rhs } in rec.elems.iter_mut() { insertion_idx = self.desugar_nested_param_pattern( body, @@ -1102,12 +1118,17 @@ impl Desugarer { vis: VisModifierSpec::Public(Token::DUMMY), ..lhs.clone() }; - let infer = Token::new(TokenKind::Try, "?", line, 0); + let infer = Token::new_fake(TokenKind::Try, "?", line, 0, 0); let expr = rhs .t_spec .as_ref() .map(|ts| *ts.t_spec_as_expr.clone()) - .unwrap_or(Expr::local("Obj", infer.lineno, infer.col_begin)); + .unwrap_or(Expr::local( + "Obj", + infer.lineno, + infer.col_begin, + infer.col_end, + )); let attr = Def::new(Signature::new_var(lhs.clone()), DefBody::new_single(expr)); attrs.push(attr); @@ -1120,7 +1141,10 @@ impl Desugarer { .clone(), )); } - let t_spec = TypeSpec::Record(tys); + let t_spec = TypeSpec::Record(RecordTypeSpec::new( + Some((rec.l_brace.clone(), rec.r_brace.clone())), + tys, + )); let t_spec_as_expr = Expr::from(NormalRecord::new( rec.l_brace.clone(), rec.r_brace.clone(), @@ -1162,6 +1186,7 @@ impl Desugarer { buf_name, sig.ln_begin().unwrap_or(1), sig.col_begin().unwrap_or(0), + sig.col_end().unwrap_or(0), ); let acc = match buf_index { BufIndex::Tuple(n) => obj.tuple_attr(Literal::nat(n, sig.ln_begin().unwrap_or(1))), @@ -1189,7 +1214,7 @@ impl Desugarer { let line = sig.ln_begin().unwrap_or(1); match &mut sig.pat { ParamPattern::Tuple(tup) => { - let (buf_name, buf_sig) = self.gen_buf_nd_param(line); + let (buf_name, buf_sig) = self.gen_buf_nd_param(tup.loc()); let ident = Identifier::private(Str::from(&buf_name)); new_body.insert( insertion_idx, @@ -1212,12 +1237,17 @@ impl Desugarer { BufIndex::Tuple(n), insertion_idx, ); - let infer = Token::new(TokenKind::Try, "?", line, 0); + let infer = Token::new_fake(TokenKind::Try, "?", line, 0, 0); let ty_expr = elem .t_spec .as_ref() .map(|ts| *ts.t_spec_as_expr.clone()) - .unwrap_or(Expr::local("Obj", infer.lineno, infer.col_begin)); + .unwrap_or(Expr::local( + "Obj", + infer.lineno, + infer.col_begin, + infer.col_end, + )); ty_exprs.push(PosArg::new(ty_expr)); tys.push( elem.t_spec @@ -1239,7 +1269,7 @@ impl Desugarer { insertion_idx } ParamPattern::Array(arr) => { - let (buf_name, buf_sig) = self.gen_buf_nd_param(line); + let (buf_name, buf_sig) = self.gen_buf_nd_param(arr.loc()); new_body.insert( insertion_idx, Expr::Def(Def::new( @@ -1262,8 +1292,14 @@ impl Desugarer { } if sig.t_spec.is_none() { let len = arr.elems.non_defaults.len(); - let len = Literal::new(Token::new(TokenKind::NatLit, len.to_string(), line, 0)); - let infer = Token::new(TokenKind::Try, "?", line, 0); + let len = Literal::new(Token::new_fake( + TokenKind::NatLit, + len.to_string(), + line, + 0, + 0, + )); + let infer = Token::new_fake(TokenKind::Try, "?", line, 0, 0); let t_spec = ArrayTypeSpec::new(TypeSpec::Infer(infer), ConstExpr::Lit(len.clone())); let t_spec_as_expr = Self::dummy_array_expr(len); @@ -1277,7 +1313,7 @@ impl Desugarer { insertion_idx } ParamPattern::Record(rec) => { - let (buf_name, buf_sig) = self.gen_buf_nd_param(line); + let (buf_name, buf_sig) = self.gen_buf_nd_param(rec.loc()); new_body.insert( insertion_idx, Expr::Def(Def::new( @@ -1303,12 +1339,17 @@ impl Desugarer { BufIndex::Record(&lhs), insertion_idx, ); - let infer = Token::new(TokenKind::Try, "?", line, 0); + let infer = Token::new_fake(TokenKind::Try, "?", line, 0, 0); let expr = rhs .t_spec .as_ref() .map(|ts| *ts.t_spec_as_expr.clone()) - .unwrap_or(Expr::local("Obj", infer.lineno, infer.col_begin)); + .unwrap_or(Expr::local( + "Obj", + infer.lineno, + infer.col_begin, + infer.col_end, + )); let attr = Def::new(Signature::new_var(lhs.clone()), DefBody::new_single(expr)); attrs.push(attr); tys.push(( @@ -1321,7 +1362,10 @@ impl Desugarer { )); } if sig.t_spec.is_none() { - let t_spec = TypeSpec::Record(tys); + let t_spec = TypeSpec::Record(RecordTypeSpec::new( + Some((rec.l_brace.clone(), rec.r_brace.clone())), + tys, + )); let t_spec_as_expr = Expr::from(NormalRecord::new( rec.l_brace.clone(), rec.r_brace.clone(), @@ -1358,11 +1402,12 @@ impl Desugarer { } ParamPattern::Lit(l) => { let lit = l.clone(); - sig.pat = ParamPattern::Discard(Token::new( + sig.pat = ParamPattern::Discard(Token::new_fake( TokenKind::UBar, "_", l.ln_begin().unwrap_or(1), l.col_begin().unwrap_or(1), + l.col_end().unwrap_or(1), )); let t_spec = TypeSpec::enum_t_spec(vec![lit.clone()]); let t_spec_as_expr = Self::dummy_set_expr(lit); diff --git a/crates/erg_parser/parse.rs b/crates/erg_parser/parse.rs index 90d54f457..a33211379 100644 --- a/crates/erg_parser/parse.rs +++ b/crates/erg_parser/parse.rs @@ -3333,11 +3333,12 @@ impl Parser { Str::from(format!("\"{}", right.content.trim_start_matches('}'))); right.kind = StrLit; let right = Expr::Literal(Literal::from(right)); - let op = Token::new( + let op = Token::new_fake( Plus, "+", right.ln_begin().unwrap(), right.col_begin().unwrap(), + right.col_end().unwrap(), ); expr = Expr::BinOp(BinOp::new(op, expr, right)); debug_exit_info!(self); @@ -3369,13 +3370,15 @@ impl Parser { "str", mid_expr.ln_begin().unwrap(), mid_expr.col_begin().unwrap(), + mid_expr.col_end().unwrap(), ); let call = Call::new(str_func, None, Args::single(PosArg::new(mid_expr))); - let op = Token::new( + let op = Token::new_fake( Plus, "+", call.ln_begin().unwrap(), call.col_begin().unwrap(), + call.col_end().unwrap(), ); let bin = BinOp::new(op, expr, Expr::Call(call)); expr = Expr::BinOp(bin); @@ -3387,11 +3390,12 @@ impl Parser { )); mid.kind = StrLit; let mid = Expr::Literal(Literal::from(mid)); - let op = Token::new( + let op = Token::new_fake( Plus, "+", mid.ln_begin().unwrap(), mid.col_begin().unwrap(), + mid.col_end().unwrap(), ); expr = Expr::BinOp(BinOp::new(op, expr, mid)); } diff --git a/crates/erg_parser/token.rs b/crates/erg_parser/token.rs index eb1f14699..e6ebb10d9 100644 --- a/crates/erg_parser/token.rs +++ b/crates/erg_parser/token.rs @@ -340,8 +340,11 @@ pub struct Token { /// 1 origin // TODO: 複数行文字列リテラルもあるのでタプルにするのが妥当? pub lineno: u32, - /// a pointer from which the token starts (0 origin) + /// A pointer from which the token starts (0 origin) pub col_begin: u32, + /// A pointer to the end position of the token. + /// `col_end - col_start` does not necessarily equal `content.len()` + pub col_end: u32, } pub const COLON: Token = Token::dummy(TokenKind::Colon, ":"); @@ -356,6 +359,7 @@ impl fmt::Debug for Token { .field("content", &self.content.replace('\n', "\\n")) .field("lineno", &self.lineno) .field("col_begin", &self.col_begin) + .field("col_end", &self.col_end) .finish() } } @@ -387,12 +391,7 @@ impl Locational for Token { if self.lineno == 0 { Location::Unknown } else { - Location::range( - self.lineno, - self.col_begin, - self.lineno, - self.col_begin + self.content.len() as u32, - ) + Location::range(self.lineno, self.col_begin, self.lineno, self.col_end) } } @@ -408,6 +407,7 @@ impl Token { content: Str::ever("DUMMY"), lineno: 1, col_begin: 0, + col_end: 0, }; pub const fn dummy(kind: TokenKind, content: &'static str) -> Self { @@ -416,16 +416,47 @@ impl Token { content: Str::ever(content), lineno: 1, col_begin: 0, + col_end: 0, } } #[inline] pub fn new>(kind: TokenKind, cont: S, lineno: u32, col_begin: u32) -> Self { + let content = cont.into(); + let col_end = col_begin + content.chars().count() as u32; + Token { + kind, + content, + lineno, + col_begin, + col_end, + } + } + + #[inline] + pub fn new_fake>( + kind: TokenKind, + cont: S, + lineno: u32, + col_begin: u32, + col_end: u32, + ) -> Self { Token { kind, content: cont.into(), lineno, col_begin, + col_end, + } + } + + pub fn new_with_loc(kind: TokenKind, cont: impl Into, loc: Location) -> Self { + Token { + kind, + content: cont.into(), + lineno: loc.ln_begin().unwrap_or(0), + col_begin: loc.col_begin().unwrap_or(0), + col_end: loc.col_end().unwrap_or(1), } } @@ -436,6 +467,7 @@ impl Token { content: Str::rc(cont), lineno: 0, col_begin: 0, + col_end: 0, } } @@ -451,6 +483,7 @@ impl Token { content: Str::rc(cont), lineno, col_begin: 0, + col_end: 1, } } @@ -460,6 +493,7 @@ impl Token { content: cont.into(), lineno: loc.ln_begin().unwrap_or(0), col_begin: loc.col_begin().unwrap_or(0), + col_end: loc.col_end().unwrap_or(1), } } @@ -469,6 +503,7 @@ impl Token { content: Str::ever(s), lineno: 0, col_begin: 0, + col_end: 1, } } diff --git a/crates/erg_parser/typespec.rs b/crates/erg_parser/typespec.rs index 81e36a71d..b605c8687 100644 --- a/crates/erg_parser/typespec.rs +++ b/crates/erg_parser/typespec.rs @@ -376,8 +376,10 @@ impl Parser { } } - fn dict_to_dict_type_spec(dict: Dict) -> Result, ParseError> { - match dict { + fn dict_to_dict_type_spec(dict: Dict) -> Result { + let (l, r) = dict.braces(); + let braces = (l.clone(), r.clone()); + let kvs = match dict { Dict::Normal(dic) => { let mut kvs = vec![]; for kv in dic.kvs.into_iter() { @@ -391,13 +393,14 @@ impl Parser { let err = ParseError::simple_syntax_error(line!() as usize, other.loc()); Err(err) } - } + }?; + Ok(DictTypeSpec::new(Some(braces), kvs)) } - fn record_to_record_type_spec( - record: Record, - ) -> Result, ParseError> { - match record { + fn record_to_record_type_spec(record: Record) -> Result { + let (l, r) = record.braces(); + let braces = (l.clone(), r.clone()); + let attrs = match record { Record::Normal(rec) => rec .attrs .into_iter() @@ -423,7 +426,8 @@ impl Parser { } }) .collect::, ParseError>>(), - } + }?; + Ok(RecordTypeSpec::new(Some(braces), attrs)) } fn tuple_to_tuple_type_spec(tuple: Tuple) -> Result {