From 9821c321830ad7f6070f6a9f663c3a0c5739edf2 Mon Sep 17 00:00:00 2001 From: NeilKleistGao Date: Sun, 15 Oct 2023 00:07:16 +0800 Subject: [PATCH] WIP: Add new syntax --- shared/src/main/scala/mlscript/NewLexer.scala | 4 + .../src/main/scala/mlscript/NewParser.scala | 118 ++++++++++++++---- shared/src/main/scala/mlscript/Token.scala | 2 + shared/src/test/diff/qq/NuSyntax.mls | 46 +++++++ shared/src/test/diff/qq/PEPM2.mls | 8 ++ 5 files changed, 156 insertions(+), 22 deletions(-) create mode 100644 shared/src/test/diff/qq/NuSyntax.mls create mode 100644 shared/src/test/diff/qq/PEPM2.mls diff --git a/shared/src/main/scala/mlscript/NewLexer.scala b/shared/src/main/scala/mlscript/NewLexer.scala index 68e84bf99..6383d2446 100644 --- a/shared/src/main/scala/mlscript/NewLexer.scala +++ b/shared/src/main/scala/mlscript/NewLexer.scala @@ -41,6 +41,7 @@ class NewLexer(origin: Origin, raise: Diagnostic => Unit, dbg: Bool) { ":", ";;", "#", + "`" // ".", // "<", // ">", @@ -115,6 +116,8 @@ class NewLexer(origin: Origin, raise: Diagnostic => Unit, dbg: Bool) { val j = i + 1 // go(j, COMMA) lex(j, ind, next(j, COMMA)) + case '`' => + lex(i + 1, ind, next(i + 1, QUOTE)) case 'c' if isQuasiquoteOpening(i) || isQuasiquoteTripleOpening(i) => val isTripleQuoteQQ = isQuasiquoteTripleOpening(i) val bracket_kind = if (isTripleQuoteQQ) @@ -363,6 +366,7 @@ object NewLexer { case (INDENT, _) => "→" case (DEINDENT, _) => "←" case (ERROR, _) => "" + case (QUOTE, _) => "`" case (LITVAL(lv), _) => lv.idStr case (KEYWORD(name: String), _) => "#" + name case (IDENT(name: String, symbolic: Bool), _) => name diff --git a/shared/src/main/scala/mlscript/NewParser.scala b/shared/src/main/scala/mlscript/NewParser.scala index 65ccd6a73..98d26c956 100644 --- a/shared/src/main/scala/mlscript/NewParser.scala +++ b/shared/src/main/scala/mlscript/NewParser.scala @@ -12,6 +12,7 @@ object NewParser { type ExpectThen >: Bool type FoundErr >: Bool // may be better done as: class FoundErr(var found: Bool) + type IsQuoted >: Bool final def expectThen(implicit ptr: ExpectThen): Bool = ptr === true final def foundErr(implicit ptr: FoundErr): Bool = ptr === true @@ -221,8 +222,8 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo } */ - final def typingUnit: TypingUnit = { - val ts = block(false, false) + final def typingUnit(implicit qenv: Set[Str] = Set.empty, quoted: IsQuoted = false): TypingUnit = { + val ts = block(false, false, qenv, quoted) val es = ts.map { case L(t) => err(msg"Unexpected 'then'/'else' clause" -> t.toLoc :: Nil) @@ -234,13 +235,13 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo } TypingUnit(es) } - final def typingUnitMaybeIndented(implicit fe: FoundErr): TypingUnit = yeetSpaces match { + final def typingUnitMaybeIndented(implicit fe: FoundErr, qenv: Set[Str], quoted: IsQuoted): TypingUnit = yeetSpaces match { case (br @ BRACKETS(Indent, toks), _) :: _ => consume rec(toks, S(br.innerLoc), br.describe).concludeWith(_.typingUnit) case _ => typingUnit } - final def curlyTypingUnit(implicit fe: FoundErr): TypingUnit = yeetSpaces match { + final def curlyTypingUnit(implicit fe: FoundErr, qenv: Set[Str], quoted: IsQuoted): TypingUnit = yeetSpaces match { case (br @ BRACKETS(Curly, toks), l1) :: _ => consume rec(toks, S(br.innerLoc), br.describe).concludeWith(_.typingUnitMaybeIndented).withLoc(S(l1)) @@ -252,7 +253,7 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo case t: Tuple => t case _ => Tuple((N, Field(None, t)) :: Nil) } - final def typ(prec: Int = 0)(implicit fe: FoundErr, l: Line): Type = + final def typ(prec: Int = 0)(implicit fe: FoundErr, l: Line, qenv: Set[Str], quoted: IsQuoted): Type = mkType(expr(prec)) case class ModifierSet(mods: Map[Str, Loc]) { @@ -299,7 +300,7 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo S(res, _cur) } } - final def block(implicit et: ExpectThen, fe: FoundErr): Ls[IfBody \/ Statement] = + final def block(implicit et: ExpectThen, fe: FoundErr, qenv: Set[Str], quoted: IsQuoted): Ls[IfBody \/ Statement] = cur match { case Nil => Nil case (NEWLINE, _) :: _ => consume; block @@ -546,7 +547,7 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo || tkloc._1.isInstanceOf[COMMENT] // TODO properly retrieve and sotre all comments in AST? ) && { consume; true }) - final def funParams(implicit et: ExpectThen, fe: FoundErr, l: Line): Ls[Tup] = wrap(()) { l => + final def funParams(implicit et: ExpectThen, fe: FoundErr, l: Line, qenv: Set[Str], quoted: IsQuoted): Ls[Tup] = wrap(()) { l => yeetSpaces match { case (KEYWORD("=" | ":"), _) :: _ => Nil case Nil => Nil @@ -567,8 +568,8 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo } } - final def expr(prec: Int, allowSpace: Bool = true)(implicit fe: FoundErr, l: Line): Term = wrap(prec,allowSpace) { l => - exprOrIf(prec, allowSpace)(et = false, fe = fe, l = implicitly) match { + final def expr(prec: Int, allowSpace: Bool = true)(implicit fe: FoundErr, l: Line, qenv: Set[Str], quoted: IsQuoted): Term = wrap(prec,allowSpace) { l => + exprOrIf(prec, allowSpace)(et = false, fe = fe, l = implicitly, qenv = qenv, quoted = false) match { case R(e) => e case L(e) => err(msg"Expected an expression; found a 'then'/'else' clause instead" -> e.toLoc :: Nil) @@ -580,7 +581,7 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo raise(WarningReport(msg"[${cur.headOption.map(_._1).mkString}] ${""+msg}" -> loco :: Nil, newDefs = true)) - final def exprOrIf(prec: Int, allowSpace: Bool = true)(implicit et: ExpectThen, fe: FoundErr, l: Line): IfBody \/ Term = wrap(prec, allowSpace) { l => + final def exprOrIf(prec: Int, allowSpace: Bool = true)(implicit et: ExpectThen, fe: FoundErr, l: Line, qenv: Set[Str], quoted: IsQuoted): IfBody \/ Term = wrap(prec, allowSpace) { l => cur match { case (SPACE, l0) :: _ if allowSpace => // Q: do we really need the `allowSpace` flag? consume @@ -593,6 +594,27 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo val ts = rec(toks, S(br.innerLoc), br.describe).concludeWith(_.block) val es = ts.map { case L(t) => return L(IfBlock(ts)); case R(e) => e } R(Blk(es)) + case (QUOTE, loc) :: _ if quoted === false => + consume + cur match { + case (QUOTE, l0) :: _ => + err(( + msg"Nested quotation is not supported yet." -> S(l0) :: Nil)) + R(Var("")) + case (LITVAL(lit), l0) :: _ => + consume + exprCont(Quoted(lit.withLoc(S(l0))).withLoc(S(loc)), prec, allowNewlines = false) + case (br @ BRACKETS(bk @ (Round | Square | Curly), toks), loc) :: _ => + consume + val res = rec(toks, S(br.innerLoc), br.describe).concludeWith(_.argsMaybeIndented()) + val bra = (bk, res) match { + case (Round, (N, Fld(FldFlags(false, false, _), elt)) :: Nil) => + Quoted(Bra(false, elt)) + case _ => ??? // TODO + } + exprCont(bra.withLoc(S(loc)), prec, allowNewlines = false) + case _ => ??? + } case (LITVAL(lit), l0) :: _ => consume exprCont(lit.withLoc(S(l0)), prec, allowNewlines = false) @@ -601,7 +623,8 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo exprCont(UnitLit(kwStr === "undefined").withLoc(S(l0)), prec, allowNewlines = false) case (IDENT(nme, false), l0) :: _ => consume - exprCont(Var(nme).withLoc(S(l0)), prec, allowNewlines = false) + val res = if (qenv.contains(nme)) Quoted(Var(nme)) else Var(nme) + exprCont(res.withLoc(S(l0)), prec, allowNewlines = false) case (br @ BRACKETS(Quasiquote | QuasiquoteTriple, toks), loc) :: _ => consume val body = rec(toks, S(br.innerLoc), br.describe).concludeWith(_.expr(0)) @@ -641,6 +664,11 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo consume val e = expr(0) Lam(Tup(res), e) + case (QUOTE, l0) :: (KEYWORD("=>"), l1) :: _ => + exprCont(Tup(res), 0, true) match { + case L(_) => ??? + case R(t) => t + } case (IDENT("->", true), l1) :: _ => consume val rhs = expr(opPrec("->")._2) @@ -725,7 +753,7 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo L(IfElse(e).withLoc(S(l0 ++ e.toLoc))) case (KEYWORD("case"), l0) :: _ => consume - exprOrIf(0)(et = true, fe = fe, l = implicitly) match { + exprOrIf(0)(et = true, fe = fe, l = implicitly, qenv = qenv, quoted = quoted) match { case L(body) => R(Lam(PlainTup(Var("case$scrut")), If(IfOpApp(Var("case$scrut"), Var("is"), body), N))) case R(rhs) => @@ -738,7 +766,7 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo consume cur match { case _ => - exprOrIf(0)(et = true, fe = fe, l = implicitly) match { + exprOrIf(0)(et = true, fe = fe, l = implicitly, qenv = qenv, quoted = quoted) match { case L(body) => val els = yeetSpaces match { case (KEYWORD("else"), _) :: _ => @@ -810,15 +838,61 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo case (tk, l0) :: _ => err(msg"Unexpected ${tk.describe} in expression position" -> S(l0) :: Nil) consume - exprOrIf(prec)(et = et, fe = true, l = implicitly) + exprOrIf(prec)(et = et, fe = true, l = implicitly, qenv = qenv, quoted = quoted) }} private def errExpr = // Tup(Nil).withLoc(lastLoc) // TODO FIXME produce error term instead UnitLit(true).withLoc(lastLoc) // TODO FIXME produce error term instead + + private def quoteOp(lhs: Term, rhs: Term, op: Var) = (lhs, rhs) match { + case (Quoted(lhs), Quoted(rhs)) => Quoted(App(op, PlainTup(lhs, rhs))) + case (_, Quoted(rhs)) => Quoted(App(op, PlainTup(Unquoted(lhs), rhs))) + case (Quoted(lhs), _) => Quoted(App(op, PlainTup(lhs, Unquoted(rhs)))) + case _ => Quoted(App(op, PlainTup(Unquoted(lhs), Unquoted(rhs)))) + } + + private def declQenv(params: Term) = params match { + case Var(name) => Set[Str](name) + case Tup(params) => params.map { + case N -> Fld(_, Var(nme)) => nme + case S(Var(nme)) -> _ => nme + case _ => ??? + }.toSet + case _ => Set.empty + } - final def exprCont(acc: Term, prec: Int, allowNewlines: Bool)(implicit et: ExpectThen, fe: FoundErr, l: Line): IfBody \/ Term = wrap(prec, s"`$acc`", allowNewlines) { l => + final def exprCont(acc: Term, prec: Int, allowNewlines: Bool)(implicit et: ExpectThen, fe: FoundErr, l: Line, qenv: Set[Str], quoted: IsQuoted): IfBody \/ Term = wrap(prec, s"`$acc`", allowNewlines) { l => cur match { + case (QUOTE, _) :: _ if quoted === false => // TODO: refactor + consume + cur match { + case (KEYWORD(opStr @ "=>"), l0) :: _ if opPrec(opStr)._1 > prec => + consume + val newQenv = declQenv(acc) + val rhs = expr(1)(fe, implicitly, qenv ++ newQenv, quoted) + val param = acc match { + case t: Tup => t + case _ => PlainTup(acc) + } + val res = rhs match { + case Quoted(rhs) => Quoted(Lam(param, rhs)) + case _ => Quoted(Lam(param, Unquoted(rhs))) + } + exprCont(res, prec, allowNewlines) + case (IDENT(opStr, true), l0) :: _ if /* isInfix(opStr) && */ opPrec(opStr)._1 > prec => + consume + val v = Var(opStr).withLoc(S(l0)) + // printDbg(s">>> $opStr ${opPrec(opStr)}") + exprOrIf(opPrec(opStr)._2) match { + case L(rhs) => ??? + // TODO: L(IfOpApp(acc, v, rhs)) + case R(rhs) => + // TODO: match opStr + exprCont(quoteOp(acc, rhs, v), prec, allowNewlines) + } + case _ => ??? + } case (KEYWORD(opStr @ "=>"), l0) :: (NEWLINE, l1) :: _ if opPrec(opStr)._1 > prec => consume val rhs = Blk(typingUnit.entities) @@ -998,7 +1072,7 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo } } - final def opBlock(acc: Term, opStr: Str, opLoc: Loc)(implicit et: ExpectThen, fe: FoundErr, l: Line): IfBody \/ Term = wrap(s"`$acc`", opStr) { l => + final def opBlock(acc: Term, opStr: Str, opLoc: Loc)(implicit et: ExpectThen, fe: FoundErr, l: Line, qenv: Set[Str], quoted: IsQuoted): IfBody \/ Term = wrap(s"`$acc`", opStr) { l => val opv = Var(opStr).withLoc(S(opLoc)) val rhs = exprOrIf(0) // val rhs = exprOrIf(1) @@ -1026,7 +1100,7 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo L(IfOpsApp(acc, opIfBlock(opv -> rhs :: Nil))) } } - final def opIfBlock(acc: Ls[Var -> IfBody])(implicit et: ExpectThen, fe: FoundErr): Ls[Var -> IfBody] = wrap(acc) { l => + final def opIfBlock(acc: Ls[Var -> IfBody])(implicit et: ExpectThen, fe: FoundErr, qenv: Set[Str], quoted: IsQuoted): Ls[Var -> IfBody] = wrap(acc) { l => cur match { case (NEWLINE, _) :: c => // TODO allow let bindings... consume @@ -1102,7 +1176,7 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo case _ => f(this, false) } - final def argsMaybeIndented()(implicit fe: FoundErr, et: ExpectThen): Ls[Opt[Var] -> Fld] = + final def argsMaybeIndented()(implicit fe: FoundErr, et: ExpectThen, qenv: Set[Str], quoted: IsQuoted): Ls[Opt[Var] -> Fld] = maybeIndented(_.args(_)) // final def argsMaybeIndented()(implicit fe: FoundErr, et: ExpectThen): Ls[Opt[Var] -> Fld] = // cur match { @@ -1116,9 +1190,9 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo // } // TODO support comma-less arg blocks...? - final def args(allowNewlines: Bool, prec: Int = NoElsePrec)(implicit fe: FoundErr, et: ExpectThen): Ls[Opt[Var] -> Fld] = + final def args(allowNewlines: Bool, prec: Int = NoElsePrec)(implicit fe: FoundErr, et: ExpectThen, qenv: Set[Str], quoted: IsQuoted): Ls[Opt[Var] -> Fld] = // argsOrIf(Nil).map{case (_, L(x))=> ???; case (n, R(x))=>n->x} // TODO - argsOrIf(Nil, Nil, allowNewlines, prec).flatMap{case (n, L(x))=> + argsOrIf(Nil, Nil, allowNewlines, prec)(fe, et, qenv, quoted).flatMap{case (n, L(x))=> err(msg"Unexpected 'then'/'else' clause" -> x.toLoc :: Nil) n->Fld(FldFlags.empty, errExpr)::Nil case (n, R(x))=>n->x::Nil} // TODO @@ -1134,7 +1208,7 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo } */ final def argsOrIf(acc: Ls[Opt[Var] -> (IfBody \/ Fld)], seqAcc: Ls[Statement], allowNewlines: Bool, prec: Int = NoElsePrec) - (implicit fe: FoundErr, et: ExpectThen): Ls[Opt[Var] -> (IfBody \/ Fld)] = + (implicit fe: FoundErr, et: ExpectThen, qenv: Set[Str], quoted: IsQuoted): Ls[Opt[Var] -> (IfBody \/ Fld)] = wrap(acc, seqAcc) { l => cur match { @@ -1230,7 +1304,7 @@ abstract class NewParser(origin: Origin, tokens: Ls[Stroken -> Loc], newDefs: Bo case (IDENT(id, false), l0) :: _ => consume val (success, _) = skip(KEYWORD("=")) // TODO kw? - val rhs = expr(0)(fe = foundErr || !success, l = implicitly) + val rhs = expr(0)(fe = foundErr || !success, l = implicitly, qenv = Set.empty, quoted = false) // TODO // cur.dropWhile(_ === SPACE) match { // case (KEYWORD("in"), _) :: _ => // acc.reverse diff --git a/shared/src/main/scala/mlscript/Token.scala b/shared/src/main/scala/mlscript/Token.scala index 72638db76..a9ee48ef1 100644 --- a/shared/src/main/scala/mlscript/Token.scala +++ b/shared/src/main/scala/mlscript/Token.scala @@ -12,6 +12,7 @@ sealed abstract class Token { case INDENT => "indentation" case DEINDENT => "deindentation" case ERROR => "error" + case QUOTE => "quote" case LITVAL(value) => "literal" case KEYWORD(name) => if (name.headOption.exists(_.isLetter)) s"'$name' keyword" else s"'$name'" @@ -35,6 +36,7 @@ case object NEWLINE extends Token with Stroken // TODO rm case object INDENT extends Token case object DEINDENT extends Token case object ERROR extends Token with Stroken +case object QUOTE extends Token with Stroken final case class LITVAL(value: Lit) extends Token with Stroken final case class KEYWORD(name: String) extends Token with Stroken final case class IDENT(name: String, symbolic: Bool) extends Token with Stroken diff --git a/shared/src/test/diff/qq/NuSyntax.mls b/shared/src/test/diff/qq/NuSyntax.mls new file mode 100644 index 000000000..7652b7d2f --- /dev/null +++ b/shared/src/test/diff/qq/NuSyntax.mls @@ -0,0 +1,46 @@ +:NewDefs +:NoJS + +`42 +//│ Code[42, nothing] + +x `=> x +//│ Code[forall 'a. 'a -> 'a, nothing] + +x `=> + x +//│ Code[forall 'a. 'a -> 'a, nothing] + +run(x `=> x)(42) +//│ 42 + +`1 `+ `1 +//│ Code[Int, nothing] + +`(1 + 1) +//│ Code[Int, nothing] + +x `=> x `+ `1 +//│ Code[Int -> Int, nothing] + +y `=> `0 `- y +//│ Code[Int -> Int, nothing] + +:e +z `=> z + 1 +//│ ╔══[ERROR] Type mismatch in operator application: +//│ ║ l.30: z `=> z + 1 +//│ ║ ^^^^^ +//│ ╙── expression of type `Code[?a, ?z]` is not an instance of `Int` +//│ ╔══[ERROR] Type mismatch in unquote: +//│ ║ l.30: z `=> z + 1 +//│ ║ ^^^^^ +//│ ╙── operator application of type `Int` is not an instance of `Code[?a, ?z | ?b]` +//│ Code[anything -> error, nothing] + + +(x, y) `=> x `+ y +//│ Code[(Int, Int) -> Int, nothing] + +run((x, y) `=> x `+ y)(1, 2) +//│ Int diff --git a/shared/src/test/diff/qq/PEPM2.mls b/shared/src/test/diff/qq/PEPM2.mls new file mode 100644 index 000000000..ad0433f55 --- /dev/null +++ b/shared/src/test/diff/qq/PEPM2.mls @@ -0,0 +1,8 @@ +:NewDefs +:NoJS + + +fun power(x) = case + 0 then `1 + n then x `* power(x)(n - 1) +//│ fun power: forall 'a. Code[Int, 'a] -> (0 | Int & ~0) -> Code[Int, 'a]