Skip to content
This repository was archived by the owner on Jun 15, 2023. It is now read-only.

Commit 25b87c9

Browse files
authored
Take more care lexing keywords (#29)
1 parent b5114d7 commit 25b87c9

File tree

2 files changed

+18
-28
lines changed

2 files changed

+18
-28
lines changed

src/SqlSquared/Parser/Tokenizer.purs

Lines changed: 17 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,11 @@ import Prelude
88

99
import Control.Alt ((<|>))
1010
import Data.Array as A
11-
import Data.Char as Ch
1211
import Data.Either (Either)
1312
import Data.HugeInt as HI
1413
import Data.HugeNum as HN
1514
import Data.Json.Extended.Signature.Parse as EJP
16-
import Data.Maybe (isJust)
15+
import Data.Set as Set
1716
import Data.String as S
1817
import SqlSquared.Utils ((∘))
1918
import Text.Parsing.Parser as P
@@ -94,8 +93,8 @@ op = map Op $ PC.choice
9493
, PS.string "%"
9594
]
9695

97-
keywords Array String
98-
keywords =
96+
keywords Set.Set String
97+
keywords = Set.fromFoldable
9998
[ "where"
10099
, "when"
101100
, "values"
@@ -152,8 +151,8 @@ keywords =
152151
digits Array Char
153152
digits = ['0','1','2','3','4','5','6','7','8','9' ]
154153

155-
ident m. Monad m P.ParserT String m Token
156-
ident = map Identifier $ PC.try quotedIdent <|> PC.try notQuotedIdent
154+
identOrKeyword m. Monad m P.ParserT String m Token
155+
identOrKeyword = PC.try quotedIdent <|> notQuotedIdentOrKeyword
157156

158157
oneLineComment m. Monad m P.ParserT String m Token
159158
oneLineComment =
@@ -178,25 +177,27 @@ multiLineComment = do
178177
_ →
179178
collectBeforeComment $ acc <> S.fromCharArray [ c ]
180179

181-
quotedIdent m. Monad m P.ParserT String m String
180+
quotedIdent m. Monad m P.ParserT String m Token
182181
quotedIdent =
183-
PC.between (PS.string "`") (PS.string "`")
184-
$ map S.fromCharArray
185-
$ A.some identChar
182+
map Identifier
183+
$ PC.between (PS.string "`") (PS.string "`")
184+
$ map S.fromCharArray
185+
$ A.some identChar
186186
where
187187
identChar = PC.try identEscape <|> identLetter
188188
identLetter = PS.satisfy (not ∘ eq '`')
189189
identEscape = PS.string "\\`" $> '`'
190190

191-
notQuotedIdent m. Monad m P.ParserT String m String
192-
notQuotedIdent = do
191+
notQuotedIdentOrKeyword m. Monad m P.ParserT String m Token
192+
notQuotedIdentOrKeyword = do
193193
first ← PT.letter
194194
other ← A.many (PT.alphaNum <|> PS.char '_')
195195
let
196196
str = S.fromCharArray $ A.cons first other
197-
if isJust $ A.elemIndex (S.toLower str) keywords
198-
then P.fail "unexpected keyword"
199-
else pure str
197+
low = S.toLower str
198+
pure if Set.member low keywords
199+
then Kw low
200+
else Identifier str
200201

201202
stringLit m. Monad m P.ParserT String m Token
202203
stringLit = LitString <$> EJP.parseStringLiteral
@@ -207,26 +208,14 @@ numLit = Lit ∘ Decimal <$> EJP.parseDecimalLiteral
207208
intLit m. Monad m P.ParserT String m Token
208209
intLit = LitInteger <$> EJP.parseHugeIntLiteral
209210

210-
keyword m. Monad m P.ParserT String m Token
211-
keyword = map Kw $ PC.choice $ map (PC.try ∘ parseKeyWord) keywords
212-
213-
parseKeyWord m. Monad m String P.ParserT String m String
214-
parseKeyWord s =
215-
map S.fromCharArray $ A.foldM foldFn [ ] $ S.toCharArray s
216-
where
217-
foldFn acc ch = do
218-
c ← PC.try $ PS.oneOf [ Ch.toUpper ch, Ch.toLower ch ]
219-
pure $ A.snoc acc c
220-
221211
tokens m. Monad m P.ParserT String m (Array Token)
222212
tokens = do
223213
PS.skipSpaces
224214
A.some $ PC.choice
225215
[ skipped oneLineComment
226216
, skipped multiLineComment
227217
, skipped op
228-
, skipped keyword
229-
, skipped ident
218+
, skipped identOrKeyword
230219
, skipped numLit
231220
, skipped intLit
232221
, skipped stringLit

test/src/Parse.purs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ select 12
5555
, E.Right """select * from foo LEFT JOIN bar on baz"""
5656
, E.Right """select * from foo RIGHT OUTER JOIN bar on baz"""
5757
, E.Right """select * from foo RIGHT JOIN bar on baz"""
58+
, E.Right """industry"""
5859
]
5960

6061
testSuite e. TestSuite (testOutput Console.TESTOUTPUT | e)

0 commit comments

Comments
 (0)