{{ Lex usage: lex filename [options] Reads filename.spn on SD card, writes tokens to filename.tok. }} con _stack = 1000 obj ios: "ios" kw: "keywords" str: "stringx" bt: "bintree" floatString: "FltStr" var byte parastr[64] pub Main | err, p ios.start err := \Try if err if err > 0 ios.print( err ) ios.printnl else ios.print( string("Error ") ) ios.printdec( err ) ios.printchar( 13 ) ios.printdec( TokLineNumber ) ios.printchar( "," ) ios.printdec( TokColumn + 1 ) ios.printchar( " " ) ios.printchar( "'" ) ios.print( pTokenText ) ios.printchar( "'" ) ios.printchar( 13 ) ios.sdclose bt.Stop ios.stop var long pTokenText pri Try bt.Init( 0, 0 ) ProcessCommandLine if verbosity => 1 ios.print( string("version 100225", 13) ) Start pri ProcessCommandLine | nArgs, l ios.parastart repeat while ios.paranext(@parastr) 'parameter einlesen if byte[@parastr][0] == "/" 'option? case byte[@parastr][1] "?": ios.print(string("help: lex /i fn.spn /o fn.tok [options]")) '/? ios.printnl abort "i": ifnot ios.paranext(@spinFilename) '/i fn.spn ios.print(string("error: input-filname")) ios.printnl "o": ifnot ios.paranext(@outputFilename) '/o fn.tok ios.print(string("error: output-filname")) ios.printnl "c": compile~~ "l": compile~~ "v": if ios.paranext(@parastr) '/v n verbosity := str2dec(@parastr) else ios.print(string("error: verbosity-level")) ios.printnl else ios.print(string("usage: lex /i fn.spn /o fn.tok [options]")) pri str2dec(stradr)|buffer,counter buffer := byte[stradr] counter := (strsize(stradr) <# 11) repeat while(counter--) result *= 10 result += lookdownz(byte[stradr++]: "0".."9") if(buffer == "-") -result con MAXFILENAMELENGTH = 8 + 1 + 3 ' 8.3 MAXSTRINGBUFFERLENGTH = 32 var byte spinFilename[MAXFILENAMELENGTH+1] ' input .spi file byte outputFilename[MAXFILENAMELENGTH+1] ' output .tok file byte stringBuffer[MAXSTRINGBUFFERLENGTH+1] ' temp string buffer dat verbosity byte 0 ' set by /V option compile byte 0 ' set by /C or /L: if non-zero, run compile.bin automatically. type long 0 lineNum word 0 column word 0 pri Start | v kw.Init pTokenText := TokGetPText if verbosity => 2 ios.print( string("Reading ") ) ios.print( @spinFilename ) ios.printchar( 13 ) TokOpen( @spinFilename ) if verbosity => 2 ios.print( string("Writing ") ) ios.print( @outputFilename ) ios.printchar( 13 ) if ios.sdnewfile( @outputFilename ) abort string("can't open o-file") ios.sdopen( "W", @outputFilename ) repeat type := TokType lineNum := TokLineNumber column := Column ios.sdputblk( 8, @type ) ' long + word + word if type == kw#kINTLITERAL or type == kw#kFLOATLITERAL v := TokValue ios.sdputblk( 4, @v ) else ios.sdputblk( strsize(pTokenText) + 1, pTokenText ) if TokType == kw#kEOF quit TokAdvance ios.sdclose dat 'TOKENIZER pub TokOpen( pFilename ) if ios.sdopen( "R", pFilename ) <> 0 abort string("can't open file") TokInitReadLine insideDAT~ TokAdvance ' prime the pump pub TokClose ios.sdclose { bytes~ InitReadLine GetNextToken ' prime the pump t := cnt repeat case tokenType EOL_TOKEN: term.str( @tokenText ) EOF_TOKEN: quit ID_TOKEN: term.str( @tokenText ) OP_TOKEN: INT_TOKEN: term.out( "#" ) term.dec( tokenValue ) FLOAT_TOKEN: term.out( "F" ) term.str( floatString.FloatToString( tokenValue ) ) term.out( "\" ) GetNextToken term.out( 13 ) '} { repeat while SkipWhitespace term.dec( lineNo ) term.out( "," ) term.dec( cp+1 ) term.out( ":" ) if cp == lineLength term.str( string("(eol)")) else term.out( lineBuffer[cp++] ) term.out( 13 ) '} { repeat r := sdfat.pread( @buf, BUFFERSIZE ) if r < 0 quit bytes += r '} { repeat 100 r := sdfat.pgetc if r < 0 quit ++bytes term.dec(r) term.out( "," ) '} con MAXTOKENLENGTH = 32 BUFFERSIZE = 200 var byte tokenText[MAXTOKENLENGTH+1] long tokenType word tokenLineNumber word tokenCol long tokenValue byte insideDAT pub TokGetPText return @tokenText pub TokType return tokenType pub TokColumn return tokenCol pub TokLineNumber return tokenLineNumber pub TokValue return tokenValue pub TokAdvance | i, radix, fp, x, hack if inStringLiteral if comma comma~ ifnot tokenValue abort string("Unterminated string") if lineBuffer[cp] <> $22 ' double-quote tokenType := kw#kCOMMA return ++cp inStringLiteral~ else comma~~ tokenType := kw#kINTLITERAL tokenValue := lineBuffer[cp++] return if TokSkipWhitespace if cp == lineLength tokenType := kw#kEOL bytemove( @tokenText, string("(EOL)"), 6 ) tokenLineNumber := lineNo tokenCol := cp elseif lineBuffer[cp] == $22 ' double-quote tokenLineNumber := LineNo tokenCol := cp ++cp if LineBuffer[cp] == $22 abort string("Empty string") comma~ inStringLiteral~~ TokAdvance elseif lineBuffer[cp] == "%" or lineBuffer[cp] == "$" tokenType := kw#kINTLITERAL tokenLineNumber := lineNo tokenCol := cp if lineBuffer[cp] == "%" if lineBuffer[cp+1] == "%" radix := 4 ++cp else radix := 2 else radix := 16 ++cp ifnot TokIsDigit( lineBuffer[cp], radix, false ) abort string("Bad character in number") tokenValue~ repeat while TokIsDigit( lineBuffer[cp], radix, true ) if lineBuffer[cp] <> "_" tokenValue := tokenValue * radix + TokDigitValue( linebuffer[cp] ) ++cp elseif TokIsDigit( lineBuffer[cp], 10, false ) tokenLineNumber := lineNo tokenCol := cp i~ fp~ {0 => integer 1 => have seen . 2 => have seen e or E 3 => have seen + or - } repeat while TokIsDigit( lineBuffer[cp], 10, true ) { } or ( fp < 1 and lineBuffer[cp] == "." and lineBuffer[cp+1] <> "." ) { } or ( fp < 2 and (lineBuffer[cp] == "e" or lineBuffer[cp] == "E") ) { } or ( (fp == 1 or fp == 2) and (lineBuffer[cp] == "+" or lineBuffer[cp] == "-") ) if lineBuffer[cp] == "." fp := 1 if lineBuffer[cp] == "e" or lineBuffer[cp] == "E" fp := 2 if lineBuffer[cp] == "+" or lineBuffer[cp] == "-" fp := 3 if lineBuffer[cp] <> "_" if i => MAXTOKENLENGTH abort string("Token too long") tokenText[i++] := lineBuffer[cp] ++cp tokenText[i]~ if fp tokenType := kw#kFLOATLITERAL tokenValue := floatString.StringToFloat( @tokenText ) else tokenType := kw#kINTLITERAL tokenValue~ i~ repeat while tokenText[i] tokenValue := tokenValue * 10 + TokDigitValue( tokenText[i++] ) elseif TokIsAlpha( lineBuffer[cp] ) i~ tokenLineNumber := lineNo tokenCol := cp repeat while TokIsAlpha( lineBuffer[cp] ) or TokIsDigit( lineBuffer[cp], 10, true ) if i => MAXTOKENLENGTH abort string("Token too long") tokenText[i++] := TokToUpper( lineBuffer[cp++] ) tokenText[i]~ x := kw.KeywordLookup( @tokenText ) if x tokenType := bt.PeekL( x ) if kw#kCON =< tokenType and tokenType =< kw#kVAR and tokenCol == 0 insideDAT := tokenType == kw#kDAT else tokenType := kw#kID else ' non-alphanumeric tokenLineNumber := lineNo tokenCol := cp i~ if lineBuffer[cp] < $20 or lineBuffer[cp] => $80 abort string("Illegal character (UNICODE?)") if insideDat and lineBuffer[cp] == ":" and TokIsAlpha( lineBuffer[cp+1] ) tokenText[i++] := lineBuffer[cp++] repeat while TokIsAlpha( lineBuffer[cp] ) or TokIsDigit( lineBuffer[cp], 10, true ) if i => MAXTOKENLENGTH abort string("Token too long") tokenText[i++] := TokToUpper( lineBuffer[cp++] ) tokenText[i]~ tokenType := kw#kID else repeat if cp < lineLength tokenText[i++] := lineBuffer[cp++] else tokenText[i++] := " " ++cp tokenText[i]~ while x := kw.KeywordLookup( @tokenText ) if i > 1 tokenText[--i]~ ' shorten token by 1 --cp tokenType := bt.PeekL( kw.KeywordLookup( @tokenText ) ) else tokenType := kw#kUNKNOWN if TokIsBinary ' This next block is a total hack to see if a binary op hack := blankLine ' hack is followed by "=" making it an assignment op TokSkipWhitespace ' It parses "and ==" as "(and=) =" instead of "(and)(==)" if lineBuffer[cp] == "=" ' PropTool does the latter. So this is wrong, but hopefully tokenText[i++] := lineBuffer[cp++] ' not *too* wrong. tokenText[i]~ tokenType |= kw#kASSIGNMENT else blankLine := hack ' restore (this hack is necessary to reset the eol condition trigger in SkipWhitespace) else tokenType := kw#kEOF bytemove( @tokenText, string("(EOF)"), 6 ) tokenLineNumber := lineNo tokenCol := cp pri TokToUpper( ch ) if "a" =< ch and ch =< "z" ch += constant( "A" - "a" ) return ch pri TokIsAlpha( ch ) return "a" =< ch and ch =< "z" or "A" =< ch and ch =< "Z" or ch == "_" pri TokIsDigit( ch, radix, u ) | upper {{ Returns non-zero if ch is an acceptable digit given radix (2, 4, 10, or 16), false otherwise. Pass u = true if the underscore ("_") is an acceptable value for ch. }} if ch == "_" return u upper := constant("0"-1) + radix if radix =< 10 return "0" =< ch and ch =< upper ' else radix == 16 return "0" =< ch and ch =< "9" or "a" =< ch and ch =< "f" or "A" =< ch and ch =< "F" pri TokDigitValue( ch ) if ch =< "9" return ch - "0" if ch =< "F" return ch + constant( 10 - "A" ) if ch =< "f" return ch + constant( 10 - "a" ) pub TokIsBinary {{ Returns non-zero (not necessarily -1) if token is a binary operator. Returns 0 otherwise. }} return ( tokenType & constant(kw#kUNARY|kw#kBINARY) ) == kw#kBINARY var byte lineBuffer[BUFFERSIZE] byte lineLength byte lineRemainder byte eolLength word lineNo byte cp byte blankLine byte inStringLiteral ' indicates we are splitting up a string literal byte comma ' flag for returning commas when splitting up a string literal pri TokSkipWhitespace '' Returns true on success, false on failure (eof). '' lineBuffer[cp] is the next non-whitespace character '' (comments count as whitespace). repeat repeat while cp => lineLength ifnot blankLine blankLine~~ return true ifnot TokReadLine return false if lineBuffer[cp] == 9 abort string("Illegal TAB character") if lineBuffer[cp] == "'" ' quote comment? cp := lineLength ' skip to end of line elseif lineBuffer[cp] == "}" abort string("Unexpected '}'") elseif lineBuffer[cp] == "{" TokSkipComment elseif lineBuffer[cp] == " " ++cp else ' non-blank character blankLine~ return true pri TokSkipComment | depth if cp+1 < lineLength and lineBuffer[cp+1] == "{" ++cp repeat repeat while cp => lineLength ifnot TokReadLine abort string("Unterminated comment") if cp+1 < lineLength and lineBuffer[cp] == "}" and lineBuffer[cp+1] == "}" cp += 2 return ++cp else ++cp depth := 1 repeat repeat while cp => lineLength ifnot TokReadLine abort string("Unterminated comment") if lineBuffer[cp] == "{" ++cp ++depth elseif lineBuffer[cp] == "}" ++cp ifnot --depth return else ++cp pri TokInitReadLine lineLength~ lineRemainder~ eolLength~ lineNo~ cp~ blankLine~~ inStringLiteral~ { |<--------------- BUFFERSIZE --------------->| +--------------------------------------------+ | | | | | +--------------------------------------------+ |<-lineLength ->| | | eolLength ->| |<- | |<- lineRemainder ->| } pri TokReadLine | bytes, i '' Reads a line into lineBuff. Returns true on success, false on failure (eof) '' The line in lineBuff is terminated with a null byte. Calling programs should '' not modify any memory in lineBuff after the null (the "remainder" section in '' the diagram above) because that's the following line(s). ' First, move remainder up to start of buffer bytemove( @lineBuffer, @lineBuffer[lineLength+eolLength], lineRemainder ) ' Fill the rest of the buffer with new data bytes := ios.sdgetblk( BUFFERSIZE-lineRemainder, @lineBuffer[lineRemainder] ) ' pread returns #bytes read, but after eof it returns negative numbers if bytes > 0 lineRemainder += bytes ifnot lineRemainder ++lineNo cp~ return false repeat i from 0 to (lineRemainder-2) #> 0 if lineBuffer[i] == 13 or lineBuffer[i] == 10 eolLength := 1 ' cr or lf if lineBuffer[i] == 13 and lineBuffer[i+1] == 10 ++eolLength ' cr+lf lineBuffer[i]~ ' set terminating null lineLength := i lineRemainder -= lineLength + eolLength ++lineNo ' first line of file is line 1 cp~ return true if lineRemainder < BUFFERSIZE lineLength := lineRemainder~ lineBuffer[lineLength]~ ++lineNo cp~ return true abort string("Input line too long") {{ Copyright (c) 2009 Michael Park +------------------------------------------------------------------------------------------------------------------------------+ | TERMS OF USE: MIT License | +------------------------------------------------------------------------------------------------------------------------------+ |Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation | |files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, | |modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software| |is furnished to do so, subject to the following conditions: | | | |The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.| | | |THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE | |WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR | |COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. | +------------------------------------------------------------------------------------------------------------------------------+ }}