1636 ! |
1636 ! |
1637 |
1637 |
1638 nextToken |
1638 nextToken |
1639 "return the next token from the source-stream" |
1639 "return the next token from the source-stream" |
1640 |
1640 |
1641 |skipping actionBlock v ch| |
1641 |skipping actionBlock v ch tok| |
1642 |
1642 |
1643 peekChar notNil ifTrue:[ |
1643 [true] whileTrue:[ |
1644 peekChar isSeparator ifTrue:[ |
1644 peekChar notNil ifTrue:[ |
1645 peekChar == (Character cr) ifTrue:[ |
1645 peekChar isSeparator ifTrue:[ |
1646 lineNr := lineNr + 1. |
1646 peekChar == (Character cr) ifTrue:[ |
1647 ]. |
1647 lineNr := lineNr + 1. |
|
1648 ]. |
|
1649 peekChar := peekChar2. |
|
1650 peekChar2 := nil. |
|
1651 ]. |
|
1652 ]. |
|
1653 |
|
1654 peekChar notNil ifTrue:[ |
|
1655 ch := peekChar. |
1648 peekChar := peekChar2. |
1656 peekChar := peekChar2. |
1649 peekChar2 := nil. |
1657 peekChar2 := nil. |
1650 ]. |
1658 hereChar := nil. |
1651 ]. |
1659 ] ifFalse:[ |
1652 |
1660 skipping := true. |
1653 peekChar notNil ifTrue:[ |
1661 [skipping] whileTrue:[ |
1654 ch := peekChar. |
1662 |
1655 peekChar := peekChar2. |
1663 outStream notNil ifTrue:[ |
1656 peekChar2 := nil. |
1664 hereChar := source peekOrNil. |
1657 hereChar := nil. |
1665 [(hereChar == Character space) |
1658 ] ifFalse:[ |
1666 or:[hereChar isSeparator] |
1659 skipping := true. |
1667 ] whileTrue:[ |
1660 [skipping] whileTrue:[ |
1668 source next. |
1661 |
1669 outStream space. |
1662 outStream notNil ifTrue:[ |
1670 outCol := outCol + 1. |
1663 hereChar := source peekOrNil. |
1671 ] |
1664 [(hereChar == Character space) |
1672 ] ifFalse:[ |
1665 or:[hereChar isSeparator] |
1673 hereChar := source skipSeparatorsExceptCR. |
1666 ] whileTrue:[ |
1674 ]. |
|
1675 |
|
1676 hereChar == (Character cr) ifTrue:[ |
|
1677 lineNr := lineNr + 1. |
1667 source next. |
1678 source next. |
1668 outStream space. |
|
1669 outCol := outCol + 1. |
|
1670 ] |
|
1671 ] ifFalse:[ |
|
1672 hereChar := source skipSeparatorsExceptCR. |
|
1673 ]. |
|
1674 |
|
1675 hereChar == (Character cr) ifTrue:[ |
|
1676 lineNr := lineNr + 1. |
|
1677 source next. |
|
1678 outStream notNil ifTrue:[ |
|
1679 outStream cr. |
|
1680 outCol := 1 |
|
1681 ] |
|
1682 ] ifFalse:[ |
|
1683 hereChar == (Character return) ifTrue:[ |
|
1684 outStream notNil ifTrue:[ |
1679 outStream notNil ifTrue:[ |
1685 outStream nextPut:hereChar. |
1680 outStream cr. |
1686 outCol := 1 |
1681 outCol := 1 |
1687 ]. |
1682 ] |
1688 source next. |
|
1689 ] ifFalse:[ |
1683 ] ifFalse:[ |
1690 (self isCommentCharacter:hereChar) ifTrue:[ |
1684 hereChar == (Character return) ifTrue:[ |
1691 "start of a comment" |
1685 outStream notNil ifTrue:[ |
1692 |
1686 outStream nextPut:hereChar. |
1693 self skipComment. |
1687 outCol := 1 |
1694 hereChar := source peekOrNil. |
1688 ]. |
|
1689 source next. |
1695 ] ifFalse:[ |
1690 ] ifFalse:[ |
1696 skipping := false |
1691 (self isCommentCharacter:hereChar) ifTrue:[ |
|
1692 "start of a comment" |
|
1693 |
|
1694 self skipComment. |
|
1695 hereChar := source peekOrNil. |
|
1696 ] ifFalse:[ |
|
1697 skipping := false |
|
1698 ] |
1697 ] |
1699 ] |
1698 ] |
1700 ] |
1699 ] |
1701 ]. |
1700 ]. |
1702 hereChar isNil ifTrue:[ |
1701 hereChar isNil ifTrue:[ |
1703 token := nil. |
1702 token := nil. |
1704 tokenType := #EOF. |
1703 tokenType := #EOF. |
1705 ^ tokenType |
1704 ^ tokenType |
1706 ]. |
1705 ]. |
1707 ch := hereChar |
1706 ch := hereChar |
1708 ]. |
1707 ]. |
1709 tokenPosition := source position. |
1708 tokenPosition := source position. |
1710 tokenLineNr := lineNr. |
1709 tokenLineNr := lineNr. |
1711 |
1710 |
1712 (v := ch asciiValue) == 0 ifTrue:[ |
1711 (v := ch asciiValue) == 0 ifTrue:[ |
1713 v := Character space asciiValue |
1712 v := Character space asciiValue |
1714 ]. |
1713 ]. |
1715 actionBlock := actionArray at:v. |
1714 actionBlock := actionArray at:v. |
1716 actionBlock notNil ifTrue:[ |
1715 actionBlock notNil ifTrue:[ |
1717 tok := actionBlock value:self value:ch. |
1716 ^ actionBlock value:self value:ch |
1718 tok notNil ifTrue:[^ tok]. |
1717 ]. |
1719 ] ifFalse:[ |
1718 |
1720 self syntaxError:('invalid character: ''' , ch asString , ''' ', |
1719 self syntaxError:('invalid character: ''' , ch asString , ''' ', |
1721 '(' , v printString , ')') |
1720 '(' , v printString , ')') |
1722 position:tokenPosition to:tokenPosition. |
1721 position:tokenPosition to:tokenPosition. |
1723 source next. |
1722 source next. |
1724 tokenName := nil. |
1723 tokenName := nil. |
1725 tokenType := token := #Error. |
1724 tokenType := token := #Error. |
1726 ^ #Error |
1725 ^ #Error |
1727 ] |
|
1728 ]. |
1726 |
1729 |
1727 "Modified: / 13.9.1995 / 12:56:14 / claus" |
1730 "Modified: / 13.9.1995 / 12:56:14 / claus" |
1728 "Modified: / 18.6.1998 / 23:11:24 / cg" |
1731 "Modified: / 22.10.1998 / 22:15:27 / cg" |
1729 ! |
1732 ! |
1730 |
1733 |
1731 nextToken:aCharacter |
1734 nextToken:aCharacter |
1732 tokenType := token := aCharacter. |
1735 tokenType := token := aCharacter. |
1733 hereChar notNil ifTrue:[source next]. |
1736 hereChar notNil ifTrue:[source next]. |