1 /* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd

     2    See the file COPYING for copying permission.

     3 */

     4 

     5 /* This file is included! */

     6 #ifdef XML_TOK_IMPL_C

     7 

     8 #ifndef IS_INVALID_CHAR

     9 #define IS_INVALID_CHAR(enc, ptr, n) (0)

    10 #endif

    11 

    12 #define INVALID_LEAD_CASE(n, ptr, nextTokPtr) \

    13     case BT_LEAD ## n: \

    14       if (end - ptr < n) \

    15         return XML_TOK_PARTIAL_CHAR; \

    16       if (IS_INVALID_CHAR(enc, ptr, n)) { \

    17         *(nextTokPtr) = (ptr); \

    18         return XML_TOK_INVALID; \

    19       } \

    20       ptr += n; \

    21       break;

    22 

    23 #define INVALID_CASES(ptr, nextTokPtr) \

    24   INVALID_LEAD_CASE(2, ptr, nextTokPtr) \

    25   INVALID_LEAD_CASE(3, ptr, nextTokPtr) \

    26   INVALID_LEAD_CASE(4, ptr, nextTokPtr) \

    27   case BT_NONXML: \

    28   case BT_MALFORM: \

    29   case BT_TRAIL: \

    30     *(nextTokPtr) = (ptr); \

    31     return XML_TOK_INVALID;

    32 

    33 #define CHECK_NAME_CASE(n, enc, ptr, end, nextTokPtr) \

    34    case BT_LEAD ## n: \

    35      if (end - ptr < n) \

    36        return XML_TOK_PARTIAL_CHAR; \

    37      if (!IS_NAME_CHAR(enc, ptr, n)) { \

    38        *nextTokPtr = ptr; \

    39        return XML_TOK_INVALID; \

    40      } \

    41      ptr += n; \

    42      break;

    43 

    44 #define CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) \

    45   case BT_NONASCII: \

    46     if (!IS_NAME_CHAR_MINBPC(enc, ptr)) { \

    47       *nextTokPtr = ptr; \

    48       return XML_TOK_INVALID; \

    49     } \

    50   case BT_NMSTRT: \

    51   case BT_HEX: \

    52   case BT_DIGIT: \

    53   case BT_NAME: \

    54   case BT_MINUS: \

    55     ptr += MINBPC(enc); \

    56     break; \

    57   CHECK_NAME_CASE(2, enc, ptr, end, nextTokPtr) \

    58   CHECK_NAME_CASE(3, enc, ptr, end, nextTokPtr) \

    59   CHECK_NAME_CASE(4, enc, ptr, end, nextTokPtr)

    60 

    61 #define CHECK_NMSTRT_CASE(n, enc, ptr, end, nextTokPtr) \

    62    case BT_LEAD ## n: \

    63      if (end - ptr < n) \

    64        return XML_TOK_PARTIAL_CHAR; \

    65      if (!IS_NMSTRT_CHAR(enc, ptr, n)) { \

    66        *nextTokPtr = ptr; \

    67        return XML_TOK_INVALID; \

    68      } \

    69      ptr += n; \

    70      break;

    71 

    72 #define CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) \

    73   case BT_NONASCII: \

    74     if (!IS_NMSTRT_CHAR_MINBPC(enc, ptr)) { \

    75       *nextTokPtr = ptr; \

    76       return XML_TOK_INVALID; \

    77     } \

    78   case BT_NMSTRT: \

    79   case BT_HEX: \

    80     ptr += MINBPC(enc); \

    81     break; \

    82   CHECK_NMSTRT_CASE(2, enc, ptr, end, nextTokPtr) \

    83   CHECK_NMSTRT_CASE(3, enc, ptr, end, nextTokPtr) \

    84   CHECK_NMSTRT_CASE(4, enc, ptr, end, nextTokPtr)

    85 

    86 #ifndef PREFIX

    87 #define PREFIX(ident) ident

    88 #endif

    89 

    90 /* ptr points to character following "<!-" */

    91 

    92 static int PTRCALL

    93 PREFIX(scanComment)(const ENCODING *enc, const char *ptr,

    94                     const char *end, const char **nextTokPtr)

    95 {

    96   if (ptr != end) {

    97     if (!CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {

    98       *nextTokPtr = ptr;

    99       return XML_TOK_INVALID;

   100     }

   101     ptr += MINBPC(enc);

   102     while (ptr != end) {

   103       switch (BYTE_TYPE(enc, ptr)) {

   104       INVALID_CASES(ptr, nextTokPtr)

   105       case BT_MINUS:

   106         if ((ptr += MINBPC(enc)) == end)

   107           return XML_TOK_PARTIAL;

   108         if (CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {

   109           if ((ptr += MINBPC(enc)) == end)

   110             return XML_TOK_PARTIAL;

   111           if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {

   112             *nextTokPtr = ptr;

   113             return XML_TOK_INVALID;

   114           }

   115           *nextTokPtr = ptr + MINBPC(enc);

   116           return XML_TOK_COMMENT;

   117         }

   118         break;

   119       default:

   120         ptr += MINBPC(enc);

   121         break;

   122       }

   123     }

   124   }

   125   return XML_TOK_PARTIAL;

   126 }

   127 

   128 /* ptr points to character following "<!" */

   129 

   130 static int PTRCALL

   131 PREFIX(scanDecl)(const ENCODING *enc, const char *ptr,

   132                  const char *end, const char **nextTokPtr)

   133 {

   134   if (ptr == end)

   135     return XML_TOK_PARTIAL;

   136   switch (BYTE_TYPE(enc, ptr)) {

   137   case BT_MINUS:

   138     return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   139   case BT_LSQB:

   140     *nextTokPtr = ptr + MINBPC(enc);

   141     return XML_TOK_COND_SECT_OPEN;

   142   case BT_NMSTRT:

   143   case BT_HEX:

   144     ptr += MINBPC(enc);

   145     break;

   146   default:

   147     *nextTokPtr = ptr;

   148     return XML_TOK_INVALID;

   149   }

   150   while (ptr != end) {

   151     switch (BYTE_TYPE(enc, ptr)) {

   152     case BT_PERCNT:

   153       if (ptr + MINBPC(enc) == end)

   154         return XML_TOK_PARTIAL;

   155       /* don't allow <!ENTITY% foo "whatever"> */

   156       switch (BYTE_TYPE(enc, ptr + MINBPC(enc))) {

   157       case BT_S: case BT_CR: case BT_LF: case BT_PERCNT:

   158         *nextTokPtr = ptr;

   159         return XML_TOK_INVALID;

   160       }

   161       /* fall through */

   162     case BT_S: case BT_CR: case BT_LF:

   163       *nextTokPtr = ptr;

   164       return XML_TOK_DECL_OPEN;

   165     case BT_NMSTRT:

   166     case BT_HEX:

   167       ptr += MINBPC(enc);

   168       break;

   169     default:

   170       *nextTokPtr = ptr;

   171       return XML_TOK_INVALID;

   172     }

   173   }

   174   return XML_TOK_PARTIAL;

   175 }

   176 

   177 static int PTRCALL

   178 PREFIX(checkPiTarget)(const ENCODING *enc, const char *ptr,

   179                       const char *end, int *tokPtr)

   180 {

   181   int upper = 0;

   182   *tokPtr = XML_TOK_PI;

   183   if (end - ptr != MINBPC(enc)*3)

   184     return 1;

   185   switch (BYTE_TO_ASCII(enc, ptr)) {

   186   case ASCII_x:

   187     break;

   188   case ASCII_X:

   189     upper = 1;

   190     break;

   191   default:

   192     return 1;

   193   }

   194   ptr += MINBPC(enc);

   195   switch (BYTE_TO_ASCII(enc, ptr)) {

   196   case ASCII_m:

   197     break;

   198   case ASCII_M:

   199     upper = 1;

   200     break;

   201   default:

   202     return 1;

   203   }

   204   ptr += MINBPC(enc);

   205   switch (BYTE_TO_ASCII(enc, ptr)) {

   206   case ASCII_l:

   207     break;

   208   case ASCII_L:

   209     upper = 1;

   210     break;

   211   default:

   212     return 1;

   213   }

   214   if (upper)

   215     return 0;

   216   *tokPtr = XML_TOK_XML_DECL;

   217   return 1;

   218 }

   219 

   220 /* ptr points to character following "<?" */

   221 

   222 static int PTRCALL

   223 PREFIX(scanPi)(const ENCODING *enc, const char *ptr,

   224                const char *end, const char **nextTokPtr)

   225 {

   226   int tok;

   227   const char *target = ptr;

   228   if (ptr == end)

   229     return XML_TOK_PARTIAL;

   230   switch (BYTE_TYPE(enc, ptr)) {

   231   CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   232   default:

   233     *nextTokPtr = ptr;

   234     return XML_TOK_INVALID;

   235   }

   236   while (ptr != end) {

   237     switch (BYTE_TYPE(enc, ptr)) {

   238     CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)

   239     case BT_S: case BT_CR: case BT_LF:

   240       if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {

   241         *nextTokPtr = ptr;

   242         return XML_TOK_INVALID;

   243       }

   244       ptr += MINBPC(enc);

   245       while (ptr != end) {

   246         switch (BYTE_TYPE(enc, ptr)) {

   247         INVALID_CASES(ptr, nextTokPtr)

   248         case BT_QUEST:

   249           ptr += MINBPC(enc);

   250           if (ptr == end)

   251             return XML_TOK_PARTIAL;

   252           if (CHAR_MATCHES(enc, ptr, ASCII_GT)) {

   253             *nextTokPtr = ptr + MINBPC(enc);

   254             return tok;

   255           }

   256           break;

   257         default:

   258           ptr += MINBPC(enc);

   259           break;

   260         }

   261       }

   262       return XML_TOK_PARTIAL;

   263     case BT_QUEST:

   264       if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {

   265         *nextTokPtr = ptr;

   266         return XML_TOK_INVALID;

   267       }

   268       ptr += MINBPC(enc);

   269       if (ptr == end)

   270         return XML_TOK_PARTIAL;

   271       if (CHAR_MATCHES(enc, ptr, ASCII_GT)) {

   272         *nextTokPtr = ptr + MINBPC(enc);

   273         return tok;

   274       }

   275       /* fall through */

   276     default:

   277       *nextTokPtr = ptr;

   278       return XML_TOK_INVALID;

   279     }

   280   }

   281   return XML_TOK_PARTIAL;

   282 }

   283 

   284 static int PTRCALL

   285 PREFIX(scanCdataSection)(const ENCODING *enc, const char *ptr,

   286                          const char *end, const char **nextTokPtr)

   287 {

   288   static const char CDATA_LSQB[] = { ASCII_C, ASCII_D, ASCII_A,

   289                                      ASCII_T, ASCII_A, ASCII_LSQB };

   290   int i;

   291   /* CDATA[ */

   292   if (end - ptr < 6 * MINBPC(enc))

   293     return XML_TOK_PARTIAL;

   294   for (i = 0; i < 6; i++, ptr += MINBPC(enc)) {

   295     if (!CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) {

   296       *nextTokPtr = ptr;

   297       return XML_TOK_INVALID;

   298     }

   299   }

   300   *nextTokPtr = ptr;

   301   return XML_TOK_CDATA_SECT_OPEN;

   302 }

   303 

   304 static int PTRCALL

   305 PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr,

   306                         const char *end, const char **nextTokPtr)

   307 {

   308   if (ptr == end)

   309     return XML_TOK_NONE;

   310   if (MINBPC(enc) > 1) {

   311     size_t n = end - ptr;

   312     if (n & (MINBPC(enc) - 1)) {

   313       n &= ~(MINBPC(enc) - 1);

   314       if (n == 0)

   315         return XML_TOK_PARTIAL;

   316       end = ptr + n;

   317     }

   318   }

   319   switch (BYTE_TYPE(enc, ptr)) {

   320   case BT_RSQB:

   321     ptr += MINBPC(enc);

   322     if (ptr == end)

   323       return XML_TOK_PARTIAL;

   324     if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))

   325       break;

   326     ptr += MINBPC(enc);

   327     if (ptr == end)

   328       return XML_TOK_PARTIAL;

   329     if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {

   330       ptr -= MINBPC(enc);

   331       break;

   332     }

   333     *nextTokPtr = ptr + MINBPC(enc);

   334     return XML_TOK_CDATA_SECT_CLOSE;

   335   case BT_CR:

   336     ptr += MINBPC(enc);

   337     if (ptr == end)

   338       return XML_TOK_PARTIAL;

   339     if (BYTE_TYPE(enc, ptr) == BT_LF)

   340       ptr += MINBPC(enc);

   341     *nextTokPtr = ptr;

   342     return XML_TOK_DATA_NEWLINE;

   343   case BT_LF:

   344     *nextTokPtr = ptr + MINBPC(enc);

   345     return XML_TOK_DATA_NEWLINE;

   346   INVALID_CASES(ptr, nextTokPtr)

   347   default:

   348     ptr += MINBPC(enc);

   349     break;

   350   }

   351   while (ptr != end) {

   352     switch (BYTE_TYPE(enc, ptr)) {

   353 #define LEAD_CASE(n) \

   354     case BT_LEAD ## n: \

   355       if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) { \

   356         *nextTokPtr = ptr; \

   357         return XML_TOK_DATA_CHARS; \

   358       } \

   359       ptr += n; \

   360       break;

   361     LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)

   362 #undef LEAD_CASE

   363     case BT_NONXML:

   364     case BT_MALFORM:

   365     case BT_TRAIL:

   366     case BT_CR:

   367     case BT_LF:

   368     case BT_RSQB:

   369       *nextTokPtr = ptr;

   370       return XML_TOK_DATA_CHARS;

   371     default:

   372       ptr += MINBPC(enc);

   373       break;

   374     }

   375   }

   376   *nextTokPtr = ptr;

   377   return XML_TOK_DATA_CHARS;

   378 }

   379 

   380 /* ptr points to character following "</" */

   381 

   382 static int PTRCALL

   383 PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr,

   384                    const char *end, const char **nextTokPtr)

   385 {

   386   if (ptr == end)

   387     return XML_TOK_PARTIAL;

   388   switch (BYTE_TYPE(enc, ptr)) {

   389   CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   390   default:

   391     *nextTokPtr = ptr;

   392     return XML_TOK_INVALID;

   393   }

   394   while (ptr != end) {

   395     switch (BYTE_TYPE(enc, ptr)) {

   396     CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)

   397     case BT_S: case BT_CR: case BT_LF:

   398       for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) {

   399         switch (BYTE_TYPE(enc, ptr)) {

   400         case BT_S: case BT_CR: case BT_LF:

   401           break;

   402         case BT_GT:

   403           *nextTokPtr = ptr + MINBPC(enc);

   404           return XML_TOK_END_TAG;

   405         default:

   406           *nextTokPtr = ptr;

   407           return XML_TOK_INVALID;

   408         }

   409       }

   410       return XML_TOK_PARTIAL;

   411 #ifdef XML_NS

   412     case BT_COLON:

   413       /* no need to check qname syntax here,

   414          since end-tag must match exactly */

   415       ptr += MINBPC(enc);

   416       break;

   417 #endif

   418     case BT_GT:

   419       *nextTokPtr = ptr + MINBPC(enc);

   420       return XML_TOK_END_TAG;

   421     default:

   422       *nextTokPtr = ptr;

   423       return XML_TOK_INVALID;

   424     }

   425   }

   426   return XML_TOK_PARTIAL;

   427 }

   428 

   429 /* ptr points to character following "&#X" */

   430 

   431 static int PTRCALL

   432 PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr,

   433                        const char *end, const char **nextTokPtr)

   434 {

   435   if (ptr != end) {

   436     switch (BYTE_TYPE(enc, ptr)) {

   437     case BT_DIGIT:

   438     case BT_HEX:

   439       break;

   440     default:

   441       *nextTokPtr = ptr;

   442       return XML_TOK_INVALID;

   443     }

   444     for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) {

   445       switch (BYTE_TYPE(enc, ptr)) {

   446       case BT_DIGIT:

   447       case BT_HEX:

   448         break;

   449       case BT_SEMI:

   450         *nextTokPtr = ptr + MINBPC(enc);

   451         return XML_TOK_CHAR_REF;

   452       default:

   453         *nextTokPtr = ptr;

   454         return XML_TOK_INVALID;

   455       }

   456     }

   457   }

   458   return XML_TOK_PARTIAL;

   459 }

   460 

   461 /* ptr points to character following "&#" */

   462 

   463 static int PTRCALL

   464 PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr,

   465                     const char *end, const char **nextTokPtr)

   466 {

   467   if (ptr != end) {

   468     if (CHAR_MATCHES(enc, ptr, ASCII_x))

   469       return PREFIX(scanHexCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   470     switch (BYTE_TYPE(enc, ptr)) {

   471     case BT_DIGIT:

   472       break;

   473     default:

   474       *nextTokPtr = ptr;

   475       return XML_TOK_INVALID;

   476     }

   477     for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) {

   478       switch (BYTE_TYPE(enc, ptr)) {

   479       case BT_DIGIT:

   480         break;

   481       case BT_SEMI:

   482         *nextTokPtr = ptr + MINBPC(enc);

   483         return XML_TOK_CHAR_REF;

   484       default:

   485         *nextTokPtr = ptr;

   486         return XML_TOK_INVALID;

   487       }

   488     }

   489   }

   490   return XML_TOK_PARTIAL;

   491 }

   492 

   493 /* ptr points to character following "&" */

   494 

   495 static int PTRCALL

   496 PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end,

   497                 const char **nextTokPtr)

   498 {

   499   if (ptr == end)

   500     return XML_TOK_PARTIAL;

   501   switch (BYTE_TYPE(enc, ptr)) {

   502   CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   503   case BT_NUM:

   504     return PREFIX(scanCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   505   default:

   506     *nextTokPtr = ptr;

   507     return XML_TOK_INVALID;

   508   }

   509   while (ptr != end) {

   510     switch (BYTE_TYPE(enc, ptr)) {

   511     CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)

   512     case BT_SEMI:

   513       *nextTokPtr = ptr + MINBPC(enc);

   514       return XML_TOK_ENTITY_REF;

   515     default:

   516       *nextTokPtr = ptr;

   517       return XML_TOK_INVALID;

   518     }

   519   }

   520   return XML_TOK_PARTIAL;

   521 }

   522 

   523 /* ptr points to character following first character of attribute name */

   524 

   525 static int PTRCALL

   526 PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end,

   527                  const char **nextTokPtr)

   528 {

   529 #ifdef XML_NS

   530   int hadColon = 0;

   531 #endif

   532   while (ptr != end) {

   533     switch (BYTE_TYPE(enc, ptr)) {

   534     CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)

   535 #ifdef XML_NS

   536     case BT_COLON:

   537       if (hadColon) {

   538         *nextTokPtr = ptr;

   539         return XML_TOK_INVALID;

   540       }

   541       hadColon = 1;

   542       ptr += MINBPC(enc);

   543       if (ptr == end)

   544         return XML_TOK_PARTIAL;

   545       switch (BYTE_TYPE(enc, ptr)) {

   546       CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   547       default:

   548         *nextTokPtr = ptr;

   549         return XML_TOK_INVALID;

   550       }

   551       break;

   552 #endif

   553     case BT_S: case BT_CR: case BT_LF:

   554       for (;;) {

   555         int t;

   556 

   557         ptr += MINBPC(enc);

   558         if (ptr == end)

   559           return XML_TOK_PARTIAL;

   560         t = BYTE_TYPE(enc, ptr);

   561         if (t == BT_EQUALS)

   562           break;

   563         switch (t) {

   564         case BT_S:

   565         case BT_LF:

   566         case BT_CR:

   567           break;

   568         default:

   569           *nextTokPtr = ptr;

   570           return XML_TOK_INVALID;

   571         }

   572       }

   573     /* fall through */

   574     case BT_EQUALS:

   575       {

   576         int open;

   577 #ifdef XML_NS

   578         hadColon = 0;

   579 #endif

   580         for (;;) {

   581           ptr += MINBPC(enc);

   582           if (ptr == end)

   583             return XML_TOK_PARTIAL;

   584           open = BYTE_TYPE(enc, ptr);

   585           if (open == BT_QUOT || open == BT_APOS)

   586             break;

   587           switch (open) {

   588           case BT_S:

   589           case BT_LF:

   590           case BT_CR:

   591             break;

   592           default:

   593             *nextTokPtr = ptr;

   594             return XML_TOK_INVALID;

   595           }

   596         }

   597         ptr += MINBPC(enc);

   598         /* in attribute value */

   599         for (;;) {

   600           int t;

   601           if (ptr == end)

   602             return XML_TOK_PARTIAL;

   603           t = BYTE_TYPE(enc, ptr);

   604           if (t == open)

   605             break;

   606           switch (t) {

   607           INVALID_CASES(ptr, nextTokPtr)

   608           case BT_AMP:

   609             {

   610               int tok = PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, &ptr);

   611               if (tok <= 0) {

   612                 if (tok == XML_TOK_INVALID)

   613                   *nextTokPtr = ptr;

   614                 return tok;

   615               }

   616               break;

   617             }

   618           case BT_LT:

   619             *nextTokPtr = ptr;

   620             return XML_TOK_INVALID;

   621           default:

   622             ptr += MINBPC(enc);

   623             break;

   624           }

   625         }

   626         ptr += MINBPC(enc);

   627         if (ptr == end)

   628           return XML_TOK_PARTIAL;

   629         switch (BYTE_TYPE(enc, ptr)) {

   630         case BT_S:

   631         case BT_CR:

   632         case BT_LF:

   633           break;

   634         case BT_SOL:

   635           goto sol;

   636         case BT_GT:

   637           goto gt;

   638         default:

   639           *nextTokPtr = ptr;

   640           return XML_TOK_INVALID;

   641         }

   642         /* ptr points to closing quote */

   643         for (;;) {

   644           ptr += MINBPC(enc);

   645           if (ptr == end)

   646             return XML_TOK_PARTIAL;

   647           switch (BYTE_TYPE(enc, ptr)) {

   648           CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   649           case BT_S: case BT_CR: case BT_LF:

   650             continue;

   651           case BT_GT:

   652           gt:

   653             *nextTokPtr = ptr + MINBPC(enc);

   654             return XML_TOK_START_TAG_WITH_ATTS;

   655           case BT_SOL:

   656           sol:

   657             ptr += MINBPC(enc);

   658             if (ptr == end)

   659               return XML_TOK_PARTIAL;

   660             if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {

   661               *nextTokPtr = ptr;

   662               return XML_TOK_INVALID;

   663             }

   664             *nextTokPtr = ptr + MINBPC(enc);

   665             return XML_TOK_EMPTY_ELEMENT_WITH_ATTS;

   666           default:

   667             *nextTokPtr = ptr;

   668             return XML_TOK_INVALID;

   669           }

   670           break;

   671         }

   672         break;

   673       }

   674     default:

   675       *nextTokPtr = ptr;

   676       return XML_TOK_INVALID;

   677     }

   678   }

   679   return XML_TOK_PARTIAL;

   680 }

   681 

   682 /* ptr points to character following "<" */

   683 

   684 static int PTRCALL

   685 PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end,

   686                const char **nextTokPtr)

   687 {

   688 #ifdef XML_NS

   689   int hadColon;

   690 #endif

   691   if (ptr == end)

   692     return XML_TOK_PARTIAL;

   693   switch (BYTE_TYPE(enc, ptr)) {

   694   CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   695   case BT_EXCL:

   696     if ((ptr += MINBPC(enc)) == end)

   697       return XML_TOK_PARTIAL;

   698     switch (BYTE_TYPE(enc, ptr)) {

   699     case BT_MINUS:

   700       return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   701     case BT_LSQB:

   702       return PREFIX(scanCdataSection)(enc, ptr + MINBPC(enc),

   703                                       end, nextTokPtr);

   704     }

   705     *nextTokPtr = ptr;

   706     return XML_TOK_INVALID;

   707   case BT_QUEST:

   708     return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   709   case BT_SOL:

   710     return PREFIX(scanEndTag)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   711   default:

   712     *nextTokPtr = ptr;

   713     return XML_TOK_INVALID;

   714   }

   715 #ifdef XML_NS

   716   hadColon = 0;

   717 #endif

   718   /* we have a start-tag */

   719   while (ptr != end) {

   720     switch (BYTE_TYPE(enc, ptr)) {

   721     CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)

   722 #ifdef XML_NS

   723     case BT_COLON:

   724       if (hadColon) {

   725         *nextTokPtr = ptr;

   726         return XML_TOK_INVALID;

   727       }

   728       hadColon = 1;

   729       ptr += MINBPC(enc);

   730       if (ptr == end)

   731         return XML_TOK_PARTIAL;

   732       switch (BYTE_TYPE(enc, ptr)) {

   733       CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   734       default:

   735         *nextTokPtr = ptr;

   736         return XML_TOK_INVALID;

   737       }

   738       break;

   739 #endif

   740     case BT_S: case BT_CR: case BT_LF:

   741       {

   742         ptr += MINBPC(enc);

   743         while (ptr != end) {

   744           switch (BYTE_TYPE(enc, ptr)) {

   745           CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   746           case BT_GT:

   747             goto gt;

   748           case BT_SOL:

   749             goto sol;

   750           case BT_S: case BT_CR: case BT_LF:

   751             ptr += MINBPC(enc);

   752             continue;

   753           default:

   754             *nextTokPtr = ptr;

   755             return XML_TOK_INVALID;

   756           }

   757           return PREFIX(scanAtts)(enc, ptr, end, nextTokPtr);

   758         }

   759         return XML_TOK_PARTIAL;

   760       }

   761     case BT_GT:

   762     gt:

   763       *nextTokPtr = ptr + MINBPC(enc);

   764       return XML_TOK_START_TAG_NO_ATTS;

   765     case BT_SOL:

   766     sol:

   767       ptr += MINBPC(enc);

   768       if (ptr == end)

   769         return XML_TOK_PARTIAL;

   770       if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {

   771         *nextTokPtr = ptr;

   772         return XML_TOK_INVALID;

   773       }

   774       *nextTokPtr = ptr + MINBPC(enc);

   775       return XML_TOK_EMPTY_ELEMENT_NO_ATTS;

   776     default:

   777       *nextTokPtr = ptr;

   778       return XML_TOK_INVALID;

   779     }

   780   }

   781   return XML_TOK_PARTIAL;

   782 }

   783 

   784 static int PTRCALL

   785 PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end,

   786                    const char **nextTokPtr)

   787 {

   788   if (ptr == end)

   789     return XML_TOK_NONE;

   790   if (MINBPC(enc) > 1) {

   791     size_t n = end - ptr;

   792     if (n & (MINBPC(enc) - 1)) {

   793       n &= ~(MINBPC(enc) - 1);

   794       if (n == 0)

   795         return XML_TOK_PARTIAL;

   796       end = ptr + n;

   797     }

   798   }

   799   switch (BYTE_TYPE(enc, ptr)) {

   800   case BT_LT:

   801     return PREFIX(scanLt)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   802   case BT_AMP:

   803     return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);

   804   case BT_CR:

   805     ptr += MINBPC(enc);

   806     if (ptr == end)

   807       return XML_TOK_TRAILING_CR;

   808     if (BYTE_TYPE(enc, ptr) == BT_LF)

   809       ptr += MINBPC(enc);

   810     *nextTokPtr = ptr;

   811     return XML_TOK_DATA_NEWLINE;

   812   case BT_LF:

   813     *nextTokPtr = ptr + MINBPC(enc);

   814     return XML_TOK_DATA_NEWLINE;

   815   case BT_RSQB:

   816     ptr += MINBPC(enc);

   817     if (ptr == end)

   818       return XML_TOK_TRAILING_RSQB;

   819     if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))

   820       break;

   821     ptr += MINBPC(enc);

   822     if (ptr == end)

   823       return XML_TOK_TRAILING_RSQB;

   824     if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {

   825       ptr -= MINBPC(enc);

   826       break;

   827     }

   828     *nextTokPtr = ptr;

   829     return XML_TOK_INVALID;

   830   INVALID_CASES(ptr, nextTokPtr)

   831   default:

   832     ptr += MINBPC(enc);

   833     break;

   834   }

   835   while (ptr != end) {

   836     switch (BYTE_TYPE(enc, ptr)) {

   837 #define LEAD_CASE(n) \

   838     case BT_LEAD ## n: \

   839       if (end - ptr < n || IS_INVALID_CHAR(enc, ptr, n)) { \

   840         *nextTokPtr = ptr; \

   841         return XML_TOK_DATA_CHARS; \

   842       } \

   843       ptr += n; \

   844       break;

   845     LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)

   846 #undef LEAD_CASE

   847     case BT_RSQB:

   848       if (ptr + MINBPC(enc) != end) {

   849          if (!CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) {

   850            ptr += MINBPC(enc);

   851            break;

   852          }

   853          if (ptr + 2*MINBPC(enc) != end) {

   854            if (!CHAR_MATCHES(enc, ptr + 2*MINBPC(enc), ASCII_GT)) {

   855              ptr += MINBPC(enc);

   856              break;

   857            }

   858            *nextTokPtr = ptr + 2*MINBPC(enc);

   859            return XML_TOK_INVALID;

   860          }

   861       }

   862       /* fall through */

   863     case BT_AMP:

   864     case BT_LT:

   865     case BT_NONXML:

   866     case BT_MALFORM:

   867     case BT_TRAIL:

   868     case BT_CR:

   869     case BT_LF:

   870       *nextTokPtr = ptr;

   871       return XML_TOK_DATA_CHARS;

   872     default:

   873       ptr += MINBPC(enc);

   874       break;

   875     }

   876   }

   877   *nextTokPtr = ptr;

   878   return XML_TOK_DATA_CHARS;

   879 }

   880 

   881 /* ptr points to character following "%" */

   882 

   883 static int PTRCALL

   884 PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end,

   885                     const char **nextTokPtr)

   886 {

   887   if (ptr == end)

   888     return -XML_TOK_PERCENT;

   889   switch (BYTE_TYPE(enc, ptr)) {

   890   CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   891   case BT_S: case BT_LF: case BT_CR: case BT_PERCNT:

   892     *nextTokPtr = ptr;

   893     return XML_TOK_PERCENT;

   894   default:

   895     *nextTokPtr = ptr;

   896     return XML_TOK_INVALID;

   897   }

   898   while (ptr != end) {

   899     switch (BYTE_TYPE(enc, ptr)) {

   900     CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)

   901     case BT_SEMI:

   902       *nextTokPtr = ptr + MINBPC(enc);

   903       return XML_TOK_PARAM_ENTITY_REF;

   904     default:

   905       *nextTokPtr = ptr;

   906       return XML_TOK_INVALID;

   907     }

   908   }

   909   return XML_TOK_PARTIAL;

   910 }

   911 

   912 static int PTRCALL

   913 PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end,

   914                       const char **nextTokPtr)

   915 {

   916   if (ptr == end)

   917     return XML_TOK_PARTIAL;

   918   switch (BYTE_TYPE(enc, ptr)) {

   919   CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)

   920   default:

   921     *nextTokPtr = ptr;

   922     return XML_TOK_INVALID;

   923   }

   924   while (ptr != end) {

   925     switch (BYTE_TYPE(enc, ptr)) {

   926     CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)

   927     case BT_CR: case BT_LF: case BT_S:

   928     case BT_RPAR: case BT_GT: case BT_PERCNT: case BT_VERBAR:

   929       *nextTokPtr = ptr;

   930       return XML_TOK_POUND_NAME;

   931     default:

   932       *nextTokPtr = ptr;

   933       return XML_TOK_INVALID;

   934     }

   935   }

   936   return -XML_TOK_POUND_NAME;

   937 }

   938 

   939 static int PTRCALL

   940 PREFIX(scanLit)(int open, const ENCODING *enc,

   941                 const char *ptr, const char *end,

   942                 const char **nextTokPtr)

   943 {

   944   while (ptr != end) {

   945     int t = BYTE_TYPE(enc, ptr);

   946     switch (t) {

   947     INVALID_CASES(ptr, nextTokPtr)

   948     case BT_QUOT:

   949     case BT_APOS:

   950       ptr += MINBPC(enc);

   951       if (t != open)

   952         break;

   953       if (ptr == end)

   954         return -XML_TOK_LITERAL;

   955       *nextTokPtr = ptr;

   956       switch (BYTE_TYPE(enc, ptr)) {

   957       case BT_S: case BT_CR: case BT_LF:

   958       case BT_GT: case BT_PERCNT: case BT_LSQB:

   959         return XML_TOK_LITERAL;

   960       default:

   961         return XML_TOK_INVALID;

   962       }

   963     default:

   964       ptr += MINBPC(enc);

   965       break;

   966     }

   967   }

   968   return XML_TOK_PARTIAL;

   969 }

   970 

   971 static int PTRCALL

   972 PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end,

   973                   const char **nextTokPtr)

   974 {

   975   int tok;

   976   if (ptr == end)

   977     return XML_TOK_NONE;

   978   if (MINBPC(enc) > 1) {

   979     size_t n = end - ptr;

   980     if (n & (MINBPC(enc) - 1)) {

   981       n &= ~(MINBPC(enc) - 1);

   982       if (n == 0)

   983         return XML_TOK_PARTIAL;

   984       end = ptr + n;

   985     }

   986   }

   987   switch (BYTE_TYPE(enc, ptr)) {

   988   case BT_QUOT:

   989     return PREFIX(scanLit)(BT_QUOT, enc, ptr + MINBPC(enc), end, nextTokPtr);

   990   case BT_APOS:

   991     return PREFIX(scanLit)(BT_APOS, enc, ptr + MINBPC(enc), end, nextTokPtr);

   992   case BT_LT:

   993     {

   994       ptr += MINBPC(enc);

   995       if (ptr == end)

   996         return XML_TOK_PARTIAL;

   997       switch (BYTE_TYPE(enc, ptr)) {

   998       case BT_EXCL:

   999         return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr);

  1000       case BT_QUEST:

  1001         return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);

  1002       case BT_NMSTRT:

  1003       case BT_HEX:

  1004       case BT_NONASCII:

  1005       case BT_LEAD2:

  1006       case BT_LEAD3:

  1007       case BT_LEAD4:

  1008         *nextTokPtr = ptr - MINBPC(enc);

  1009         return XML_TOK_INSTANCE_START;

  1010       }

  1011       *nextTokPtr = ptr;

  1012       return XML_TOK_INVALID;

  1013     }

  1014   case BT_CR:

  1015     if (ptr + MINBPC(enc) == end) {

  1016       *nextTokPtr = end;

  1017       /* indicate that this might be part of a CR/LF pair */

  1018       return -XML_TOK_PROLOG_S;

  1019     }

  1020     /* fall through */

  1021   case BT_S: case BT_LF:

  1022     for (;;) {

  1023       ptr += MINBPC(enc);

  1024       if (ptr == end)

  1025         break;

  1026       switch (BYTE_TYPE(enc, ptr)) {

  1027       case BT_S: case BT_LF:

  1028         break;

  1029       case BT_CR:

  1030         /* don't split CR/LF pair */

  1031         if (ptr + MINBPC(enc) != end)

  1032           break;

  1033         /* fall through */

  1034       default:

  1035         *nextTokPtr = ptr;

  1036         return XML_TOK_PROLOG_S;

  1037       }

  1038     }

  1039     *nextTokPtr = ptr;

  1040     return XML_TOK_PROLOG_S;

  1041   case BT_PERCNT:

  1042     return PREFIX(scanPercent)(enc, ptr + MINBPC(enc), end, nextTokPtr);

  1043   case BT_COMMA:

  1044     *nextTokPtr = ptr + MINBPC(enc);

  1045     return XML_TOK_COMMA;

  1046   case BT_LSQB:

  1047     *nextTokPtr = ptr + MINBPC(enc);

  1048     return XML_TOK_OPEN_BRACKET;

  1049   case BT_RSQB:

  1050     ptr += MINBPC(enc);

  1051     if (ptr == end)

  1052       return -XML_TOK_CLOSE_BRACKET;

  1053     if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) {

  1054       if (ptr + MINBPC(enc) == end)

  1055         return XML_TOK_PARTIAL;

  1056       if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_GT)) {

  1057         *nextTokPtr = ptr + 2*MINBPC(enc);

  1058         return XML_TOK_COND_SECT_CLOSE;

  1059       }

  1060     }

  1061     *nextTokPtr = ptr;

  1062     return XML_TOK_CLOSE_BRACKET;

  1063   case BT_LPAR:

  1064     *nextTokPtr = ptr + MINBPC(enc);

  1065     return XML_TOK_OPEN_PAREN;

  1066   case BT_RPAR:

  1067     ptr += MINBPC(enc);

  1068     if (ptr == end)

  1069       return -XML_TOK_CLOSE_PAREN;

  1070     switch (BYTE_TYPE(enc, ptr)) {

  1071     case BT_AST:

  1072       *nextTokPtr = ptr + MINBPC(enc);

  1073       return XML_TOK_CLOSE_PAREN_ASTERISK;

  1074     case BT_QUEST:

  1075       *nextTokPtr = ptr + MINBPC(enc);

  1076       return XML_TOK_CLOSE_PAREN_QUESTION;

  1077     case BT_PLUS:

  1078       *nextTokPtr = ptr + MINBPC(enc);

  1079       return XML_TOK_CLOSE_PAREN_PLUS;

  1080     case BT_CR: case BT_LF: case BT_S:

  1081     case BT_GT: case BT_COMMA: case BT_VERBAR:

  1082     case BT_RPAR:

  1083       *nextTokPtr = ptr;

  1084       return XML_TOK_CLOSE_PAREN;

  1085     }

  1086     *nextTokPtr = ptr;

  1087     return XML_TOK_INVALID;

  1088   case BT_VERBAR:

  1089     *nextTokPtr = ptr + MINBPC(enc);

  1090     return XML_TOK_OR;

  1091   case BT_GT:

  1092     *nextTokPtr = ptr + MINBPC(enc);

  1093     return XML_TOK_DECL_CLOSE;

  1094   case BT_NUM:

  1095     return PREFIX(scanPoundName)(enc, ptr + MINBPC(enc), end, nextTokPtr);

  1096 #define LEAD_CASE(n) \

  1097   case BT_LEAD ## n: \

  1098     if (end - ptr < n) \

  1099       return XML_TOK_PARTIAL_CHAR; \

  1100     if (IS_NMSTRT_CHAR(enc, ptr, n)) { \

  1101       ptr += n; \

  1102       tok = XML_TOK_NAME; \

  1103       break; \

  1104     } \

  1105     if (IS_NAME_CHAR(enc, ptr, n)) { \

  1106       ptr += n; \

  1107       tok = XML_TOK_NMTOKEN; \

  1108       break; \

  1109     } \

  1110     *nextTokPtr = ptr; \

  1111     return XML_TOK_INVALID;

  1112     LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)

  1113 #undef LEAD_CASE

  1114   case BT_NMSTRT:

  1115   case BT_HEX:

  1116     tok = XML_TOK_NAME;

  1117     ptr += MINBPC(enc);

  1118     break;

  1119   case BT_DIGIT:

  1120   case BT_NAME:

  1121   case BT_MINUS:

  1122 #ifdef XML_NS

  1123   case BT_COLON:

  1124 #endif

  1125     tok = XML_TOK_NMTOKEN;

  1126     ptr += MINBPC(enc);

  1127     break;

  1128   case BT_NONASCII:

  1129     if (IS_NMSTRT_CHAR_MINBPC(enc, ptr)) {

  1130       ptr += MINBPC(enc);

  1131       tok = XML_TOK_NAME;

  1132       break;

  1133     }

  1134     if (IS_NAME_CHAR_MINBPC(enc, ptr)) {

  1135       ptr += MINBPC(enc);