View Javadoc
1   /*
2    * Licensed under the GPL License. You may not use this file except in compliance with the License.
3    * You may obtain a copy of the License at
4    *
5    *   https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
6    *
7    * THIS PACKAGE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
8    * WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR
9    * PURPOSE.
10   */
11  package psiprobe.tokenizer;
12  
13  import static org.junit.jupiter.api.Assertions.assertEquals;
14  import static org.junit.jupiter.api.Assertions.assertFalse;
15  import static org.junit.jupiter.api.Assertions.assertNull;
16  import static org.junit.jupiter.api.Assertions.assertTrue;
17  
18  import java.io.IOException;
19  import java.io.StringReader;
20  
21  import org.junit.jupiter.api.BeforeEach;
22  import org.junit.jupiter.api.Test;
23  
24  class TokenizerTest {
25  
26    private Tokenizer tokenizer;
27  
28    /**
29     * Sets the up.
30     */
31    @BeforeEach
32    void setUp() {
33      tokenizer = new Tokenizer();
34    }
35  
36    @Test
37    void testTokenizeSimpleString() throws IOException {
38      String input = "hello world";
39      tokenizer.setReader(new StringReader(input));
40      Token token1 = tokenizer.nextToken();
41      assertEquals("hello world", token1.getText());
42      assertEquals(Tokenizer.TT_TOKEN, token1.getType());
43      assertFalse(tokenizer.hasMore());
44    }
45  
46    @Test
47    void testAddAndDetectSymbol() throws IOException {
48      tokenizer.addSymbol(";");
49      tokenizer.setReader(new StringReader("abc;def"));
50      Token token1 = tokenizer.nextToken();
51      assertEquals("abc", token1.getText());
52      assertEquals(Tokenizer.TT_TOKEN, token1.getType());
53  
54      Token token2 = tokenizer.nextToken();
55      assertEquals(";", token2.getText());
56      assertEquals(Tokenizer.TT_SYMBOL, token2.getType());
57      assertNull(token2.getName());
58  
59      Token token3 = tokenizer.nextToken();
60      assertEquals("def", token3.getText());
61      assertEquals(Tokenizer.TT_TOKEN, token3.getType());
62    }
63  
64    @Test
65    void testBlockSymbol() throws IOException {
66      tokenizer.addSymbol("/*", "*/", false);
67      tokenizer.setReader(new StringReader("abc/*block*/def"));
68      Token token1 = tokenizer.nextToken();
69      assertEquals("abc", token1.getText());
70      assertEquals(Tokenizer.TT_TOKEN, token1.getType());
71  
72      Token token2 = tokenizer.nextToken();
73      assertEquals("/*block*/", token2.getText());
74      assertEquals("block", token2.getInnerText());
75      assertEquals(Tokenizer.TT_BLOCK, token2.getType());
76  
77      Token token3 = tokenizer.nextToken();
78      assertEquals("def", token3.getText());
79      assertEquals(Tokenizer.TT_TOKEN, token3.getType());
80    }
81  
82    @Test
83    void testGetNextStringAndLong() throws IOException {
84      tokenizer.setReader(new StringReader("12345"));
85      assertEquals("12345", tokenizer.getNextString("default"));
86      tokenizer.setReader(new StringReader("notanumber"));
87      assertEquals(42L, tokenizer.getNextLong(42L));
88    }
89  
90    @Test
91    void testGetNextBoolean() throws IOException {
92      tokenizer.setReader(new StringReader("yes"));
93      assertTrue(tokenizer.getNextBoolean("yes", false));
94      tokenizer.setReader(new StringReader("no"));
95      assertFalse(tokenizer.getNextBoolean("yes", false));
96    }
97  
98    @Test
99    void testPushBack() throws IOException {
100     tokenizer.setReader(new StringReader("foo bar"));
101     Token token1 = tokenizer.nextToken();
102     assertEquals("foo bar", token1.getText());
103     tokenizer.pushBack();
104     Token token2 = tokenizer.nextToken();
105     assertEquals("foo bar", token2.getText());
106   }
107 }
108