1
2
3
4
5
6
7
8
9
10
11 package psiprobe.tokenizer;
12
13 import static org.junit.jupiter.api.Assertions.assertEquals;
14 import static org.junit.jupiter.api.Assertions.assertFalse;
15 import static org.junit.jupiter.api.Assertions.assertTrue;
16
17 import java.io.IOException;
18
19 import org.junit.jupiter.api.Test;
20
21
22
23
24 class StringTokenizerTest {
25
26 @Test
27 void testSimpleString() throws IOException {
28 StringTokenizer tokenizer = new StringTokenizer("hello world");
29 assertTrue(tokenizer.hasMore());
30 Token token = tokenizer.nextToken();
31 assertEquals("hello world", token.getText());
32 assertEquals(Tokenizer.TT_TOKEN, token.getType());
33 assertFalse(tokenizer.hasMore());
34 }
35
36 @Test
37 void testEmptyString() throws IOException {
38 StringTokenizer tokenizer = new StringTokenizer("");
39 assertFalse(tokenizer.hasMore());
40 }
41
42 @Test
43 void testDefaultConstructorThenSetString() throws IOException {
44 StringTokenizer tokenizer = new StringTokenizer();
45 tokenizer.setString("abc");
46 assertTrue(tokenizer.hasMore());
47 Token token = tokenizer.nextToken();
48 assertEquals("abc", token.getText());
49 }
50
51 @Test
52 void testWithSymbols() throws IOException {
53 StringTokenizer tokenizer = new StringTokenizer("a,b,c");
54 tokenizer.addSymbol(",");
55 Token t1 = tokenizer.nextToken();
56 assertEquals("a", t1.getText());
57 assertEquals(Tokenizer.TT_TOKEN, t1.getType());
58
59 Token t2 = tokenizer.nextToken();
60 assertEquals(",", t2.getText());
61 assertEquals(Tokenizer.TT_SYMBOL, t2.getType());
62
63 Token t3 = tokenizer.nextToken();
64 assertEquals("b", t3.getText());
65
66 Token t4 = tokenizer.nextToken();
67 assertEquals(",", t4.getText());
68
69 Token t5 = tokenizer.nextToken();
70 assertEquals("c", t5.getText());
71
72 assertFalse(tokenizer.hasMore());
73 }
74
75 @Test
76 void testWithBlock() throws IOException {
77 StringTokenizer tokenizer = new StringTokenizer("text[block]end");
78 tokenizer.addSymbol("[", "]", false);
79 Token t1 = tokenizer.nextToken();
80 assertEquals("text", t1.getText());
81
82 Token t2 = tokenizer.nextToken();
83 assertEquals(Tokenizer.TT_BLOCK, t2.getType());
84 assertEquals("block", t2.getInnerText());
85 assertEquals("[block]", t2.getText());
86
87 Token t3 = tokenizer.nextToken();
88 assertEquals("end", t3.getText());
89 }
90
91 @Test
92 void testSetStringOverridesOld() throws IOException {
93 StringTokenizer tokenizer = new StringTokenizer("first");
94 tokenizer.setString("second");
95 Token token = tokenizer.nextToken();
96 assertEquals("second", token.getText());
97 }
98 }