diff options
| author | bernard <[email protected]> | 2017-11-02 13:50:42 +0800 |
|---|---|---|
| committer | bernard <[email protected]> | 2017-11-02 13:50:42 +0800 |
| commit | b2057f506a09e2adef81fc77efd697d16ae14bec (patch) | |
| tree | 3affefabfe376a0ec1b3ac4a5eef957c1dd58dc8 /components/external/SQLite-3.8.1/test/fts4unicode.test | |
| parent | 23582c8129228b4961ad35319cfd2d270039345f (diff) | |
[External] Remove external from RT-Thread
Diffstat (limited to 'components/external/SQLite-3.8.1/test/fts4unicode.test')
| -rw-r--r-- | components/external/SQLite-3.8.1/test/fts4unicode.test | 563 |
1 files changed, 0 insertions, 563 deletions
diff --git a/components/external/SQLite-3.8.1/test/fts4unicode.test b/components/external/SQLite-3.8.1/test/fts4unicode.test deleted file mode 100644 index 8edc2877da..0000000000 --- a/components/external/SQLite-3.8.1/test/fts4unicode.test +++ /dev/null @@ -1,563 +0,0 @@ -# 2012 May 25 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#************************************************************************* -# -# The tests in this file focus on testing the "unicode" FTS tokenizer. -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -ifcapable !fts3_unicode { finish_test ; return } -set ::testprefix fts4unicode - -proc do_unicode_token_test {tn input res} { - set input [string map {' ''} $input] - uplevel [list do_execsql_test $tn " - SELECT fts3_tokenizer_test('unicode61', 'remove_diacritics=0', '$input'); - " [list [list {*}$res]]] -} - -proc do_unicode_token_test2 {tn input res} { - set input [string map {' ''} $input] - uplevel [list do_execsql_test $tn " - SELECT fts3_tokenizer_test('unicode61', '$input'); - " [list [list {*}$res]]] -} - -proc do_unicode_token_test3 {tn args} { - set res [lindex $args end] - set sql "SELECT fts3_tokenizer_test('unicode61'" - foreach a [lrange $args 0 end-1] { - append sql ", '" - append sql [string map {' ''} $a] - append sql "'" - } - append sql ")" - uplevel [list do_execsql_test $tn $sql [list [list {*}$res]]] -} - -do_unicode_token_test 1.0 {a B c D} {0 a a 1 b B 2 c c 3 d D} - -do_unicode_token_test 1.1 "\uC4 \uD6 \uDC" \ - "0 \uE4 \uC4 1 \uF6 \uD6 2 \uFC \uDC" - -do_unicode_token_test 1.2 "x\uC4x x\uD6x x\uDCx" \ - "0 x\uE4x x\uC4x 1 x\uF6x x\uD6x 2 x\uFCx x\uDCx" - -# 0x00DF is a small "sharp s". 0x1E9E is a capital sharp s. -do_unicode_token_test 1.3 "\uDF" "0 \uDF \uDF" -do_unicode_token_test 1.4 "\u1E9E" "0 \uDF \u1E9E" - -do_unicode_token_test 1.5 "The quick brown fox" { - 0 the The 1 quick quick 2 brown brown 3 fox fox -} -do_unicode_token_test 1.6 "The\u00bfquick\u224ebrown\u2263fox" { - 0 the The 1 quick quick 2 brown brown 3 fox fox -} - -do_unicode_token_test2 1.7 {a B c D} {0 a a 1 b B 2 c c 3 d D} -do_unicode_token_test2 1.8 "\uC4 \uD6 \uDC" "0 a \uC4 1 o \uD6 2 u \uDC" - -do_unicode_token_test2 1.9 "x\uC4x x\uD6x x\uDCx" \ - "0 xax x\uC4x 1 xox x\uD6x 2 xux x\uDCx" - -# Check that diacritics are removed if remove_diacritics=1 is specified. -# And that they do not break tokens. -do_unicode_token_test2 1.10 "xx\u0301xx" "0 xxxx xx\u301xx" - -# Title-case mappings work -do_unicode_token_test 1.11 "\u01c5" "0 \u01c6 \u01c5" - -#------------------------------------------------------------------------- -# -set docs [list { - Enhance the INSERT syntax to allow multiple rows to be inserted via the - VALUES clause. -} { - Enhance the CREATE VIRTUAL TABLE command to support the IF NOT EXISTS clause. -} { - Added the sqlite3_stricmp() interface as a counterpart to sqlite3_strnicmp(). -} { - Added the sqlite3_db_readonly() interface. -} { - Added the SQLITE_FCNTL_PRAGMA file control, giving VFS implementations the - ability to add new PRAGMA statements or to override built-in PRAGMAs. -} { - Queries of the form: "SELECT max(x), y FROM table" returns the value of y on - the same row that contains the maximum x value. -} { - Added support for the FTS4 languageid option. -} { - Documented support for the FTS4 content option. This feature has actually - been in the code since version 3.7.9 but is only now considered to be - officially supported. -} { - Pending statements no longer block ROLLBACK. Instead, the pending statement - will return SQLITE_ABORT upon next access after the ROLLBACK. -} { - Improvements to the handling of CSV inputs in the command-line shell -} { - Fix a bug introduced in version 3.7.10 that might cause a LEFT JOIN to be - incorrectly converted into an INNER JOIN if the WHERE clause indexable terms - connected by OR. -}] - -set map(a) [list "\u00C4" "\u00E4"] ; # LATIN LETTER A WITH DIAERESIS -set map(e) [list "\u00CB" "\u00EB"] ; # LATIN LETTER E WITH DIAERESIS -set map(i) [list "\u00CF" "\u00EF"] ; # LATIN LETTER I WITH DIAERESIS -set map(o) [list "\u00D6" "\u00F6"] ; # LATIN LETTER O WITH DIAERESIS -set map(u) [list "\u00DC" "\u00FC"] ; # LATIN LETTER U WITH DIAERESIS -set map(y) [list "\u0178" "\u00FF"] ; # LATIN LETTER Y WITH DIAERESIS -set map(h) [list "\u1E26" "\u1E27"] ; # LATIN LETTER H WITH DIAERESIS -set map(w) [list "\u1E84" "\u1E85"] ; # LATIN LETTER W WITH DIAERESIS -set map(x) [list "\u1E8C" "\u1E8D"] ; # LATIN LETTER X WITH DIAERESIS -foreach k [array names map] { - lappend mappings [string toupper $k] [lindex $map($k) 0] - lappend mappings $k [lindex $map($k) 1] -} -proc mapdoc {doc} { - set doc [regsub -all {[[:space:]]+} $doc " "] - string map $::mappings [string trim $doc] -} - -do_test 2.0 { - execsql { CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61, x); } - foreach doc $docs { - set d [mapdoc $doc] - execsql { INSERT INTO t2 VALUES($d) } - } -} {} - -do_test 2.1 { - set q [mapdoc "row"] - execsql { SELECT * FROM t2 WHERE t2 MATCH $q } -} [list [mapdoc { - Queries of the form: "SELECT max(x), y FROM table" returns the value of y on - the same row that contains the maximum x value. -}]] - -foreach {tn query snippet} { - 2 "row" { - ...returns the value of y on the same [row] that contains - the maximum x value. - } - 3 "ROW" { - ...returns the value of y on the same [row] that contains - the maximum x value. - } - 4 "rollback" { - ...[ROLLBACK]. Instead, the pending statement - will return SQLITE_ABORT upon next access after the [ROLLBACK]. - } - 5 "rOllback" { - ...[ROLLBACK]. Instead, the pending statement - will return SQLITE_ABORT upon next access after the [ROLLBACK]. - } - 6 "lang*" { - Added support for the FTS4 [languageid] option. - } -} { - do_test 2.$tn { - set q [mapdoc $query] - execsql { SELECT snippet(t2, '[', ']', '...') FROM t2 WHERE t2 MATCH $q } - } [list [mapdoc $snippet]] -} - -#------------------------------------------------------------------------- -# Make sure the unicode61 tokenizer does not crash if it is passed a -# NULL pointer. -reset_db -do_execsql_test 3.1 { - CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61, x, y); - INSERT INTO t1 VALUES(NULL, 'a b c'); -} - -do_execsql_test 3.2 { - SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH 'b' -} {{a [b] c}} - -do_execsql_test 3.3 { - BEGIN; - DELETE FROM t1; - INSERT INTO t1 VALUES('b b b b b b b b b b b', 'b b b b b b b b b b b b b'); - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 VALUES('a b c', NULL); - INSERT INTO t1 VALUES('a x c', NULL); - COMMIT; -} - -do_execsql_test 3.4 { - SELECT * FROM t1 WHERE t1 MATCH 'a b'; -} {{a b c} {}} - -#------------------------------------------------------------------------- -# -reset_db - -do_test 4.1 { - set a "abc\uFFFEdef" - set b "abc\uD800def" - set c "\uFFFEdef" - set d "\uD800def" - execsql { - CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61, x); - INSERT INTO t1 VALUES($a); - INSERT INTO t1 VALUES($b); - INSERT INTO t1 VALUES($c); - INSERT INTO t1 VALUES($d); - } -} {} - -do_test 4.2 { - set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}] - set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}] - set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] - set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] - execsql { - INSERT INTO t1 VALUES($a); - INSERT INTO t1 VALUES($b); - INSERT INTO t1 VALUES($c); - INSERT INTO t1 VALUES($d); - } -} {} - -do_test 4.3 { - set a [binary format c* {0xF7 0xBF 0xBF 0xBF}] - set b [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF}] - set c [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF}] - set d [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF}] - execsql { - INSERT INTO t1 VALUES($a); - INSERT INTO t1 VALUES($b); - INSERT INTO t1 VALUES($c); - INSERT INTO t1 VALUES($d); - } -} {} - -#------------------------------------------------------------------------- - -do_unicode_token_test3 5.1 {tokenchars=} { - sqlite3_reset sqlite3_column_int -} { - 0 sqlite3 sqlite3 - 1 reset reset - 2 sqlite3 sqlite3 - 3 column column - 4 int int -} - -do_unicode_token_test3 5.2 {tokenchars=_} { - sqlite3_reset sqlite3_column_int -} { - 0 sqlite3_reset sqlite3_reset - 1 sqlite3_column_int sqlite3_column_int -} - -do_unicode_token_test3 5.3 {separators=xyz} { - Laotianxhorseyrunszfast -} { - 0 laotian Laotian - 1 horse horse - 2 runs runs - 3 fast fast -} - -do_unicode_token_test3 5.4 {tokenchars=xyz} { - Laotianxhorseyrunszfast -} { - 0 laotianxhorseyrunszfast Laotianxhorseyrunszfast -} - -do_unicode_token_test3 5.5 {tokenchars=_} {separators=zyx} { - sqlite3_resetxsqlite3_column_intyhonda_phantom -} { - 0 sqlite3_reset sqlite3_reset - 1 sqlite3_column_int sqlite3_column_int - 2 honda_phantom honda_phantom -} - -do_unicode_token_test3 5.6 "separators=\u05D1" "abc\u05D1def" { - 0 abc abc 1 def def -} - -do_unicode_token_test3 5.7 \ - "tokenchars=\u2444\u2445" \ - "separators=\u05D0\u05D1\u05D2" \ - "\u2444fre\u2445sh\u05D0water\u05D2fish.\u2445timer" \ - [list \ - 0 \u2444fre\u2445sh \u2444fre\u2445sh \ - 1 water water \ - 2 fish fish \ - 3 \u2445timer \u2445timer \ - ] - -# Check that it is not possible to add a standalone diacritic codepoint -# to either separators or tokenchars. -do_unicode_token_test3 5.8 "separators=\u0301" \ - "hello\u0301world \u0301helloworld" \ - "0 helloworld hello\u0301world 1 helloworld helloworld" - -do_unicode_token_test3 5.9 "tokenchars=\u0301" \ - "hello\u0301world \u0301helloworld" \ - "0 helloworld hello\u0301world 1 helloworld helloworld" - -do_unicode_token_test3 5.10 "separators=\u0301" \ - "remove_diacritics=0" \ - "hello\u0301world \u0301helloworld" \ - "0 hello\u0301world hello\u0301world 1 helloworld helloworld" - -do_unicode_token_test3 5.11 "tokenchars=\u0301" \ - "remove_diacritics=0" \ - "hello\u0301world \u0301helloworld" \ - "0 hello\u0301world hello\u0301world 1 helloworld helloworld" - - -#------------------------------------------------------------------------- - -proc do_tokenize {tokenizer txt} { - set res [list] - foreach {a b c} [db one {SELECT fts3_tokenizer_test($tokenizer, $txt)}] { - lappend res $b - } - set res -} - -# Argument $lCodepoint must be a list of codepoints (integers) that -# correspond to whitespace characters. This command creates a string -# $W from the codepoints, then tokenizes "${W}hello{$W}world${W}" -# using tokenizer $tokenizer. The test passes if the tokenizer successfully -# extracts the two 5 character tokens. -# -proc do_isspace_test {tn tokenizer lCp} { - set whitespace [format [string repeat %c [llength $lCp]] {*}$lCp] - set txt "${whitespace}hello${whitespace}world${whitespace}" - uplevel [list do_test $tn [list do_tokenize $tokenizer $txt] {hello world}] -} - -set tokenizers [list unicode61] -ifcapable icu { lappend tokenizers icu } - -# Some tests to check that the tokenizers can both identify white-space -# codepoints. All codepoints tested below are of type "Zs" in the -# UnicodeData.txt file. -foreach T $tokenizers { - do_isspace_test 6.$T.1 $T 32 - do_isspace_test 6.$T.2 $T 160 - do_isspace_test 6.$T.3 $T 5760 - do_isspace_test 6.$T.4 $T 6158 - do_isspace_test 6.$T.5 $T 8192 - do_isspace_test 6.$T.6 $T 8193 - do_isspace_test 6.$T.7 $T 8194 - do_isspace_test 6.$T.8 $T 8195 - do_isspace_test 6.$T.9 $T 8196 - do_isspace_test 6.$T.10 $T 8197 - do_isspace_test 6.$T.11 $T 8198 - do_isspace_test 6.$T.12 $T 8199 - do_isspace_test 6.$T.13 $T 8200 - do_isspace_test 6.$T.14 $T 8201 - do_isspace_test 6.$T.15 $T 8202 - do_isspace_test 6.$T.16 $T 8239 - do_isspace_test 6.$T.17 $T 8287 - do_isspace_test 6.$T.18 $T 12288 - - do_isspace_test 6.$T.19 $T {32 160 5760 6158} - do_isspace_test 6.$T.20 $T {8192 8193 8194 8195} - do_isspace_test 6.$T.21 $T {8196 8197 8198 8199} - do_isspace_test 6.$T.22 $T {8200 8201 8202 8239} - do_isspace_test 6.$T.23 $T {8287 12288} -} - -#------------------------------------------------------------------------- -# Test that the private use ranges are treated as alphanumeric. -# -breakpoint -foreach {tn1 c} { - 1 \ue000 2 \ue001 3 \uf000 4 \uf8fe 5 \uf8ff -} { - foreach {tn2 config res} { - 1 "" "0 hello*world hello*world" - 2 "separators=*" "0 hello hello 1 world world" - } { - set config [string map [list * $c] $config] - set input [string map [list * $c] "hello*world"] - set output [string map [list * $c] $res] - do_unicode_token_test3 7.$tn1.$tn2 {*}$config $input $output - } -} - -#------------------------------------------------------------------------- -# Cursory test of remove_diacritics=0. -# -# 00C4;LATIN CAPITAL LETTER A WITH DIAERESIS -# 00D6;LATIN CAPITAL LETTER O WITH DIAERESIS -# 00E4;LATIN SMALL LETTER A WITH DIAERESIS -# 00F6;LATIN SMALL LETTER O WITH DIAERESIS -# -do_execsql_test 8.1.1 " - CREATE VIRTUAL TABLE t3 USING fts4(tokenize=unicode61 'remove_diacritics=1'); - INSERT INTO t3 VALUES('o'); - INSERT INTO t3 VALUES('a'); - INSERT INTO t3 VALUES('O'); - INSERT INTO t3 VALUES('A'); - INSERT INTO t3 VALUES('\xD6'); - INSERT INTO t3 VALUES('\xC4'); - INSERT INTO t3 VALUES('\xF6'); - INSERT INTO t3 VALUES('\xE4'); -" -do_execsql_test 8.1.2 { - SELECT rowid FROM t3 WHERE t3 MATCH 'o'; -} {1 3 5 7} -do_execsql_test 8.1.3 { - SELECT rowid FROM t3 WHERE t3 MATCH 'a'; -} {2 4 6 8} -do_execsql_test 8.2.1 { - CREATE VIRTUAL TABLE t4 USING fts4(tokenize=unicode61 "remove_diacritics=0"); - INSERT INTO t4 SELECT * FROM t3; -} -do_execsql_test 8.2.2 { - SELECT rowid FROM t4 WHERE t4 MATCH 'o'; -} {1 3} -do_execsql_test 8.2.3 { - SELECT rowid FROM t4 WHERE t4 MATCH 'a'; -} {2 4} - -#------------------------------------------------------------------------- -# -foreach {tn sql} { - 1 { - CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 [tokenchars= .]); - CREATE VIRTUAL TABLE t6 USING fts4( - tokenize=unicode61 [tokenchars=="] "tokenchars=[]"); - CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 [separators=x\xC4]); - } - 2 { - CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 "tokenchars= ."); - CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 "tokenchars=[=""]"); - CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 "separators=x\xC4"); - } - 3 { - CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 'tokenchars= .'); - CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 'tokenchars=="[]'); - CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 'separators=x\xC4'); - } - 4 { - CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 `tokenchars= .`); - CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 `tokenchars=[="]`); - CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 `separators=x\xC4`); - } -} { - do_execsql_test 9.$tn.0 { - DROP TABLE IF EXISTS t5; - DROP TABLE IF EXISTS t5aux; - DROP TABLE IF EXISTS t6; - DROP TABLE IF EXISTS t6aux; - DROP TABLE IF EXISTS t7; - DROP TABLE IF EXISTS t7aux; - } - do_execsql_test 9.$tn.1 $sql - - do_execsql_test 9.$tn.2 { - CREATE VIRTUAL TABLE t5aux USING fts4aux(t5); - INSERT INTO t5 VALUES('one two three/four.five.six'); - SELECT * FROM t5aux; - } { - four.five.six * 1 1 four.five.six 0 1 1 - {one two three} * 1 1 {one two three} 0 1 1 - } - - do_execsql_test 9.$tn.3 { - CREATE VIRTUAL TABLE t6aux USING fts4aux(t6); - INSERT INTO t6 VALUES('alpha=beta"gamma/delta[epsilon]zeta'); - SELECT * FROM t6aux; - } { - {alpha=beta"gamma} * 1 1 {alpha=beta"gamma} 0 1 1 - {delta[epsilon]zeta} * 1 1 {delta[epsilon]zeta} 0 1 1 - } - - do_execsql_test 9.$tn.4 { - CREATE VIRTUAL TABLE t7aux USING fts4aux(t7); - INSERT INTO t7 VALUES('alephxbeth\xC4gimel'); - SELECT * FROM t7aux; - } { - aleph * 1 1 aleph 0 1 1 - beth * 1 1 beth 0 1 1 - gimel * 1 1 gimel 0 1 1 - } -} - -# Check that multiple options are handled correctly. -# -do_execsql_test 10.1 { - DROP TABLE IF EXISTS t1; - CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61 - "tokenchars=xyz" "tokenchars=.=" "separators=.=" "separators=xy" - "separators=a" "separators=a" "tokenchars=a" "tokenchars=a" - ); - - INSERT INTO t1 VALUES('oneatwoxthreeyfour'); - INSERT INTO t1 VALUES('a.single=word'); - CREATE VIRTUAL TABLE t1aux USING fts4aux(t1); - SELECT * FROM t1aux; -} { - .single=word * 1 1 .single=word 0 1 1 - four * 1 1 four 0 1 1 - one * 1 1 one 0 1 1 - three * 1 1 three 0 1 1 - two * 1 1 two 0 1 1 -} - -# Test that case folding happens after tokenization, not before. -# -do_execsql_test 10.2 { - DROP TABLE IF EXISTS t2; - CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61 "separators=aB"); - INSERT INTO t2 VALUES('oneatwoBthree'); - INSERT INTO t2 VALUES('onebtwoAthree'); - CREATE VIRTUAL TABLE t2aux USING fts4aux(t2); - SELECT * FROM t2aux; -} { - one * 1 1 one 0 1 1 - onebtwoathree * 1 1 onebtwoathree 0 1 1 - three * 1 1 three 0 1 1 - two * 1 1 two 0 1 1 -} - -# Test that the tokenchars and separators options work with the -# fts3tokenize table. -# -do_execsql_test 11.1 { - CREATE VIRTUAL TABLE ft1 USING fts3tokenize( - "unicode61", "tokenchars=@.", "separators=1234567890" - ); - SELECT token FROM ft1 WHERE input = '[email protected]'; -} { - berlin@street sydney.road -} - -finish_test - - - - |
