index.js (1130B)
1 /** 2 * @license Apache-2.0 3 * 4 * Copyright (c) 2018 The Stdlib Authors. 5 * 6 * Licensed under the Apache License, Version 2.0 (the "License"); 7 * you may not use this file except in compliance with the License. 8 * You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 19 'use strict'; 20 21 /** 22 * Tokenize a string. 23 * 24 * @module @stdlib/nlp/tokenize 25 * 26 * @example 27 * var tokenize = require( '@stdlib/nlp/tokenize' ); 28 * 29 * var str = 'Hello Mrs. Maple, could you call me back?'; 30 * var out = tokenize( str ); 31 * // returns [ 'Hello', 'Mrs.', 'Maple', ',', 'could', 'you', 'call', 'me', 'back', '?' ] 32 * 33 * str = 'Hello World!'; 34 * out = tokenize( str ); 35 * // returns [ 'Hello', 'World', '!' ] 36 */ 37 38 // MODULES // 39 40 var tokenize = require( './tokenize.js' ); 41 42 43 // EXPORTS // 44 45 module.exports = tokenize;