Merge branch 'master' into bayes

Author: yooper

README.md

@@ -6,8 +6,19 @@ php-text-analysis
 
 [![Total Downloads](https://poser.pugx.org/yooper/php-text-analysis/downloads)](https://packagist.org/packages/yooper/php-text-analysis)
 
+PHP Text Analysis is a library for performing Information Retrieval (IR) and Natural Language Processing (NLP) tasks using the PHP language. 
+All the documentation for this project can be found in the book and wiki. 
+
+PHP Text Analysis Book & Wiki
+=============
+
+A book is in the works and your contributions are needed. You can find the book
+at https://github.com/yooper/php-text-analysis-book
+
+
+Also, documentation for the library resides in the wiki, too. 
+https://github.com/yooper/php-text-analysis/wiki
 
-PHP Text Analysis is a library for performing Information Retrieval (IR) and Natural Language Processing (NLP) tasks using the PHP language. All the documentation for this project can be found in the wiki. 
 
 Installation Instructions
 =============
@@ -16,9 +27,6 @@ Add PHP Text Analysis to your project
 ```
 composer require yooper/php-text-analysis
 ```
-Documentation for the library resides in the wiki. 
-https://github.com/yooper/php-text-analysis/wiki
-
 
 ### Tokenization
 ```php
@@ -63,9 +71,20 @@ Customize the ngrams
 $trigrams = ngrams($tokens,3, '|');
 ```
 
-Dictionary Installation
-=============
-
-To do
-
+### Stemming
+By default stem method uses the Porter Stemmer.
+```php
+$stemmedTokens = stem($tokens);
+```
+You can customize which type of stemmer to use by passing in the name of the stemmer class name
+```php
+$stemmedTokens = stem($tokens, \TextAnalysis\Stemmers\MorphStemmer::class);
+```
 
+### Keyword Extract with Rake
+There is a short cut method for using the Rake algorithm. You will need to clean
+your data prior to using. Second parameter is the ngram size of your keywords to extract.
+```php
+$rake = rake($tokens, 3);
+$results = $rake->getKeywordScores();
+```

composer.json

@@ -13,7 +13,7 @@
         "psr-4": {
             "TextAnalysis\\": "src/"
 	},
-        "files": ["src/helpers/storage.php", "src/helpers/print.php", "src/helpers/simplified.php", "src/helpers/helpers.php"]
+        "files": ["src/helpers/storage.php", "src/helpers/print.php", "src/helpers/simplified.php", "src/helpers/helpers.php", "src/helpers/interactive_help.php"]
     },
     "autoload-dev": {
         "files": ["tests/TestBaseCase.php"]

interactive

@@ -0,0 +1,6 @@
+#!/bin/bash
+# run the library in an interactive mode for doing analysis
+echo "Welcome to the PHP Text Analysis Interactive Console"
+echo "Type help(); to get a list of available functions"
+php -a -d auto_prepend_file=./vendor/autoload.php
+

src/Analysis/DateAnalysis.php

@@ -28,7 +28,7 @@ class DateAnalysis
      * 
      * @param type $text
      */
-    public function __construct($text) 
+    public function __construct(string $text) 
     {
         $tokenizer = new SentenceTokenizer();
         $this->sentences = $tokenizer->tokenize( $this->normalize($text)) ;        
@@ -39,7 +39,7 @@ public function __construct($text)
      * ie Mar. to March
      * @param string $text
      */
-    protected function normalize($text)
+    protected function normalize(string $text) : string
     {
         $search = ['jan.','feb.','mar.','apr.','may.','jun.','jul.','aug.','sep.','oct.','nov.','dec.'];
         $replace = [
@@ -62,7 +62,7 @@ protected function normalize($text)
     /**
      * @return DateTime[]
      */
-    public function getDates()
+    public function getDates() : array
     {        
         // return the cached copy
         if(empty($this->dates)) {

src/Collocations/CollocationFinder.php

@@ -2,20 +2,25 @@
 
 namespace TextAnalysis\Collocations;
 
-use TextAnalysis\NGrams\NGramFactory;
-use TextAnalysis\Analysis\FreqDist;
-
 /**
  * Helps find popular phrases in the given set of tokens
  * @author yooper
  */
 class CollocationFinder 
 {
+    /**
+     * The ngram size 
+     * @var int
+     */
     protected $nGramSize = 2;
 
+    /**
+     *
+     * @var array
+     */
     protected $tokens = [];
 
-    public function __construct(array $tokens, $nGramSize = 2) 
+    public function __construct(array $tokens, int $nGramSize = 2) 
     {
         $this->tokens = $tokens;
         $this->nGramSize = $nGramSize;

src/Tokenizers/SentimentTokenizer.php

@@ -111,6 +111,38 @@ function text( string $text ): \TextAnalysis\Corpus\TextCorpus {
 	}
 }
 
+
+if (! function_exists('rake')) {
+    /**
+    * Returns an instance of the Rake
+    *
+    * @param array $tokens
+    *
+    * @return \TextAnalysis\Analysis\Keywords\Rake
+    */
+    function rake(array $tokens, int $ngramSize = 3): \TextAnalysis\Analysis\Keywords\Rake 
+    {
+        return new \TextAnalysis\Analysis\Keywords\Rake(new \TextAnalysis\Documents\TokensDocument($tokens), $ngramSize);
+    }
+}
+
+if (! function_exists('stem')) {
+    /**
+    * Returns an array of stemmed tokens
+    *
+    * @param array $tokens
+    *
+    * @return \TextAnalysis\Analysis\Keywords\Rake
+    */
+    function stem(array $tokens, string $stemmerClassName = \TextAnalysis\Stemmers\PorterStemmer::class): array 
+    {
+	$stemmer = new $stemmerClassName();
+        return array_map(function($token) use($stemmer){ return $stemmer->stem($token); }, $tokens);
+    }
+}
+
+
+
 /**
  * Check if the given array has the given needle, using a case insensitive search. 
  * Keeps a local copy of the normalized haystack for quicker lookup on the same array
@@ -155,3 +187,36 @@ function array_searchi(string $needle, array $haystack)
     return array_search($needle, $localCopy); 
 }
 
+
+/**
+ * Load a book into memory
+ * @param string $filename
+ * @return string
+ */
+function gutenberg(string $filename) : string
+{
+    return file_get_contents(get_storage_path("corpora/gutenberg").$filename);
+}
+
+/**
+ * Return a list of books available
+ * @return array
+ */
+function gutenberg_list() : array
+{
+    return scan_dir(get_storage_path("corpora/gutenberg/")); 
+}
+
+/**
+ * Shortcut function for getting contents of directory
+ * @param string $dir
+ * @return array
+ */
+function scan_dir(string $dir) : array
+{
+    return array_diff(scandir($dir), ['..', '.']);    
+}
+
+
+
+

src/helpers/helpers.php

@@ -0,0 +1,15 @@
+<?php
+
+/**
+ * List out a help menu in the interactive console
+ */
+function help()
+{
+    $menu = [
+        'text(string $text) -> Return a TextCorpus object',
+        'normalize(string $text) -> Normalize text to lower case',
+        'todo ....'
+    ];
+    print_array($menu);
+}
+

src/helpers/interactive_help.php

@@ -36,6 +36,25 @@ public function testRake()
         $this->assertArrayHasKey('8/8/2016 5:51 pm', $results);         
     }
 
+    public function testSimplifiedRake()
+    {
+        $stopwords = array_map('trim', file(VENDOR_DIR.'yooper/stop-words/data/stop-words_english_1_en.txt'));
+        // all punctuation must be moved 1 over. Fixes issues with sentences
+        $testData = (new SpacePunctuationFilter([':','\/']))->transform($this->getTestData());
+        //rake MUST be split on whitespace and new lines only
+        $tokens = (new GeneralTokenizer(" \n\t\r"))->tokenize($testData);        
+        $tokenDoc = new TokensDocument($tokens);
+        $tokenDoc->applyTransformation(new LowerCaseFilter())
+                ->applyTransformation(new StopWordsFilter($stopwords), false)
+                ->applyTransformation(new PunctuationFilter(['@',':','\/']), false)
+                ->applyTransformation(new CharFilter(), false);
+
+        $rake = rake($tokenDoc->toArray(), 3);
+        $results = $rake->getKeywordScores();
+        $this->assertArrayHasKey('minimal generating sets', $results); 
+        $this->assertArrayHasKey('8/8/2016 5:51 pm', $results);                 
+    }
+    
     /**
      * Sample test data 
      * @return string

tests/TextAnalysis/Analysis/Keywords/RakeTest.php

@@ -18,4 +18,10 @@ public function testStemmer()
         $this->assertEquals('univers', $stemmer->stem('universities'));
         $this->assertEquals('judg',$stemmer->stem('judges'));
     }
+    
+    public function testSimplifiedStemmer()
+    {
+        $this->assertEquals(['univers','judg'], stem(['universities', 'judges']));
+    }    
+    
 }