1 raco pkg install sqlite-table

1 (require sqlite-table)

1 cd Racket-AI-book-code/nlp/
2 raco make *.rkt
3 raco pkg install --scope user

 1 > (require nlp)
 2 loading lex-hash......done.
 3 > (parts-of-speech (list->vector '("the" "cat" "ran")))
 4 '#("DT" "NN" "VBD")
 5 > (find-place-names
 6     '#("George" "Bush" "went" "to" "San" "Diego"
 7        "and" "London") '())
 8 '("London" "San Diego")
 9 > (find-place-names
10     '#("George" "Bush" "went" "to" "San" "Diego"
11        "and" "London") '())
12 '("London" "San Diego")
13 > 

1 > (range 5)
2 '(0 1 2 3 4)
3 > (define (1+ n) (+ n 1))
4 > (map 1+ (range 5))
5 '(1 2 3 4 5)
6 > (map + (range 5) '(100 101 102 103 104))
7 '(100 102 104 106 108)
8 > 

 1 #lang racket
 2 
 3 (define h1 (hash "dog" '("friendly" 5) "cat" '("not friendly" 2))) ;; not mutable
 4 
 5 (define cat (hash-ref h1 "cat"))
 6 
 7 (define h2 (make-hash)) ;; mutable
 8 (hash-set! h2 "snake" '("smooth" 4))
 9 
10 ;; make-hash also accepts a second argument that is a list of pairs:
11 (define h3 (make-hash '(("dog" '("friendly" 5)) ("cat" '("not friendly" 2)))))
12 (hash-set! h3 "snake" '("smooth" 4))
13 (hash-set! h3 "cat" '("sometimes likeable" 3)) ;; overwrite key value
14 
15 ;; for can be used with hash tables:
16 
17 (for ([(k v) h3]) (println '("key:" k "value:" v)))

 1 Welcome to DrRacket, version 8.10 [cs].
 2 Language: racket, with debugging; memory limit: 128 MB.
 3 '("key:" k "value:" v)
 4 '("key:" k "value:" v)
 5 '("key:" k "value:" v)
 6 > h1
 7 '#hash(("cat" . ("not friendly" 2)) ("dog" . ("friendly" 5)))
 8 > h2
 9 '#hash(("snake" . ("smooth" 4)))
10 > h3
11 '#hash(("cat" . ("sometimes likeable" 3))
12                 ("dog" . ('("friendly" 5)))
13                 ("snake" . ("smooth" 4)))
14 > 

 1 > (struct person (name age email) #:mutable)
 2 > (define henry (person "Henry Higgans" 45 "henry@higgans.com"))
 3 > (person-age henry)
 4 45
 5 > (set-person-age! henry 46)
 6 > (person-age henry)
 7 46
 8 > (set-person-email! henry "henryh674373551@gmail.com")
 9 > (person-email henry)
10 "henryh674373551@gmail.com"
11 > 

 1 #lang racket
 2 
 3 (require net/http-easy)
 4 (require html-parsing)
 5 (require net/url xml xml/path)
 6 (require racket/pretty)
 7 
 8 (define res-stream
 9   (get "https://markwatson.com" #:stream? #t))
10 
11 (define lst
12   (html->xexp (response-output res-stream)))
13 
14 (response-close! res-stream)
15 
16 (displayln "\nParagraph text:\n")
17 
18 (pretty-print (take (se-path*/list '(p) lst) 8))
19 
20 (displayln "\nLI text:\n")
21 
22 (pretty-print (take (se-path*/list '(li) lst) 8))

 1 Paragraph text:
 2 
 3 '("My customer list includes: Google, Capital One, Babylist, Olive AI, CompassLabs, \
 4 Mind AI, Disney, SAIC, Americast, PacBell, CastTV, Lutris Technology, Arctan Group, \
 5 Sitescout.com, Embed.ly, and Webmind Corporation."
 6   "I have worked in the fields of general\n"
 7   "      artificial intelligence, machine learning, semantic web and linked data, an\
 8 d\n"
 9   "      natural language processing since 1982."
10   "My eBooks are available to read for FREE or you can purchase them at "
11   (a (@ (href "https://leanpub.com/u/markwatson")) "leanpub")
12   "."
13   "My standard ")
14 
15 LI text:
16 
17 '((@ (class "list f4 f3-ns fw4 dib pr3"))
18   "\n"
19   "            "
20   (& nbsp)
21   (& nbsp)
22   (a
23    (@
24     (class "hover-white no-underline white-90")
25     (href "https://mark-watson.blogspot.com")
26     (target "new")
27     (title "Mark's Blog on Blogspot"))
28    "\n"
29    "              Read my Blog\n"
30    "            ")
31   "\n"
32   "          ")

1 $ cat ~/.racketrc
2 (define (foo-list x)
3   (list x x x))
4 $ racket
5 Welcome to Racket v8.10 [cs].
6 > (foo-list 3.14159)
7 '(3.14159 3.14159 3.14159)
8 > 

  1 #lang at-exp racket
  2 
  3 (provide sparql-dbpedia-person-uri)
  4 (provide sparql-query->hash)
  5 (provide json->listvals)
  6 (provide sparql-dbpedia)
  7 
  8 (require net/url)
  9 (require net/uri-codec)
 10 (require json)
 11 (require racket/pretty)
 12 
 13 (define (sparql-dbpedia-for-person person-uri)
 14   @string-append{
 15      SELECT
 16       (GROUP_CONCAT(DISTINCT ?website; SEPARATOR="  |  ")
 17                                    AS ?website) ?comment {
 18       OPTIONAL {
 19        @person-uri
 20        <http://www.w3.org/2000/01/rdf-schema#comment>
 21        ?comment . FILTER (lang(?comment) = 'en')
 22       } .
 23       OPTIONAL {
 24        @person-uri
 25        <http://dbpedia.org/ontology/wikiPageExternalLink>
 26        ?website
 27         . FILTER( !regex(str(?website), "dbpedia", "i"))
 28       }
 29      } LIMIT 4})
 30 
 31 (define (sparql-dbpedia-person-uri person-name)
 32   @string-append{
 33     SELECT DISTINCT ?personuri ?comment {
 34       ?personuri
 35         <http://xmlns.com/foaf/0.1/name>
 36         "@person-name"@"@"en .
 37       ?personuri
 38         <http://www.w3.org/2000/01/rdf-schema#comment>
 39         ?comment .
 40              FILTER  (lang(?comment) = 'en') .
 41 }})
 42 
 43 
 44 (define (sparql-query->hash query)
 45   (call/input-url
 46    (string->url
 47     (string-append
 48      "https://dbpedia.org/sparql?query="
 49      (uri-encode query)))
 50    get-pure-port
 51    (lambda (port)
 52      (string->jsexpr (port->string port)))
 53    '("Accept: application/json")))
 54 
 55 (define (json->listvals a-hash)
 56   (let ((bindings (hash->list a-hash)))
 57     (let* ((head (first bindings))
 58            (vars (hash-ref (cdr head) 'vars))
 59            (results (second bindings)))
 60       (let* ((x (cdr results))
 61              (b (hash-ref x 'bindings)))
 62         (for/list
 63             ([var vars])
 64           (for/list ([bc b])
 65             (let ((bcequal
 66                    (make-hash (hash->list bc))))
 67               (let ((a-value
 68                      (hash-ref
 69                       (hash-ref
 70                        bcequal
 71                        (string->symbol var)) 'value)))
 72                 (list var a-value)))))))))
 73 
 74 
 75 (define gd (lambda (data)
 76 
 77     (let ((jd (json->listvals data)))
 78 
 79       (define gg1
 80         (lambda (jd) (map list (car jd))))
 81       (define gg2
 82         (lambda (jd) (map list (car jd) (cadr jd))))
 83       (define gg3
 84         (lambda (jd)
 85           (map list (car jd) (cadr jd) (caddr jd))))
 86       (define gg4
 87         (lambda (jd)
 88           (map list
 89                (car jd) (cadr jd)
 90                (caddr jd) (cadddr jd))))
 91 
 92       (case (length (json->listvals data))
 93         [(1) (gg1 (json->listvals data))]
 94         [(2) (gg2 (json->listvals data))]
 95         [(3) (gg3 (json->listvals data))]
 96         [(4) (gg4 (json->listvals data))]
 97         [else
 98          (error "sparql queries with 1 to 4 vars")]))))
 99 
100 
101 (define sparql-dbpedia
102   (lambda (sparql)
103     (gd (sparql-query->hash sparql))))
104 
105 ;; (sparql-dbpedia (sparql-dbpedia-person-uri "Steve Jobs"))

 1 '((("personuri" "http://dbpedia.org/resource/Steve_Jobs")
 2    ("comment"
 3     "Steven Paul Jobs (February 24, 1955 – October 5, 2011) was an American entrepre\
 4 neur, industrial designer, media proprietor, and investor. He was the co-founder, ch\
 5 airman, and CEO of Apple; the chairman and majority shareholder of Pixar; a member o\
 6 f The Walt Disney Company's board of directors following its acquisition of Pixar; a\
 7 nd the founder, chairman, and CEO of NeXT. He is widely recognized as a pioneer of t\
 8 he personal computer revolution of the 1970s and 1980s, along with his early busines\
 9 s partner and fellow Apple co-founder Steve Wozniak."))
10   (("personuri" "http://dbpedia.org/resource/Steve_Jobs_(film)")
11    ("comment"
12     "Steve Jobs is a 2015 biographical drama film directed by Danny Boyle and writte\
13 n by Aaron Sorkin. A British-American co-production, it was adapted from the 2011 bi\
14 ography by Walter Isaacson and interviews conducted by Sorkin, and covers 14 years (\
15 1984–1998) in the life of Apple Inc. co-founder Steve Jobs. Jobs is portrayed by Mic\
16 hael Fassbender, with Kate Winslet as Joanna Hoffman and Seth Rogen, Katherine Water\
17 ston, Michael Stuhlbarg, and Jeff Daniels in supporting roles."))
18   (("personuri" "http://dbpedia.org/resource/Steve_Jobs_(book)")
19    ("comment"
20     "Steve Jobs is the authorized self-titled biography of American business magnate\
21  and Apple co-founder Steve Jobs. The book was written at the request of Jobs by Wal\
22 ter Isaacson, a former executive at CNN and TIME who has written best-selling biogra\
23 phies of Benjamin Franklin and Albert Einstein. The book was released on October 24,\
24  2011, by Simon & Schuster in the United States, 19 days after Jobs's death. A film \
25 adaptation written by Aaron Sorkin and directed by Danny Boyle, with Michael Fassben\
26 der starring in the title role, was released on October 9, 2015.")))

 1 $ racket
 2 Welcome to Racket v8.10 [cs].
 3 > (require db)
 4 > (require sqlite-table)
 5 > (define db-file "test.db")
 6 > (define db (sqlite3-connect #:database db-file #:mode 'create))
 7 > (query-exec db
 8      "create temporary table the_numbers (n integer, d varchar(20))")
 9 > (query-exec db
10      "create  table person (name varchar(30), age integer, email varchar(20))")
11 > (query-exec db
12      "insert into person values ('Mary', 34, 'mary@test.com')")
13 > (query-rows db "select * from person")
14 '(#("Mary" 34 "mary@test.com"))
15 > 

 1 (define (test)
 2   (set! rdf-store '())
 3 
 4   (add-triple "John" "age" "30")
 5   (add-triple "John" "likes" "pizza")
 6   (add-triple "Mary" "age" "25")
 7   (add-triple "Mary" "likes" "sushi")
 8   (add-triple "Bob" "age" "35")
 9   (add-triple "Bob" "likes" "burger")
10 
11   (print-all-triples)
12 
13   (define (print-query-results query-string)
14     (printf "Query: ~a\n" query-string)
15     (let ([results (execute-sparql-query query-string)])
16       (printf "Final Results:\n")
17       (if (null? results)
18           (printf "  No results\n")
19           (for ([result results])
20             (printf "  ~a\n"
21                     (string-join
22                      (map (lambda (pair)
23                             (format "~a: ~a" (car pair) (cdr pair)))
24                           result)
25                      ", "))))
26       (printf "\n")))
27 
28   (print-query-results "select * where { ?name age ?age . ?name likes ?food }")
29   (print-query-results "select ?s ?o where { ?s likes ?o }")
30   (print-query-results "select * where { ?name age ?age . ?name likes pizza }"))

 1 All triples in the datastore:
 2 Bob likes burger
 3 Bob age 35
 4 Mary likes sushi
 5 Mary age 25
 6 John likes pizza
 7 John age 30
 8 
 9 Query: select * where { ?name age ?age . ?name likes ?food }
10 Final Results:
11   ?age: 35, ?name: Bob, ?food: burger
12   ?age: 25, ?name: Mary, ?food: sushi
13   ?age: 30, ?name: John, ?food: pizza
14 
15 Query: select ?s ?o where { ?s likes ?o }
16 Final Results:
17   ?s: Bob, ?o: burger
18   ?s: Mary, ?o: sushi
19   ?s: John, ?o: pizza
20 
21 Query: select * where { ?name age ?age . ?name likes pizza }
22 Final Results:
23   ?age: 30, ?name: John

1 (struct triple (subject predicate object) #:transparent)

1 (define rdf-store '())

1 (define (add-triple subject predicate object)
2   (set! rdf-store (cons (triple subject predicate object) rdf-store)))

1 (define (remove-triple subject predicate object)
2   (set! rdf-store
3         (filter (lambda (t)
4                   (not (and (equal? (triple-subject t) subject)
5                             (equal? (triple-predicate t) predicate)
6                             (equal? (triple-object t) object))))
7                 rdf-store)))

1 (struct sparql-query (select-vars where-patterns) #:transparent)

 1 (define (parse-sparql-query query-string)
 2   (define tokens (filter (lambda (token) (not (member token '("{" "}") string=?)))
 3                          (split-string query-string)))
 4   (define select-index (index-of tokens "select" string-ci=?))
 5   (define where-index (index-of tokens "where" string-ci=?))
 6   (define (sublist lst start end)
 7     (take (drop lst start) (- end start)))
 8   (define select-vars (sublist tokens (add1 select-index) where-index))
 9   (define where-clause (drop tokens (add1 where-index)))
10   (define where-patterns (parse-where-patterns where-clause))
11   (sparql-query select-vars where-patterns))

1 (define (execute-sparql-query query-string)
2   (let* ([query (parse-sparql-query query-string)]
3          [where-patterns (sparql-query-where-patterns query)]
4          [select-vars (sparql-query-select-vars query)]
5          [results (execute-where-patterns where-patterns)]
6          [projected-results (project-results results select-vars)])
7     projected-results))

 1 (define (variable? str)
 2   (and (string? str) (> (string-length str) 0) (char=? (string-ref str 0) #\?)))
 3 
 4 (define (triple-to-binding t [pattern #f])
 5   (define binding '())
 6   (when (and pattern (variable? (first pattern)))
 7     (set! binding (cons (cons (first pattern) (triple-subject t)) binding)))
 8   (when (and pattern (variable? (second pattern)))
 9     (set! binding (cons (cons (second pattern) (triple-predicate t)) binding)))
10   (when (and pattern (variable? (third pattern)))
11     (set! binding (cons (cons (third pattern) (triple-object t)) binding)))
12   binding)
13 
14 (define (query-triples subject predicate object)
15   (filter
16    (lambda (t)
17     (and
18       (or (not subject) (variable? subject) (equal? (triple-subject t) subject))
19       (or (not predicate) (variable? predicate) (equal? (triple-predicate t) predica\
20 te))
21       (or (not object) (variable? object) (equal? (triple-object t) object))))
22    rdf-store))
23 
24 (define (apply-bindings pattern bindings)
25   (map (lambda (item)
26          (if (variable? item)
27              (or (dict-ref bindings item #f) item)
28              item))
29        pattern))
30 
31 (define (merge-bindings binding1 binding2)
32   (append binding1 binding2))
33 
34 (define (project-results results select-vars)
35   (if (equal? select-vars '("*"))
36       (map remove-duplicate-bindings results)
37       (map (lambda (result)
38              (remove-duplicate-bindings
39               (map (lambda (var)
40                      (cons var (dict-ref result var #f)))
41                    select-vars)))
42            results)))

 1 $ racket
 2 Welcome to Racket v8.10 [cs].
 3 > (require net/http-easy)
 4 > (require html-parsing)
 5 > (require net/url xml xml/path)
 6 > (require racket/pretty)
 7 > (define res-stream
 8     (get "https://markwatson.com" #:stream? #t))
 9 > res-stream
10 #<response>
11 > (define lst
12     (html->xexp (response-output res-stream)))
13 > lst
14 '(*TOP*
15   (*DECL* DOCTYPE html)
16   "\n"
17   (html
18    (@ (lang "en-us"))
19    "\n"
20    "  "
21    (head
22     (title
23      "Mark Watson: AI Practitioner and Author of 20+ AI Books | Mark Watson")
24   ...

 1 > (se-path*/list '(p) lst) ;; get text from all p elements
 2 '("My customer list includes: Google, Capital One, Babylist, Olive AI, CompassLabs, \
 3 Mind AI, Disney, SAIC, Americast, PacBell, CastTV, Lutris Technology, Arctan Group, \
 4 Sitescout.com, Embed.ly, and Webmind Corporation."
 5   "I have worked in the fields of general\n"
 6   "   artificial intelligence, machine learning, semantic web and linked data, and\n"
 7   "      natural language processing since 1982."
 8   "My eBooks are available to read for FREE or you can   purchase them at "
 9   (a (@ (href "https://leanpub.com/u/markwatson")) "leanpub")
10   ...

1 > (define lst-p (se-path*/list '(p) lst))
2 > (filter (lambda (s) (string? s)) lst-p) ;; keep only text strings
3 '("My customer list includes: Google, Capital One, Babylist, Olive AI, CompassLabs, \
4 Mind AI, Disney, SAIC, Americast, PacBell, CastTV, Lutris Technology, Arctan Group, \
5 Sitescout.com, Embed.ly, and Webmind Corporation."

 1 #<procedure:string-normalize-spaces>
 2 > (string-normalize-spaces
 3    (string-join
 4     (filter (lambda (s) (string? s)) lst-p)
 5     "\n"))
 6 "My customer list includes: Google, Capital One, Babylist, Olive AI, CompassLabs, Mi\
 7 nd AI, Disney, SAIC, Americast, PacBell, CastTV, Lutris Technology, Arctan Group, Si\
 8 tescout.com, Embed.ly, and Webmind Corporation. I have worked in the fields of gener\
 9 al artificial intelligence, machine learning, semantic web and linked data, and natu\
10 ral language processing since 1982.
11   ...
12 "

 1 > (se-path*/list '(href) lst) ;; get all links from HTML as a lisp
 2 '("/index.css"
 3   "https://mark-watson.blogspot.com"
 4   "#fun"
 5   "#books"
 6   "#opensource"
 7   "https://markwatson.com/privacy.html"
 8   "https://leanpub.com/u/markwatson"
 9   "/nda.txt"
10   "https://mastodon.social/@mark_watson"
11   "https://twitter.com/mark_l_watson"
12   "https://github.com/mark-watson"
13   "https://www.linkedin.com/in/marklwatson/"
14   "https://markwatson.com/index.rdf"
15   "https://www.wikidata.org/wiki/Q18670263"
16  ...
17 )

 1 #lang racket
 2 
 3 (require net/http-easy)
 4 (require html-parsing)
 5 (require net/url xml xml/path)
 6 (require srfi/13) ;; for strings
 7 
 8 (define (web-uri->xexp a-uri)
 9   (let* ((a-stream
10           (get a-uri #:stream? #t))
11          (lst (html->xexp (response-output a-stream))))
12     (response-close! a-stream)
13     lst))
14 
15 (define (web-uri->text a-uri)
16   (let* ((a-xexp
17           (web-uri->xexp a-uri))
18          (p-elements (se-path*/list '(p) a-xexp))
19          (lst-strings
20           (filter
21            (lambda (s) (string? s))
22            p-elements)))
23     (string-normalize-spaces
24      (string-join lst-strings "\n"))))
25 
26 (define (web-uri->links a-uri)
27   (let* ((a-xexp
28           (web-uri->xexp a-uri)))
29     ;; we want only external links so filter out local links:
30     (filter
31      (lambda (s) (string-prefix? "http" s))
32      (se-path*/list '(href) a-xexp))))

 1 > (web-uri->xexp "https://knowledgebooks.com")
 2 '(*TOP*
 3   (*DECL* DOCTYPE html)
 4   "\n"
 5   (html
 6    "\n"
 7    "\n"
 8    (head
 9     "\n"
10     "    "
11     (title "KnowledgeBooks.com - research on the Knowledge Management, and the Seman\
12 tic Web ")
13     "\n"
14  ...
15 
16 > (web-uri->text "https://knowledgebooks.com")
17 "With the experience of working on Machine Learning and Knowledge Graph applications
18  ...
19 
20 > (web-uri->links "https://knowledgebooks.com")
21 '("http://markwatson.com"
22   "https://oceanprotocol.com/"
23   "https://commoncrawl.org/"
24   "http://markwatson.com/consulting/"
25   "http://kbsportal.com")

 1 #lang racket
 2 
 3 (require net/http-easy)
 4 (require racket/set)
 5  (require racket/pretty)
 6 
 7 (provide question-openai completion-openai embeddings-openai)
 8 
 9 (define (helper-openai prefix prompt)
10   (let* ((prompt-data
11           (string-join
12            (list
13             (string-append
14              "{\"messages\": [ {\"role\": \"user\","
15              " \"content\": \"" prefix ": "
16              prompt
17              "\"}], \"model\": \"gpt-4o\"}"))))
18          (auth (lambda (uri headers params)
19                  (values
20                   (hash-set*
21                    headers
22                    'authorization
23                    (string-join
24                     (list
25                      "Bearer "
26                      (getenv "OPENAI_API_KEY")))
27                    'content-type "application/json")
28                   params)))
29          (p
30           (post
31            "https://api.openai.com/v1/chat/completions"
32            #:auth auth
33            #:data prompt-data))
34          (r (response-json p)))
35     ;;(pretty-print r)
36     (hash-ref
37      (hash-ref (first (hash-ref r 'choices)) 'message)
38      'content)))
39 
40 
41 (define (question-openai prompt)
42   (helper-openai "Answer the question: " prompt))
43 
44 (define (completion-openai prompt)
45   (helper-openai "Continue writing from the following text: "
46     prompt))
47 
48 (define (embeddings-openai text)
49     (let* ((prompt-data
50             (string-join
51              (list
52               (string-append
53                "{\"input\": \"" text "\","
54                " \"model\": \"text-embedding-ada-002\"}"))))
55            (auth (lambda (uri headers params)
56                  (values
57                   (hash-set*
58                    headers
59                    'authorization
60                    (string-join
61                     (list
62                      "Bearer "
63                      (getenv "OPENAI_API_KEY")))
64                    'content-type "application/json")
65                   params)))
66          (p
67           (post
68            "https://api.openai.com/v1/embeddings"
69            #:auth auth
70            #:data prompt-data))
71          (r (response-json p)))
72      (hash-ref
73        (first (hash-ref r 'data))
74        'embedding)))

 1 > (question "Mary is 30 and Harry is 25. Who is older?")
 2 "Mary is older than Harry."
 3 > (displayln
 4     (completion
 5       "Frank bought a new sports car. Frank drove"))
 6 Frank bought a new sports car. Frank drove it out of the dealership with a wide grin\
 7  on his face. The sleek, aerodynamic design of the car hugged the road as he acceler\
 8 ated, feeling the power under his hands. The adrenaline surged through his veins, an\
 9 d he couldn't help but let out a triumphant shout as he merged onto the highway.
10 
11 As he cruised down the open road, the wind whipping through his hair, Frank couldn't\
12  help but reflect on how far he had come. It had been a lifelong dream of his to own\
13  a sports car, a symbol of success and freedom in his eyes. He had worked tirelessly\
14 , saving every penny, making sacrifices along the way to finally make this dream a r\
15 eality.
16 ...
17 > 

 1 #lang racket
 2 
 3 (require net/http-easy)
 4 (require racket/set)
 5 (require pprint)
 6 
 7 (provide question completion)
 8 
 9 (define (question prompt max-tokens)
10   (let* ((prompt-data
11           (string-join
12            (list
13             (string-append
14              "{\"prompt\": \"\\n\\nHuman: "
15              prompt
16              "\\n\\nAssistant: \", \"max_tokens_to_sample\": "
17              (number->string  max-tokens)
18              ", \"model\": \"claude-instant-1\" }"))))
19          (auth (lambda (uri headers params)
20                  (values
21                   (hash-set*
22                    headers
23                    'x-api-key
24                      (getenv "ANTHROPIC_API_KEY")
25                    'anthropic-version "2023-06-01"
26                    'content-type "application/json")
27                   params)))
28          (p
29           (post
30            "https://api.anthropic.com/v1/complete"
31            #:auth auth
32            #:data prompt-data))
33          (r (response-json p)))
34     (string-trim (hash-ref r 'completion))))
35 
36 (define (completion prompt max-tokens)
37   (question
38    (string-append
39     "Continue writing from the following text: "
40     prompt)
41    max-tokens))

 1 $ racket
 2 > (require "anthropic.rkt")
 3 > (question "Mary is 30 and Harry is 25. Who is older?" 20)
 4 "Mary is older than Harry. Mary is 30 years old and Harry is 25 years old."
 5 > (completion "Frank bought a new sports car. Frank drove" 200)
 6 "Here is a possible continuation of the story:\n\nFrank bought a new sports car. Fra\
 7 nk drove excitedly to show off his new purchase. The sleek red convertible turned he\
 8 ads as he cruised down the street with the top down. While stopping at a red light, \
 9 Frank saw his neighbor Jane walking down the sidewalk. He pulled over and called out\
10  to her, \"Hey Jane, check out my new ride! Want to go for a spin?\" Jane smiled and\
11  said \"Wow that is one nice car! I'd love to go for a spin.\" She hopped in and the\
12 y sped off down the road, the wind in their hair. Frank was thrilled to show off his\
13  new sports car and even more thrilled to share it with his beautiful neighbor Jane.\
14  Little did he know this joyride would be the beginning of something more between th\
15 em."
16 > 

1 git clone https://github.com/ggerganov/llama.cpp.git
2 make
3 mkdir models

1 $ ls -lh models
2 8.6G openassistant-llama2-13b-orca-8k-3319.Q5_K_M.gguf

1 ./server -m models/openassistant-llama2-13b-orca-8k-3319.Q5_K_M.gguf -c 2048

 1  $ curl --request POST \
 2     --url http://localhost:8080/completion \
 3     --header "Content-Type: application/json" \
 4     --data '{"prompt": "Answer the question: Mary is 30 years old and Sam is 25. Who\
 5  is older and by how much?","n_predict": 128, "top_k": 1}'
 6 {"content":"\nAnswer: Mary is older than Sam by 5 years.","generation_settings":{"fr\
 7 equency_penalty":0.0,"grammar":"","ignore_eos":false,"logit_bias":[],"mirostat":0,"m\
 8 irostat_eta":0.10000000149011612,"mirostat_tau":5.0,"model":"models/openassistant-ll\
 9 ama2-13b-orca-8k-3319.Q5_K_M.gguf","n_ctx":2048,"n_keep":0,"n_predict":128,"n_probs"\
10 :0,"penalize_nl":true,"presence_penalty":0.0,"repeat_last_n":64,"repeat_penalty":1.1\
11 00000023841858,"seed":4294967295,"stop":[],"stream":false,"temp":0.800000011920929,"\
12 tfs_z":1.0,"top_k":1,"top_p":0.949999988079071,"typical_p":1.0},"model":"models/open\
13 assistant-llama2-13b-orca-8k-3319.Q5_K_M.gguf","prompt":"Answer the question: Mary i\
14 s 30 years old and Sam is 25. Who is older and by how much?","stop":true,"stopped_eo\
15 s":true,"stopped_limit":false,"stopped_word":false,"stopping_word":"","timings":{"pr\
16 edicted_ms":960.595,"predicted_n":13,"predicted_per_second":13.53327885321077,"predi\
17 cted_per_token_ms":73.89192307692308,"prompt_ms":539.3580000000001,"prompt_n":27,"pr\
18 ompt_per_second":50.05951520140611,"prompt_per_token_ms":19.976222222222223},"tokens\
19 _cached":40,"tokens_evaluated":27,"tokens_predicted":13,"truncated":false}

1 "content":"Answer: Mary is older than Sam by 5 years."

 1 #lang racket
 2 
 3 (require net/http-easy)
 4 (require racket/set)
 5 (require pprint)
 6 
 7 (define (helper prompt)
 8   (let* ((prompt-data
 9           (string-join
10            (list
11             (string-append
12              "{\"prompt\": \""
13              prompt
14              "\", \"n_predict\": 256, \"top_k\": 1}"))))
15          (ignore (displayln prompt-data))
16          (p
17           (post
18            "http://localhost:8080/completion"
19            #:data prompt-data))
20          (r (response-json p)))
21     (hash-ref r 'content)))
22 
23 (define (question question)
24   (helper (string-append "Answer: " question)))
25 
26 (define (completion prompt)
27   (helper
28    (string-append
29     "Continue writing from the following text: "
30     prompt)))

 1 > (question "Mary is 30 and Harry is 25. Who is older?")
 2 {"prompt": "Answer: Mary is 30 and Harry is 25. Who is older?", "n_predict": 256, "t\
 3 op_k": 1}
 4 "\nAnswer: Mary is older than Harry."
 5 > (completion "Frank bought a new sports car. Frank drove")
 6 {"prompt": "Continue writing from the following text: Frank bought a new sports car.\
 7  Frank drove", "n_predict": 256, "top_k": 1}
 8 " his new sports car to work every day. He was very happy with his new sports car. O\
 9 ne day, while he was driving his new sports car, he saw a beautiful girl walking on \
10 the side of the road. He stopped his new sports car and asked her if she needed a ri\
11 de. The beautiful girl said yes, so Frank gave her a ride in his new sports car. The\
12 y talked about many things during the ride to work. When they arrived at work, Frank\
13  asked the beautiful girl for her phone number. She gave him her phone number, and h\
14 e promised to call her later that day...."
15 > (question "Mary is 30 and Harry is 25. Who is older and by how much?")
16 {"prompt": "Answer: Mary is 30 and Harry is 25. Who is older and by how much?", "n_p\
17 redict": 256, "top_k": 1}
18 "\nAnswer: Mary is older than Harry by 5 years."
19 > 

 1  $ ollama run mistral
 2 pulling manifest
 3 pulling 6ae280299950... 100% |███████████████████████████████████████████████| (4.1/\
 4 4.1 GB, 13 MB/s)           
 5 pulling fede2d8d6c1f... 100% |██████████████████████████████████████████████████████\
 6 | (29/29 B, 20 B/s)        
 7 pulling b96850d2e482... 100% |███████████████████████████████████████████████████| (\
 8 307/307 B, 170 B/s)        
 9 verifying sha256 digest
10 writing manifest
11 removing any unused layers
12 success
13 >>> Mary is 30 and Bill is 25. Who is older and by how much?
14 Mary is older than Bill by 5 years.
15 
16 >>> /?
17 Available Commands:
18   /set         Set session variables
19   /show        Show model information
20   /bye         Exit
21   /?, /help    Help for a command
22 
23 Use """ to begin a multi-line message.
24 
25 >>>

 1 $ ollama run mistral
 2 >>> ^D
 3 $ ollama run mistral
 4 >>> If I am driving between Sedona Arizona and San Diego, what sites should I visit \
 5 as a tourist?
 6     
 7 There are many great sites to visit when driving from Sedona, Arizona to San Diego. \
 8 Here are some 
 9 suggestions:
10 
11 * Grand Canyon National Park - A must-see attraction in the area, the Grand Canyon i\
12 s a massive and 
13 awe-inspiring natural wonder that offers countless opportunities for outdoor activit\
14 ies such as hiking, 
15 camping, and rafting.
16 * Yuma Territorial Prison State Historic Park - Located in Yuma, Arizona, this forme\
17 r prison was once the 
18 largest and most secure facility of its kind in the world. Today, visitors can explo\
19 re the site and learn 
20 about its history through exhibits and guided tours.
21 * Joshua Tree National Park - A unique and otherworldly landscape in southern Califo\
22 rnia, Joshua Tree 
23 National Park is home to a variety of natural wonders, including towering trees, gia\
24 nt boulders, and 
25 scenic trails for hiking and camping.
26 * La Jolla Cove - Located just north of San Diego, La Jolla Cove is a beautiful beac\
27 h and tidal pool area 
28 that offers opportunities for snorkeling, kayaking, and exploring marine life.
29 * Balboa Park - A cultural and recreational hub in the heart of San Diego, Balboa Pa\
30 rk is home to numerous
31 museums, gardens, theaters, and other attractions that offer a glimpse into the city\
32 's history and culture.
33 
34 >>> 

 1 (require net/http-easy)
 2 (require racket/set)
 3 (require pprint)
 4 
 5 (define (helper prompt)
 6   (let* ((prompt-data
 7           (string-join
 8            (list
 9             (string-append
10              "{\"prompt\": \""
11              prompt
12              "\", \"model\": \"mistral\", \"stream\": false}"))))
13          (ignore (displayln prompt-data))
14          (p
15           (post
16            "http://localhost:11434/api/generate"
17            #:data prompt-data))
18          (r (response-json p)))
19     (hash-ref r 'response)))
20 
21 (define (question-ollama-ai-local question)
22   (helper (string-append "Answer: " question)))
23 
24 (define (completion-ollama-ai-local prompt)
25   (helper
26    (string-append
27     "Continue writing from the following text: "
28     prompt)))
29 
30 ;; EMBEDDINGS:
31 
32 (define (embeddings-ollama text)
33     (let* ((prompt-data
34             (string-join
35              (list
36               (string-append
37                "{\"prompt\": \"" text "\","
38                " \"model\": \"mistral\"}"))))
39            (p
40             (post
41              "http://localhost:11434/api/embeddings"
42              #:data prompt-data))
43            (r (response-json p)))
44       (hash-ref r 'embedding)))
45 
46 
47 ;; (embeddings-ollama "Here is an article about llamas...")

 1 > (question "Mary is 30 and Harry is 25. Who is older and by how much?")
 2 {"prompt": "Answer: Mary is 30 and Harry is 25. Who is older and by how much?", "mod\
 3 el": "mistral", "stream": false}
 4 "Answer: Mary is older than Harry by 5 years."
 5 > (completion "Frank bought a new sports car. Frank drove")
 6 {"prompt": "Continue writing from the following text: Frank bought a new sports car.\
 7  Frank drove", "model": "mistral", "stream": false}
 8 "Frank drove his new sports car around town, enjoying the sleek design and powerful \
 9 engine. The car was a bright red, which caught the attention of everyone on the road\
10 . Frank couldn't help but smile as he cruised down the highway, feeling the wind in \
11 his hair and the sun on his face.\n\nAs he drove, Frank couldn't resist the urge to \
12 test out the car's speed and agility. He weaved through traffic, expertly maneuverin\
13 g the car around curves and turns. The car handled perfectly, and Frank felt a rush \
14 of adrenaline as he pushed it to its limits.\n\nEventually, Frank found himself at a\
15  local track where he could put his new sports car to the test. He revved up the eng\
16 ine and took off down the straightaway, hitting top speeds in no time. The car handl\
17 ed like a dream, and Frank couldn't help but feel a sense of pride as he crossed the\
18  finish line.\n\nAfterwards, Frank parked his sports car and walked over to a nearby\
19  café to grab a cup of coffee. As he sat outside, sipping his drink and watching the\
20  other cars drive by, he couldn't help but think about how much he loved his new rid\
21 e. It was the perfect addition to his collection of cars, and he knew he would be dr\
22 iving it for years to come."
23 > 

  1 #lang racket
  2 
  3 (require db)
  4 (require llmapis)
  5 
  6 (provide create-document QA)
  7 
  8 ; Function to convert list of floats to string representation
  9 (define (floats->string floats)
 10   (string-join (map number->string floats) " "))
 11 
 12 ; Function to convert string representation back to list of floats
 13 (define (string->floats str)
 14   (map string->number (string-split str)))
 15 
 16 (define (read-file infile)
 17   (with-input-from-file infile
 18     (lambda ()
 19       (let ((contents (read)))
 20         contents))))
 21 
 22 (define (join-strings separator list)
 23   (string-join list separator))
 24 
 25 (define (truncate-string string length)
 26   (substring string 0 (min length (string-length string))))
 27 
 28 (define (interleave list1 list2)
 29   (if (or (null? list1) (null? list2))
 30       (append list1 list2)
 31       (cons (car list1)
 32             (cons (car list2)
 33                   (interleave (cdr list1) (cdr list2))))))
 34 
 35 (define (break-into-chunks text chunk-size)
 36   (let loop ((start 0) (chunks '()))
 37     (if (>= start (string-length text))
 38         (reverse chunks)
 39         (loop (+ start chunk-size)
 40               (cons (substring text start (min (+ start chunk-size) (string-length t\
 41 ext))) chunks)))))
 42 
 43 (define (string-to-list str)
 44   (map string->number (string-split str)))
 45 
 46 (define (decode-row row)
 47   (let ((id (vector-ref row 0))
 48         (context (vector-ref row 1))
 49         (embedding (string-to-list (read-line (open-input-string (vector-ref row 2))\
 50 ))))
 51     (list id context embedding)))
 52 
 53 (define
 54   db
 55   (sqlite3-connect #:database "test.db" #:mode 'create
 56                    #:use-place #t))
 57 
 58 (with-handlers ([exn:fail? (lambda (ex) (void))])
 59   (query-exec
 60    db
 61    "CREATE TABLE documents (document_path TEXT, content TEXT, embedding TEXT);"))
 62 
 63 (define (insert-document document-path content embedding)
 64   (printf "~%insert-document:~%  content:~a~%~%" content)
 65   (query-exec
 66    db
 67    "INSERT INTO documents (document_path, content, embedding) VALUES (?, ?, ?);"
 68    document-path content (floats->string embedding)))
 69 
 70 (define (get-document-by-document-path document-path)
 71   (map decode-row
 72        (query-rows
 73          db
 74          "SELECT * FROM documents WHERE document_path = ?;"
 75          document-path)))
 76 
 77 (define (all-documents)
 78   (map
 79    decode-row
 80    (query-rows
 81     db
 82     "SELECT * FROM documents;")))
 83    
 84 (define (create-document fpath)
 85   (let ((contents (break-into-chunks (file->string fpath) 200)))
 86     (for-each
 87      (lambda (content)
 88        (with-handlers ([exn:fail? (lambda (ex) (void))])
 89          (let ((embedding (embeddings-openai content)))
 90            (insert-document fpath content embedding))))
 91      contents)))
 92 
 93 ;; Assuming a function to fetch documents from database
 94 (define (execute-to-list db query)
 95   (query-rows db query))
 96 
 97 ;; dot product of two lists of floating point numbers:
 98 (define (dot-product a b) 
 99   (cond
100     [(or (null? a) (null? b)) 0]
101     [else
102      (+ (* (car a) (car b))
103         (dot-product (cdr a) (cdr b)))]))
104 
105 (define (semantic-match query custom-context [cutoff 0.7])
106   (let ((emb (embeddings-openai query))
107         (ret '()))
108     (for-each
109      (lambda (doc)
110        (let* ((context (second doc))
111               (embedding (third doc))
112               (score (dot-product emb embedding)))
113          (when (> score cutoff)
114            (set! ret (cons context ret)))))
115      (all-documents))
116     (printf "~%semantic-search: ret=~a~%" ret)
117     (let* ((context (string-join (reverse ret) " . "))
118            (query-with-context (string-join (list context custom-context "Question:"\
119  query) " ")))
120       (question-openai query-with-context))))
121 
122 (define (QA query [quiet #f])
123   (let ((answer (semantic-match query "")))
124     (unless quiet
125       (printf "~%~%** query: ~a~%** answer: ~a~%~%" query answer))
126     answer))
127 
128 (define (CHAT)
129   (let ((messages '(""))
130         (responses '("")))
131     (let loop ()
132       (printf "~%Enter chat (STOP or empty line to stop) >> ")
133       (let ((string (read-line)))
134         (cond
135          ((or (string=? string "STOP") (< (string-length string) 1))
136           (list (reverse messages) (reverse responses)))
137          (else
138           (let* ((custom-context
139                   (string-append
140                    "PREVIOUS CHAT: "
141                    (string-join (reverse messages) " ")))
142                  (response (semantic-match string custom-context)))
143             (set! messages (cons string messages))
144             (set! responses (cons response responses))
145             (printf "~%Response: ~a~%" response)
146             (loop))))))))
147 
148 ;; ... test code ...
149 
150 (define (test)
151   "Test Semantic Document Search Using GPT APIs and local vector database"
152   (create-document
153     "/Users/markw/GITHUB/Racket-AI-book-code/embeddingsdb/data/sports.txt")
154   (create-document
155     "/Users/markw/GITHUB/Racket-AI-book-code/embeddingsdb/data/chemistry.txt")
156   (QA "What is the history of the science of chemistry?")
157   (QA "What are the advantages of engaging in sports?"))

 1 > (QA "What is the history of the science of chemistry?")
 2 ** query: What is the history of the science of chemistry?
 3 ** answer: The history of the science of chemistry dates back thousands of years. An\
 4 cient civilizations such as the Egyptians, Greeks, and Chinese were experimenting wi\
 5 th various substances and observing chemical reactions even before the term "chemist\
 6 ry" was coined.
 7 
 8 The foundations of modern chemistry can be traced back to the works of famous schola\
 9 rs such as alchemists in the Middle Ages. Alchemists sought to transform common meta\
10 ls into gold and discover elixirs of eternal life. Although their practices were oft\
11 en based on mysticism and folklore, it laid the groundwork for the understanding of \
12 chemical processes and experimentation.
13 
14 In the 17th and 18th centuries, significant advancements were made in the field of c\
15 hemistry. Prominent figures like Robert Boyle and Antoine Lavoisier began to underst\
16 and the fundamental principles of chemical reactions and the concept of elements. La\
17 voisier is often referred to as the "father of modern chemistry" for his work in est\
18 ablishing the law of conservation of mass and naming and categorizing elements.
19 
20 Throughout the 19th and 20th centuries, chemistry continued to progress rapidly. The\
21  development of the periodic table by Dmitri Mendeleev in 1869 revolutionized the or\
22 ganization of elements. The discovery of new elements, the formulation of atomic the\
23 ory, and the understanding of chemical bonding further expanded our knowledge.
24 
25 Chemistry also played a crucial role in various industries and technologies, such as\
26  the development of synthetic dyes, pharmaceuticals, plastics, and materials. The em\
27 ergence of quantum mechanics and spectroscopy in the early 20th century opened up ne\
28 w avenues for understanding the behavior of atoms and molecules.
29 
30 Today, chemistry is an interdisciplinary science that encompasses various fields suc\
31 h as organic chemistry, inorganic chemistry, physical chemistry, analytical chemistr\
32 y, and biochemistry. It continues to evolve and make significant contributions to so\
33 ciety, from developing sustainable materials to understanding biological processes a\
34 nd addressing global challenges such as climate change.
35 
36 In summary, the history of the science of chemistry spans centuries, starting from a\
37 ncient civilizations to the present day, with numerous discoveries and advancements \
38 shaping our understanding of the composition, properties, and transformations of mat\
39 ter.

1 ZorroOnian Alcohol is another organic compound with the formula C 6 H 10 O.

 1 > (create-document
 2    "/Users/markw/GITHUB/Racket-AI-book-code/embeddingsdb/data/chemistry.txt")
 3 
 4 insert-document:
 5   content:Amyl alcohol is an organic compound with the formula C 5 H 12 O. ZorroOnia\
 6 n Alcohol is another organic compound with the formula C 6 H 10 O. All eight isomers\
 7  of amyl alcohol are known.
 8 
 9   ...
10 
11 > (QA "what is the formula for ZorroOnian Alcohol")
12 
13 ** query: what is the formula for ZorroOnian Alcohol
14 ** answer: The formula for ZorroOnian Alcohol is C6H10O.

 1 Enter chat (STOP or empty line to stop) >> who is the chemist Robert Boyle
 2 
 3 Response: Robert Boyle was an Irish chemist and physicist who is known as one of the\
 4  pioneers of modern chemistry. He is famous for Boyle's Law, which describes the inv\
 5 erse relationship between the pressure and volume of a gas, and for his experiments \
 6 on the properties of gases. He lived from 1627 to 1691.
 7 
 8 Enter chat (STOP or empty line to stop) >> Where was he born?
 9 
10 Response: Robert Boyle was born in Lismore Castle, County Waterford, Ireland.
11 
12 Enter chat (STOP or empty line to stop) >> 

1 #lang racket/base
2 
3 (require "fasttag.rkt")
4 (require "names.rkt")
5 
6 (provide parts-of-speech)
7 (provide find-human-names)
8 (provide find-place-names)

#lang racket

(require srfi/13) ; the string SRFI
(require racket/runtime-path)

(provide parts-of-speech)

;; FastTag.lisp
;;
;; Conversion of KnowledgeBooks.com Java FastTag to Scheme
;;
;; Copyright 2002 by Mark Watson.  All rights reserved.
;;


(display "loading lex-hash...")
(log-info "loading lex-hash" "starting")
(define lex-hash
  (let ((hash (make-hash)))
    (with-input-from-file
        (string-append (path->string my-data-path) "/tag.dat")
      (lambda ()
        (let loop ()
          (let ((p (read)))
            (if (list? p) (hash-set! hash (car p) (cadr p)) #f)
            (if (eof-object? p) #f (loop))))))
    hash))
(display "...done.")
(log-info "loading lex-hash" "ending")

(define (string-suffix? pattern str)
  (let loop ((i (- (string-length pattern) 1)) (j (- (string-length str) 1)))
    (cond
     ((negative? i) #t)
     ((negative? j) #f)
     ((char=? (string-ref pattern i) (string-ref str j))
      (loop (- i 1) (- j 1)))
     (else #f))))
;;
; parts-of-speech
;
;  input: a vector of words (each a string)
;  output: a vector of parts of speech
;;

(define (parts-of-speech words)
  (display "\n+ tagging:") (display words)
  (let ((ret '())
        (r #f)
        (lastRet #f)
        (lastWord #f))
    (for-each
     (lambda (w)
       (set! r (hash-ref lex-hash w #f))
       ;; if this word is not in the hash table, try making it ll lower case:
       (if (not r)
           (set! r '("NN"))
           #f)
       ;;(if (list? r) (set! r (car r))))
       ;; apply transformation rules:
       
       ; rule 1: DT, {VBD, VBP, VB} --> DT, NN
       (if (equal? lastRet "DT")
           (if (or
                (equal? r "VBD")
                (equal? r "VBP")
                (equal? r "VB"))
               (set! r '("NN"))
               #f)
           #f)
       ; rule 2: convert a noun to a number if a "." appears in the word
       (if (string-contains "." w) (set! r '("CD")) #f)

       ; rule 3: convert a noun to a past participle if word ends with "ed"
       (if (equal? (member "N" r) 0)
           (let* ((slen (string-length w)))
             (if (and
                  (> slen 1)
                  (equal? (substring w (- slen 2)) "ed"))
                 (set! r "VBN") #f))
           #f)

       ; rule 4: convert any type to an adverb if it ends with "ly"
       (let ((i (string-suffix? "ly" w)))
         (if (equal? i (- (string-length w) 2))
             (set! r '("RB"))
             #f))

       ; rule 5: convert a common noun (NN or NNS) to an adjective
       ;         if it ends with "al"
       (if (or
            (member "NN" r)
            (member "NNS" r))
           (let ((i (string-suffix? "al" w)))
             (if (equal? i (- (string-length w) 2))
                 (set! r '("RB"))
                 #f))
           #f)

       ; rule 6: convert a noun to a verb if the receeding word is "would"
       (if (equal? (member "NN" r) 0)
           (if (equal? lastWord "would")
               (set! r '("VB"))
               #f)
           #f)

       ; rule 7: if a word has been categorized as a common noun and it
       ;         ends with "s", then set its type to a plural noun (NNS)
       (if (member "NN" r)
           (let ((i (string-suffix? "s" w)))
             (if (equal? i (- (string-length w) 1))
                 (set! r '("NNS"))
                 #f))
           #f)

       ; rule 8: convert a common noun to a present participle verb
       ;         (i.e., a gerand)
       (if (equal? (member "NN" r) 0)
           (let ((i (string-suffix? "ing" w)))
             (if (equal? i (- (string-length w) 3))
                 (set! r '("VBG"))
                 #f))
           #f)

       (set! lastRet ret)
       (set! lastWord w)
       (set! ret (cons (first r) ret)))
     (vector->list words))  ;; not very efficient !!
    (list->vector (reverse ret))))

#lang racket

(require "fasttag.rkt")
(require racket/runtime-path)
(provide find-human-names)
(provide find-place-names)

(define (process-one-word-per-line file-path func)
  (with-input-from-file file-path
	(lambda ()
	  (let loop ()
		(let ([l (read-line)])
		  (if (equal? l #f) #f (func l))
		  (if (eof-object? l) #f (loop)))))))

(define *last-name-hash* (make-hash))
(process-one-word-per-line 
  (string-append
    (path->string my-data-path)
    "/human_names/names.last")
  (lambda (x) (hash-set! *last-name-hash* x #t)))
(define *first-name-hash* (make-hash))
(process-one-word-per-line
  (string-append
    (path->string my-data-path)
    "/human_names/names.male")
  (lambda (x) (hash-set! *first-name-hash* x #t)))
(process-one-word-per-line
  (string-append
    (path->string my-data-path)
    "/human_names/names.female")
  (lambda (x) (hash-set! *first-name-hash* x #t)))

(define *place-name-hash* (make-hash))
(process-one-word-per-line
  (string-append
    (path->string my-data-path)
    "/placenames.txt")
  (lambda (x) (hash-set!  *place-name-hash* x #t)))
(define *name-prefix-list*
  '("Mr" "Mrs" "Ms" "Gen" "General" "Maj" "Major" "Doctor" "Vice" "President" 
	"Lt" "Premier" "Senator" "Congressman" "Prince" "King" "Representative"
	"Sen" "St" "Dr"))

(define (not-in-list-find-names-helper a-list start end)
  (let ((rval #t))
    (do ((x a-list (cdr x)))
	((or
	  (null? x)
	  (let ()
	    (if (or
		 (and
		  (>= start (caar x))
		  (<= start (cadar x)))
		 (and
		  (>= end (caar x))
		  (<= end (cadar x))))
		(set! rval #f)
                #f)
	    (not rval)))))
    rval))

;; return a list of sublists, each sublist looks like:
;;    (("John" "Smith") (11 12) 0.75) ; last number is an importance rating
(define (find-human-names word-vector exclusion-list)
  (define (score result-list)
    (- 1.0 (* 0.2 (- 4 (length result-list)))))
  (let ((tags (parts-of-speech word-vector))
        (ret '()) (ret2 '()) (x '())
        (len (vector-length word-vector))
        (word #f))
    (display "\ntags: ") (display tags)
    ;;(dotimes (i len)
    (for/list ([i (in-range len)])
      (set! word (vector-ref word-vector i))
      (display "\nword: ") (display word)
      ;; process 4 word names:      HUMAN NAMES
      (if (< i (- len 3))
          ;; case #1: single element from '*name-prefix-list*'
          (if (and
               (not-in-list-find-names-helper ret i (+ i 4))
               (not-in-list-find-names-helper exclusion-list i (+ i 4))
               (member word *name-prefix-list*)
               (equal? "." (vector-ref word-vector (+ i 1)))
               (hash-ref *first-name-hash* (vector-ref word-vector (+ i 2)) #f)
               (hash-ref *last-name-hash* (vector-ref word-vector (+ i 3)) #f))
              (if (and
                   (string-prefix? (vector-ref tags (+ i 2)) "NN")
                   (string-prefix? (vector-ref tags (+ i 3)) "NN"))
                  (set! ret (cons (list i (+ i 4)) ret))
                  #f)
              #f)
          ;; case #1: two elements from '*name-prefix-list*'
          (if (and
               (not-in-list-find-names-helper ret i (+ i 4))
               (not-in-list-find-names-helper exclusion-list i (+ i 4))
               (member word *name-prefix-list*)
               (member (vector-ref word-vector (+ i 1)) *name-prefix-list*)
               (hash-ref *first-name-hash* (vector-ref word-vector (+ i 2)) #f)
               (hash-ref *last-name-hash* (vector-ref word-vector (+ i 3)) #f))
              (if (and
                   (string-prefix? (vector-ref tags (+ i 2)) "NN")
                   (string-prefix? (vector-ref tags (+ i 3)) "NN"))
                  (set! ret (cons (list i (+ i 4)) ret))
                  #f)
              #f))
      ;; process 3 word names:      HUMAN NAMES
      (if (< i (- len 2))
          (if (and
               (not-in-list-find-names-helper ret i (+ i 3))
               (not-in-list-find-names-helper exclusion-list i (+ i 3)))
              (if (or
                   (and
                    (member word *name-prefix-list*)
                    (hash-ref *first-name-hash* (vector-ref word-vector (+ i 1)) #f)
                    (hash-ref *last-name-hash* (vector-ref word-vector (+ i 2)) #f)
                    (string-prefix? (vector-ref tags (+ i 1)) "NN")
                    (string-prefix? (vector-ref tags (+ i 2)) "NN"))
                   (and
                    (member word *name-prefix-list*)
                    (member (vector-ref word-vector (+ i 1)) *name-prefix-list*)
                    (hash-ref *last-name-hash* (vector-ref word-vector (+ i 2)) #f)
                    (string-prefix? (vector-ref tags (+ i 1)) "NN")
                    (string-prefix? (vector-ref tags (+ i 2)) "NN"))
                   (and
                    (member word *name-prefix-list*)
                    (equal? "." (vector-ref word-vector (+ i 1)))
                    (hash-ref *last-name-hash* (vector-ref word-vector (+ i 2)) #f)
                    (string-prefix? (vector-ref tags (+ i 2)) "NN"))
                   (and
                    (hash-ref *first-name-hash* word #f)
                    (hash-ref *first-name-hash* (vector-ref word-vector (+ i 1)) #f)
                    (hash-ref *last-name-hash* (vector-ref word-vector (+ i 2)) #f)
                    (string-prefix? (vector-ref tags i) "NN")
                    (string-prefix? (vector-ref tags (+ i 1)) "NN")
                    (string-prefix? (vector-ref tags (+ i 2)) "NN")))
                  (set! ret (cons (list i (+ i 3)) ret))
                  #f)
              #f)
          #f)
      ;; process 2 word names:      HUMAN NAMES
      (if (< i (- len 1))
          (if (and
               (not-in-list-find-names-helper ret i (+ i 2))
               (not-in-list-find-names-helper exclusion-list i (+ i 2)))
              (if (or
                   (and
                    (member word '("Mr" "Mrs" "Ms" "Doctor" "President" "Premier"))
                    (string-prefix? (vector-ref tags (+ i 1)) "NN")
                    (hash-ref *last-name-hash* (vector-ref word-vector (+ i 1)) #f))
                   (and
                    (hash-ref *first-name-hash* word #f)
                    (hash-ref *last-name-hash* (vector-ref word-vector (+ i 1)) #f)
                    (string-prefix? (vector-ref tags i) "NN")
                    (string-prefix? (vector-ref tags (+ i 1)) "NN")))
                  (set! ret (cons (list i (+ i 2)) ret))
                  #f)
              #f)
          #f)
      ;; 1 word names:      HUMAN NAMES
      (if (hash-ref *last-name-hash* word #f)
          (if (and
               (string-prefix? (vector-ref tags i) "NN")
               (not-in-list-find-names-helper ret i (+ i 1))
               (not-in-list-find-names-helper exclusion-list i (+ i 1)))
              (set! ret (cons (list i (+ i 1)) ret))
              #f)
          #f))
    ;; TBD: calculate importance rating based on number of occurences of name in tex\
t:
    (set! ret2
          (map (lambda (index-pair)
                 (string-replace
                  (string-join (vector->list (vector-copy  word-vector (car index-pa\
ir) (cadr index-pair))))
                  " ." "."))
               ret))
    ret2))

(define (find-place-names word-vector exclusion-list)  ;; PLACE
  (define (score result-list)
    (- 1.0 (* 0.2 (- 4 (length result-list)))))
  (let ((tags (parts-of-speech word-vector))
        (ret '()) (ret2 '()) (x '())
        (len (vector-length word-vector))
        (word #f))
    (display "\ntags: ") (display tags)
    ;;(dotimes (i len)
    (for/list ([i (in-range len)])
      (set! word (vector-ref word-vector i))
      (display "\nword: ") (display word) (display "\n")
      ;; process 3 word names: PLACE
      (if (< i (- len 2))
          (if (and
               (not-in-list-find-names-helper ret i (+ i 3))
               (not-in-list-find-names-helper exclusion-list i (+ i 3)))
              (let ((p-name (string-append word " " (vector-ref word-vector (+ i 1))\
 " " (vector-ref word-vector (+ i 2)))))
                (if (hash-ref *place-name-hash* p-name #f)
                    (set! ret (cons (list i (+ i 3)) ret))
                  #f))
              #f)
          #f)
      ;; process 2 word names:  PLACE
      (if (< i (- len 1))
          (if (and
               (not-in-list-find-names-helper ret i (+ i 2))
               (not-in-list-find-names-helper exclusion-list i (+ i 2)))
              (let ((p-name (string-append word " " (vector-ref word-vector (+ i 1))\
)))
                (if (hash-ref *place-name-hash* p-name #f)
                    (set! ret (cons (list i (+ i 2)) ret))
                    #f)
                #f)
              #f)
          #f)
      ;; 1 word names:   PLACE
      (if (hash-ref *place-name-hash* word #f)
          (if (and
               (string-prefix? (vector-ref tags i) "NN")
               (not-in-list-find-names-helper ret i (+ i 1))
               (not-in-list-find-names-helper exclusion-list i (+ i 1)))
              (set! ret (cons (list i (+ i 1)) ret))
              #f)
          #f))
    ;; TBD: calculate importance rating based on number of occurences of name in tex\
t: can use (count-substring..) defined in utils.rkt
    (set! ret2
          (map (lambda (index-pair)
                 (string-join (vector->list (vector-copy  word-vector (car index-pai\
r) (cadr index-pair))) " "))
               ret))
    ret2))

#|
(define nn (find-human-names '#("President" "George" "Bush" "went" "to" "San" "Diego\
" "to" "meet" "Ms" "." "Jones" "and" "Gen" "." "Pervez" "Musharraf" ".") '()))
(display (find-place-names '#("George" "Bush" "went" "to" "San" "Diego" "and" "Londo\
n") '()))
|#