initial commit

First commit for initial xquery project.

Author: zhengchun

.gitignore

@@ -0,0 +1,32 @@
+# vscode
+.vscode
+debug
+*.test
+
+./build
+
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof

LICENSE

@@ -0,0 +1,17 @@
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -0,0 +1,6 @@
+XQuery
+====
+XQuery is a package to extract data from HTML and XML using XPath selectors.
+
+Installing
+====

html/select.go

@@ -0,0 +1,175 @@
+package html
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/antchfx/gxpath"
+	"github.com/antchfx/gxpath/xpath"
+	"golang.org/x/net/html"
+)
+
+// Selector is a XPath selector for HTML.
+type Selector struct{}
+
+func (s *Selector) Find(top *html.Node, expr string) []*html.Node {
+	nav := &htmlNodeNavigator{curr: top, root: top, attr: -1}
+	var elems []*html.Node
+	t := s.Select(nav, expr)
+	for t.MoveNext() {
+		elems = append(elems, (t.Current().(*htmlNodeNavigator)).curr)
+	}
+	return elems
+}
+
+func (s *Selector) FindOne(top *html.Node, expr string) *html.Node {
+	nav := &htmlNodeNavigator{curr: top, root: top, attr: -1}
+	t := s.Select(nav, expr)
+	var elem *html.Node
+	if t.MoveNext() {
+		elem = (t.Current().(*htmlNodeNavigator)).curr
+	}
+	return elem
+}
+
+func (s *Selector) Select(root xpath.NodeNavigator, expr string) *gxpath.NodeIterator {
+	return gxpath.Select(root, expr)
+}
+
+type htmlNodeNavigator struct {
+	root, curr *html.Node
+	attr       int
+}
+
+func (h *htmlNodeNavigator) NodeType() xpath.NodeType {
+	switch h.curr.Type {
+	case html.CommentNode:
+		return xpath.CommentNode
+	case html.TextNode:
+		return xpath.TextNode
+	case html.DocumentNode:
+		return xpath.RootNode
+	case html.ElementNode:
+		if h.attr != -1 {
+			return xpath.AttributeNode
+		}
+		return xpath.ElementNode
+	}
+	panic(fmt.Sprintf("unknown HTML node type: %v", h.curr.Type))
+}
+
+func (h *htmlNodeNavigator) LocalName() string {
+	if h.attr != -1 {
+		return h.curr.Attr[h.attr].Key
+	}
+	return h.curr.Data
+}
+
+func (*htmlNodeNavigator) Prefix() string {
+	return ""
+}
+
+func (h *htmlNodeNavigator) Value() string {
+	switch h.curr.Type {
+	case html.CommentNode:
+		return h.curr.Data
+	case html.ElementNode:
+		if h.attr != -1 {
+			return h.curr.Attr[h.attr].Val
+		}
+		return InnerText(h.curr)
+	case html.TextNode:
+		return h.curr.Data
+	}
+	return ""
+}
+
+func (h *htmlNodeNavigator) Copy() xpath.NodeNavigator {
+	n := *h
+	return &n
+}
+
+func (h *htmlNodeNavigator) MoveToRoot() {
+	h.curr = h.root
+}
+
+func (h *htmlNodeNavigator) MoveToParent() bool {
+	if node := h.curr.Parent; node != nil {
+		h.curr = node
+		return true
+	}
+	return false
+}
+
+func (h *htmlNodeNavigator) MoveToNextAttribute() bool {
+	if h.attr >= len(h.curr.Attr)-1 {
+		return false
+	}
+	h.attr++
+	return true
+}
+
+func (h *htmlNodeNavigator) MoveToChild() bool {
+	if node := h.curr.FirstChild; node != nil {
+		h.curr = node
+		return true
+	}
+	return false
+}
+
+func (h *htmlNodeNavigator) MoveToFirst() bool {
+	if h.curr.PrevSibling == nil {
+		return false
+	}
+	for {
+		node := h.curr.PrevSibling
+		if node == nil {
+			break
+		}
+		h.curr = node
+	}
+	return true
+}
+
+func (h *htmlNodeNavigator) String() string {
+	return h.Value()
+}
+
+func (h *htmlNodeNavigator) MoveToNext() bool {
+	if node := h.curr.NextSibling; node != nil {
+		h.curr = node
+		return true
+	}
+	return false
+}
+
+func (h *htmlNodeNavigator) MoveToPrevious() bool {
+	if node := h.curr.PrevSibling; node != nil {
+		h.curr = node
+		return true
+	}
+	return false
+}
+
+func (h *htmlNodeNavigator) MoveTo(other xpath.NodeNavigator) bool {
+	node, ok := other.(*htmlNodeNavigator)
+	if !ok || node.root != h.root {
+		return false
+	}
+
+	h.curr = node.curr
+	h.attr = node.attr
+	return true
+}
+
+// InnerText returns the text between the start and end tags of the object.
+func InnerText(n *html.Node) string {
+	if n.Type == html.TextNode || n.Type == html.CommentNode {
+		return n.Data
+	}
+	var buf bytes.Buffer
+	for child := n.FirstChild; child != nil; child = child.NextSibling {
+		buf.WriteString(InnerText(child))
+	}
+	return buf.String()
+}

html/select_test.go

@@ -0,0 +1,66 @@
+package html
+
+import (
+	"strings"
+	"testing"
+
+	"golang.org/x/net/html"
+)
+
+var (
+	doc = loadHtml()
+	sel = &Selector{}
+)
+
+func TestXPathSelect(t *testing.T) {
+	if node := sel.FindOne(doc, "/html/head/title"); node == nil {
+		t.Fatal("cannot found any node")
+	}
+	if node := sel.FindOne(doc, "//body[@bgcolor]"); node.Attr[0].Val != "ffffff" {
+		t.Fatal("body bgcolor is not #ffffff")
+	}
+	if list := sel.Find(doc, "//a"); len(list) != 2 {
+		t.Fatal("count(//a)!=2")
+	}
+	if list := sel.Find(doc, "//body/child::*"); len(list) != 9 { // ignored textnode
+		t.Fatal("count(//body/child::*)!=9")
+	}
+}
+
+func TestInnerText(t *testing.T) {
+	title := sel.FindOne(doc, "//title")
+	if txt := InnerText(title); strings.TrimSpace(txt) != "your title here" {
+		t.Fatalf("InnerText(//title): %s !=your title here", txt)
+	}
+	head := sel.FindOne(doc, "/html/head")
+	if txt := InnerText(head); strings.TrimSpace(txt) != "your title here" {
+		t.Fatalf("InnerText(/html/head): %s !=your title here", txt)
+	}
+}
+
+func loadHtml() *html.Node {
+	// http://help.websiteos.com/websiteos/example_of_a_simple_html_page.htm
+	var str = `<html>
+<head>
+<title>your title here</title>
+</head>
+<body bgcolor="ffffff">
+<center><img src="clouds.jpg" align="bottom"> </center>
+<hr>
+<a href="http://somegreatsite.com">link name</a>
+is a link to another nifty site
+<h1>this is a header</h1>
+<h2>this is a medium header</h2>
+send me mail at <a href="mailto:[email protected]">[email protected]</a>.
+<p> this is a new paragraph!
+<p> <b>this is a new paragraph!</b>
+<br> <b><i>this is a new sentence without a paragraph break, in bold italics.</i></b>
+<hr>
+</body>
+</html`
+	node, err := html.Parse(strings.NewReader(str))
+	if err != nil {
+		panic(err)
+	}
+	return node
+}

query.go

@@ -0,0 +1,34 @@
+package xquery
+
+import (
+	"fmt"
+
+	"github.com/antchfx/gxpath"
+	"github.com/antchfx/gxpath/xpath"
+	"github.com/antchfx/xquery/html"
+	"github.com/antchfx/xquery/xml"
+)
+
+// Type represents the documents types for XPath.
+type Type int
+
+const (
+	HTML Type = iota
+	XML
+)
+
+// Selector is an interface for XPath search.
+type Selector interface {
+	Select(xpath.NodeNavigator, string) *gxpath.NodeIterator
+}
+
+// New returns new Selector for the specified documents type.
+func New(typ Type) Selector {
+	switch typ {
+	case HTML:
+		return &html.Selector{}
+	case XML:
+		return &xml.Selector{}
+	}
+	panic(fmt.Errorf("unknown type : %d", typ))
+}