feat doc tree

Author: coltea

backend/api/crawler/v1/crawler.go

@@ -1,10 +1,38 @@
 package v1
 
-import "github.com/chaitin/panda-wiki/consts"
+import (
+	"github.com/chaitin/panda-wiki/consts"
+	"github.com/chaitin/panda-wiki/pkg/anydoc"
+)
 
-type ScrapeReq struct {
-	URL  string `json:"url" validate:"required"`
-	KbID string `json:"kb_id" validate:"required"`
+type CrawlerParseReq struct {
+	Key           string               `json:"key"`
+	KbID          string               `json:"kb_id" validate:"required"`
+	CrawlerSource consts.CrawlerSource `json:"crawler_source" validate:"required"`
+	Filename      string               `json:"filename"`
+	FeishuSetting FeishuSetting        `json:"feishu_setting" validate:"required_if=CrawlerSource feishu"`
+}
+
+type FeishuSetting struct {
+	UserAccessToken string `json:"user_access_token"`
+	AppID           string `json:"app_id"`
+	AppSecret       string `json:"app_secret"`
+	SpaceId         string `json:"space_id"`
+}
+
+type CrawlerParseResp struct {
+	ID   string        `json:"id"`
+	Docs *anydoc.Child `json:"docs"`
+}
+
+type CrawlerExportReq struct {
+	KbID  string `json:"kb_id" validate:"required"`
+	ID    string `json:"id" validate:"required"`
+	DocID string `json:"doc_id" validate:"required"`
+}
+
+type CrawlerExportResp struct {
+	TaskId string `json:"task_id"`
 }
 
 type ScrapeResp struct {

backend/consts/parse.go

@@ -0,0 +1,42 @@
+package consts
+
+type CrawlerSource string
+
+const (
+	// CrawlerSourceUrl key或url形式 直接走parse接口
+	CrawlerSourceUrl     CrawlerSource = "url"
+	CrawlerSourceRSS     CrawlerSource = "rss"
+	CrawlerSourceSitemap CrawlerSource = "sitemap"
+	CrawlerSourceNotion  CrawlerSource = "notion"
+	CrawlerSourceFeishu  CrawlerSource = "feishu"
+
+	// CrawlerSourceFile file形式 需要先走upload接口先上传文件
+	CrawlerSourceFile       CrawlerSource = "file"
+	CrawlerSourceEpub       CrawlerSource = "epub"
+	CrawlerSourceYuque      CrawlerSource = "yuque"
+	CrawlerSourceSiyuan     CrawlerSource = "siyuan"
+	CrawlerSourceMindoc     CrawlerSource = "mindoc"
+	CrawlerSourceWikijs     CrawlerSource = "wikijs"
+	CrawlerSourceConfluence CrawlerSource = "confluence"
+)
+
+type CrawlerSourceType string
+
+const (
+	CrawlerSourceTypeFile CrawlerSourceType = "file"
+	CrawlerSourceTypeUrl  CrawlerSourceType = "url"
+	CrawlerSourceTypeKey  CrawlerSourceType = "key"
+)
+
+func (c CrawlerSource) Type() CrawlerSourceType {
+	switch c {
+	case CrawlerSourceNotion, CrawlerSourceFeishu:
+		return CrawlerSourceTypeKey
+	case CrawlerSourceUrl, CrawlerSourceRSS, CrawlerSourceSitemap:
+		return CrawlerSourceTypeUrl
+	case CrawlerSourceFile, CrawlerSourceEpub, CrawlerSourceYuque, CrawlerSourceSiyuan, CrawlerSourceMindoc, CrawlerSourceWikijs, CrawlerSourceConfluence:
+		return CrawlerSourceTypeFile
+	default:
+		return ""
+	}
+}