Electrum Endpoints and Prometheus

.github/workflows/build-short.yml

@@ -0,0 +1,13 @@
+name: 'Build Hub'
+
+on:
+  push:
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/[email protected]
+        with:
+          go-version: 1.16.5
+      - run: go build .

.github/workflows/build.yml

@@ -0,0 +1,32 @@
+name: 'Build Hub'
+
+on:
+  push:
+    branches:
+      - master
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/[email protected]
+        with:
+          go-version: 1.16.5
+      - run: |
+          sudo apt update && \
+          sudo apt upgrade && \
+          sudo apt-get install autoconf automake libtool curl make g++ unzip && \
+          cd /tmp && \
+          wget https://github.com/protocolbuffers/protobuf/releases/download/v3.17.1/protobuf-all-3.17.1.tar.gz && \
+          tar xfzv protobuf-all-3.17.1.tar.gz && \
+          cd protobuf-3.17.1 && \
+          ./autogen.sh && \
+          ./configure && \
+          make && \
+          sudo make install && \
+          sudo ldconfig && \
+          cd
+      - run: pip install grpcio grpcio-tools github3.py
+      - run: go get github.com/golang/protobuf/protoc-gen-go google.golang.org/grpc/cmd/protoc-gen-go-grpc
+      - run: go build .
+      - run: ./protobuf/build.sh

build_and_push.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Get new tags from remote
+git fetch --tags
+
+# Get latest tag name
+latestTag=$(git describe --tags `git rev-list --tags --max-count=1`)
+
+# Checkout latest tag
+git checkout $latestTag
+
+go build .
+docker build . -t lbry/hub:latest
+docker tag lbry/hub:latest lbry/hub:$latestTag
+docker push lbry/hub:latest
+docker push lbry/hub:$latestTag

go.mod

@@ -5,11 +5,13 @@ go 1.16
 require (
 	github.com/akamensky/argparse v1.2.2
 	github.com/btcsuite/btcutil v1.0.2
+	github.com/go-zeromq/zmq4 v0.13.0
 	github.com/golang/protobuf v1.5.2
 	github.com/lbryio/lbry.go/v2 v2.7.2-0.20210625145058-2b155597bf57
 	github.com/olivere/elastic/v7 v7.0.24
+	github.com/prometheus/client_golang v1.11.0
+	github.com/ybbus/jsonrpc/v2 v2.1.6
 	golang.org/x/net v0.0.0-20210525063256-abc453219eb5 // indirect
-	golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea // indirect
 	golang.org/x/text v0.3.6
 	google.golang.org/genproto v0.0.0-20210524171403-669157292da3 // indirect
 	google.golang.org/grpc v1.38.0

main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"log"
 	"net"
 	"os"
@@ -43,12 +44,30 @@ func GetEnvironmentStandard() map[string]string {
 	})
 }
 
-func parseArgs(searchRequest *pb.SearchRequest) *server.Args {
+/*
+func makeServeCmd(parser *argparse.Parser) *argparse.Command {
+	serveCmd := parser.NewCommand("serve", "start the hub server")
+
+	host := serveCmd.String("", "rpchost", &argparse.Options{Required: false, Help: "host", Default: defaultHost})
+	port := serveCmd.String("", "rpcport", &argparse.Options{Required: false, Help: "port", Default: defaultPort})
+	esHost := serveCmd.String("", "eshost", &argparse.Options{Required: false, Help: "host", Default: defaultEsHost})
+	esPort := serveCmd.String("", "esport", &argparse.Options{Required: false, Help: "port", Default: defaultEsPort})
+	dev := serveCmd.Flag("", "dev", &argparse.Options{Required: false, Help: "port", Default: false})
+
+	return serveCmd
+}
+ */
+
+func parseArgs(searchRequest *pb.SearchRequest, blockReq *pb.BlockRequest) *server.Args {
 
 	environment := GetEnvironmentStandard()
 	parser := argparse.NewParser("hub", "hub server and client")
 
 	serveCmd := parser.NewCommand("serve", "start the hub server")
+	searchCmd := parser.NewCommand("search", "claim search")
+	getblockCmd := parser.NewCommand("getblock", "get block")
+	getblockHeaderCmd := parser.NewCommand("getblockheader", "get block header")
+	subscribeHeaderCmd := parser.NewCommand("subscribeheader", "get block header")
 
 	host := parser.String("", "rpchost", &argparse.Options{Required: false, Help: "host", Default: defaultHost})
 	port := parser.String("", "rpcport", &argparse.Options{Required: false, Help: "port", Default: defaultPort})
@@ -66,6 +85,8 @@ func parseArgs(searchRequest *pb.SearchRequest) *server.Args {
 	channelId := parser.String("", "channel_id", &argparse.Options{Required: false, Help: "channel id"})
 	channelIds := parser.StringList("", "channel_ids", &argparse.Options{Required: false, Help: "channel ids"})
 
+	hash := parser.String("", "hash", &argparse.Options{Required: false, Help: "block hash"})
+
 	// Now parse the arguments
 	err := parser.Parse(os.Args)
 	if err != nil {
@@ -74,7 +95,7 @@ func parseArgs(searchRequest *pb.SearchRequest) *server.Args {
 
 
 	args := &server.Args{
-		Serve: false,
+		CmdType: server.SearchCmd,
 		Host: *host,
 		Port: ":" + *port,
 		EsHost: *esHost,
@@ -102,7 +123,18 @@ func parseArgs(searchRequest *pb.SearchRequest) *server.Args {
 	}
 
 	if serveCmd.Happened() {
-		args.Serve = true
+		args.CmdType = server.ServeCmd
+	} else if searchCmd.Happened() {
+		args.CmdType = server.SearchCmd
+	} else if getblockCmd.Happened() {
+		args.CmdType = server.GetblockCmd
+		blockReq.Verbose = true
+	} else if getblockHeaderCmd.Happened() {
+		args.CmdType = server.GetblockHeaderCmd
+		blockReq.Verbose = true
+	} else if subscribeHeaderCmd.Happened() {
+		args.CmdType = server.SubscribeHeaderCmd
+		blockReq.Verbose = true
 	}
 
 	if *text != "" {
@@ -133,15 +165,21 @@ func parseArgs(searchRequest *pb.SearchRequest) *server.Args {
 		searchRequest.ChannelId = &pb.InvertibleField{Invert: false, Value: *channelIds}
 	}
 
+	if *hash != "" {
+		blockReq.Blockhash = *hash
+	}
+
 	return args
 }
 
 func main() {
+
 	searchRequest := &pb.SearchRequest{}
+	blockReq := &pb.BlockRequest{}
 
-	args := parseArgs(searchRequest)
+	args := parseArgs(searchRequest, blockReq)
 
-	if args.Serve {
+	if args.CmdType == server.ServeCmd {
 
 		l, err := net.Listen("tcp", args.Port)
 		if err != nil {
@@ -154,6 +192,7 @@ func main() {
 
 		log.Printf("listening on %s\n", l.Addr().String())
 		log.Println(s.Args)
+		go s.PromethusEndpoint("2112", "metrics")
 		if err := s.GrpcServer.Serve(l); err != nil {
 			log.Fatalf("failed to serve: %v", err)
 		}
@@ -174,15 +213,46 @@ func main() {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
 	defer cancel()
 
+	log.Println(args)
+	if args.CmdType == server.SearchCmd {
+		r, err := c.Search(ctx, searchRequest)
+		if err != nil {
+			log.Fatal(err)
+		}
 
-	r, err := c.Search(ctx, searchRequest)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	log.Printf("found %d results\n", r.GetTotal())
+		log.Printf("found %d results\n", r.GetTotal())
 
-	for _, t := range r.Txos {
-		fmt.Printf("%s:%d\n", util.TxHashToTxId(t.TxHash), t.Nout)
+		for _, t := range r.Txos {
+			fmt.Printf("%s:%d\n", util.TxHashToTxId(t.TxHash), t.Nout)
+		}
+	} else if args.CmdType == server.GetblockCmd {
+		r, err := c.GetBlock(ctx, blockReq)
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Println(r)
+	} else if args.CmdType == server.GetblockHeaderCmd {
+		r, err := c.GetBlockHeader(ctx, blockReq)
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Println(r)
+	} else if args.CmdType == server.SubscribeHeaderCmd {
+		ctx2, cancel2 := context.WithTimeout(context.Background(), time.Hour)
+		defer cancel2()
+		header, err := c.SubscribeHeaders(ctx2, blockReq)
+		if err != nil {
+			log.Fatal(err)
+		}
+		for {
+			x, err := header.Recv()
+			log.Println(x)
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				log.Fatalln(err)
+			}
+		}
 	}
 }

protobuf/definitions/hub.proto

@@ -7,9 +7,20 @@ import "result.proto";
 package pb;
 
 service Hub {
+  rpc SubscribeHeaders (BlockRequest) returns (stream BlockHeaderOutput) {}
   rpc Search (SearchRequest) returns (Outputs) {}
+  rpc GetBlock (BlockRequest) returns (BlockOutput) {}
+  rpc GetBlockHeader (BlockRequest) returns (BlockHeaderOutput) {}
+  rpc GetServerHeight (NoParamsThisIsSilly) returns (google.protobuf.UInt64Value) {}
+  rpc GetHeaders (BlockRequest) returns (stream BlockHeaderOutput) {}
+  rpc Ping (NoParamsThisIsSilly) returns (google.protobuf.StringValue) {}
+  rpc Version (NoParamsThisIsSilly) returns (google.protobuf.StringValue) {}
+  rpc Features (NoParamsThisIsSilly) returns (google.protobuf.StringValue) {}
+  rpc Broadcast(NoParamsThisIsSilly) returns (google.protobuf.UInt64Value) {}
 }
 
+message NoParamsThisIsSilly {}
+
 message InvertibleField {
   bool invert = 1;
   repeated string value = 2;
@@ -38,7 +49,7 @@ message SearchRequest {
   string last_take_over_height = 19;
   InvertibleField claim_id = 20;
   repeated string claim_name = 22;
-  repeated string normalized = 23;
+  repeated string normalized_name = 23;
   RangeField tx_position = 24;
   RangeField amount = 25;
   RangeField timestamp = 26;
@@ -54,7 +65,7 @@ message SearchRequest {
   repeated string author = 36;
   repeated string description = 37;
   repeated string claim_type = 38;
-  RangeField reposted = 39;
+  RangeField repost_count = 39;
   repeated string stream_type = 40;
   repeated string media_type = 41;
   RangeField fee_amount = 42;
@@ -64,7 +75,7 @@ message SearchRequest {
   RangeField censor_type = 46;
   string claims_in_channel = 47;
   RangeField channel_join = 48;
-  .google.protobuf.BoolValue signature_valid = 49;
+  .google.protobuf.BoolValue is_signature_valid = 49;
   RangeField effective_amount = 51;
   RangeField support_amount = 52;
   RangeField trending_group = 53;
@@ -78,7 +89,7 @@ message SearchRequest {
   repeated string signature = 61;
   repeated string signature_digest = 62;
   repeated string public_key_bytes = 63;
-  repeated string public_key_hash = 64;
+  // repeated string public_key_hash = 64;
   string public_key_id = 65;
   repeated bytes _id = 66;
   repeated string any_tags = 67;
@@ -93,4 +104,9 @@ message SearchRequest {
   .google.protobuf.BoolValue remove_duplicates = 76;
   .google.protobuf.BoolValue no_totals = 77;
   repeated string search_indices = 78;
+}
+
+message BlockRequest {
+  string blockhash = 1;
+  bool verbose = 2;
 }

protobuf/definitions/result.proto

@@ -60,3 +60,44 @@ message Blocked {
   uint32 count = 1;
   Output channel = 2;
 }
+
+message BlockOutput {
+  string hash = 1;
+  int32 confirmations = 2;
+}
+/*
+{                                 (json object)
+  "hash" : "hex",                 (string) the block hash (same as provided)
+  "confirmations" : n,            (numeric) The number of confirmations, or -1 if the block is not on the main chain
+  "height" : n,                   (numeric) The block height or index
+  "version" : n,                  (numeric) The block version
+  "versionHex" : "hex",           (string) The block version formatted in hexadecimal
+  "merkleroot" : "hex",           (string) The merkle root
+  "time" : xxx,                   (numeric) The block time expressed in UNIX epoch time
+  "mediantime" : xxx,             (numeric) The median block time expressed in UNIX epoch time
+  "nonce" : n,                    (numeric) The nonce
+  "bits" : "hex",                 (string) The bits
+  "difficulty" : n,               (numeric) The difficulty
+  "chainwork" : "hex",            (string) Expected number of hashes required to produce the current chain
+  "nTx" : n,                      (numeric) The number of transactions in the block
+  "previousblockhash" : "hex",    (string) The hash of the previous block
+  "nextblockhash" : "hex"         (string) The hash of the next block
+}
+ */
+message BlockHeaderOutput {
+  string hash = 1;
+  int64 confirmations = 2;
+  int64 height = 3;
+  int64 version = 4;
+  string versionHex = 5;
+  string merkleroot = 6;
+  int64 time = 7;
+  int64 mediantime = 8;
+  int64 nonce = 9;
+  string bits = 10;
+  double difficulty = 11;
+  string chainwork = 12;
+  int64 nTx = 13;
+  string previousblockhash = 14;
+  string nextblockhash = 15;
+}