Skip to content
This repository was archived by the owner on Mar 4, 2025. It is now read-only.

Commit 2364188

Browse files
justincliftjustinclift
committed
api, common. Log all API calls into our database, for stats generation
With this commit, we record basic info for each API call so we can start doing useful statistic generation for API usage. The backend database needs a new table created to hold the info: CREATE TABLE public.api_call_log ( api_call_id bigint NOT NULL, api_call_date timestamp with time zone DEFAULT now(), caller_id bigint, db_owner_id bigint, db_id bigint, api_operation text NOT NULL, api_caller_sw text ); COMMENT ON COLUMN public.api_call_log.db_owner_id IS 'This field must be nullable, as not all api calls act on a database'; COMMENT ON COLUMN public.api_call_log.db_id IS 'This field must be nullable, as not all api calls act on a database'; CREATE SEQUENCE public.api_log_log_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; ALTER SEQUENCE public.api_log_log_id_seq OWNED BY public.api_call_log.api_call_id; ALTER TABLE ONLY public.api_call_log ALTER COLUMN api_call_id SET DEFAULT nextval('public.api_log_log_id_seq'::regclass); ALTER TABLE ONLY public.api_call_log ADD CONSTRAINT api_log_users_user_id_fk FOREIGN KEY (caller_id) REFERENCES public.users(user_id);
1 parent 5c78e52 commit 2364188

File tree

4 files changed

+183
-36
lines changed

4 files changed

+183
-36
lines changed

Diff for: ‎api/handlers.go

+58-6
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,15 @@ import (
2727
// * "dbname" is the name of the database
2828
func branchesHandler(w http.ResponseWriter, r *http.Request) {
2929
// Do auth check, grab request info
30-
_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
30+
loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
3131
if err != nil {
3232
jsonErr(w, err.Error(), httpStatus)
3333
return
3434
}
3535

36+
// Record the api call in our backend database
37+
com.ApiCallLog(loggedInUser, dbOwner, dbName, "branches", r.Header.Get("User-Agent"))
38+
3639
// If the database is a live database, we return an error message
3740
isLive, _, err := com.CheckDBLive(dbOwner, dbName)
3841
if err != nil {
@@ -102,6 +105,9 @@ func columnsHandler(w http.ResponseWriter, r *http.Request) {
102105
return
103106
}
104107

108+
// Record the api call in our backend database
109+
com.ApiCallLog(loggedInUser, dbOwner, dbName, "columns", r.Header.Get("User-Agent"))
110+
105111
// Extract the table name
106112
table, err := com.GetFormTable(r, false)
107113
if err != nil {
@@ -221,12 +227,15 @@ func columnsHandler(w http.ResponseWriter, r *http.Request) {
221227
// * "dbname" is the name of the database
222228
func commitsHandler(w http.ResponseWriter, r *http.Request) {
223229
// Do auth check, grab request info
224-
_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
230+
loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
225231
if err != nil {
226232
jsonErr(w, err.Error(), httpStatus)
227233
return
228234
}
229235

236+
// Record the api call in our backend database
237+
com.ApiCallLog(loggedInUser, dbOwner, dbName, "commits", r.Header.Get("User-Agent"))
238+
230239
// If the database is a live database, we return an error message
231240
isLive, _, err := com.CheckDBLive(dbOwner, dbName)
232241
if err != nil {
@@ -271,6 +280,9 @@ func databasesHandler(w http.ResponseWriter, r *http.Request) {
271280
return
272281
}
273282

283+
// Record the api call in our backend database
284+
com.ApiCallLog(loggedInUser, "", "", "databases", r.Header.Get("User-Agent"))
285+
274286
// Get "live" boolean value, if provided by the caller
275287
var live bool
276288
live, err = com.GetFormLive(r)
@@ -336,6 +348,9 @@ func deleteHandler(w http.ResponseWriter, r *http.Request) {
336348
}
337349
dbOwner := loggedInUser
338350

351+
// Record the api call in our backend database
352+
com.ApiCallLog(loggedInUser, dbOwner, dbName, "delete", r.Header.Get("User-Agent"))
353+
339354
// Check if the database exists
340355
exists, err := com.CheckDBPermissions(loggedInUser, dbOwner, dbName, false)
341356
if err != nil {
@@ -517,6 +532,10 @@ func diffHandler(w http.ResponseWriter, r *http.Request) {
517532
return
518533
}
519534

535+
// Record the api call in our backend database
536+
// Note - Lets not bother adding additional api logging fields just for the diff function at this stage
537+
com.ApiCallLog(loggedInUser, dbOwnerA, dbNameA, "diff", r.Header.Get("User-Agent"))
538+
520539
// Check permissions of the first database
521540
var allowed bool
522541
allowed, err = com.CheckDBPermissions(loggedInUser, dbOwnerA, dbNameA, false)
@@ -593,6 +612,9 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) {
593612
return
594613
}
595614

615+
// Record the api call in our backend database
616+
com.ApiCallLog(loggedInUser, dbOwner, dbName, "download", r.Header.Get("User-Agent"))
617+
596618
// Return the requested database to the user
597619
_, err = com.DownloadDatabase(w, r, dbOwner, dbName, commitID, loggedInUser, "api")
598620
if err != nil {
@@ -630,6 +652,9 @@ func executeHandler(w http.ResponseWriter, r *http.Request) {
630652
return
631653
}
632654

655+
// Record the api call in our backend database
656+
com.ApiCallLog(loggedInUser, dbOwner, dbName, "execute", r.Header.Get("User-Agent"))
657+
633658
// Grab the incoming SQLite query
634659
rawInput := r.FormValue("sql")
635660
var sql string
@@ -709,6 +734,9 @@ func indexesHandler(w http.ResponseWriter, r *http.Request) {
709734
return
710735
}
711736

737+
// Record the api call in our backend database
738+
com.ApiCallLog(loggedInUser, dbOwner, dbName, "indexes", r.Header.Get("User-Agent"))
739+
712740
// Check if the database is a live database, and get the node/queue to send the request to
713741
isLive, liveNode, err := com.CheckDBLive(dbOwner, dbName)
714742
if err != nil {
@@ -826,12 +854,15 @@ func indexesHandler(w http.ResponseWriter, r *http.Request) {
826854
// * "dbname" is the name of the database
827855
func metadataHandler(w http.ResponseWriter, r *http.Request) {
828856
// Do auth check, grab request info
829-
_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
857+
loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
830858
if err != nil {
831859
jsonErr(w, err.Error(), httpStatus)
832860
return
833861
}
834862

863+
// Record the api call in our backend database
864+
com.ApiCallLog(loggedInUser, dbOwner, dbName, "metadata", r.Header.Get("User-Agent"))
865+
835866
// If the database is a live database, we return an error message
836867
isLive, _, err := com.CheckDBLive(dbOwner, dbName)
837868
if err != nil {
@@ -886,6 +917,9 @@ func queryHandler(w http.ResponseWriter, r *http.Request) {
886917
return
887918
}
888919

920+
// Record the api call in our backend database
921+
com.ApiCallLog(loggedInUser, dbOwner, dbName, "query", r.Header.Get("User-Agent"))
922+
889923
// Grab the incoming SQLite query
890924
rawInput := r.FormValue("sql")
891925
query, err := com.CheckUnicode(rawInput)
@@ -958,12 +992,15 @@ func queryHandler(w http.ResponseWriter, r *http.Request) {
958992
// * "dbname" is the name of the database
959993
func releasesHandler(w http.ResponseWriter, r *http.Request) {
960994
// Do auth check, grab request info
961-
_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
995+
loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
962996
if err != nil {
963997
jsonErr(w, err.Error(), httpStatus)
964998
return
965999
}
9661000

1001+
// Record the api call in our backend database
1002+
com.ApiCallLog(loggedInUser, dbOwner, dbName, "releases", r.Header.Get("User-Agent"))
1003+
9671004
// If the database is a live database, we return an error message
9681005
isLive, _, err := com.CheckDBLive(dbOwner, dbName)
9691006
if err != nil {
@@ -1033,6 +1070,9 @@ func tablesHandler(w http.ResponseWriter, r *http.Request) {
10331070
return
10341071
}
10351072

1073+
// Record the api call in our backend database
1074+
com.ApiCallLog(loggedInUser, dbOwner, dbName, "tables", r.Header.Get("User-Agent"))
1075+
10361076
// Check if the database is a live database, and get the node/queue to send the request to
10371077
isLive, liveNode, err := com.CheckDBLive(dbOwner, dbName)
10381078
if err != nil {
@@ -1108,12 +1148,15 @@ func tablesHandler(w http.ResponseWriter, r *http.Request) {
11081148
// * "dbname" is the name of the database
11091149
func tagsHandler(w http.ResponseWriter, r *http.Request) {
11101150
// Do auth check, grab request info
1111-
_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
1151+
loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
11121152
if err != nil {
11131153
jsonErr(w, err.Error(), httpStatus)
11141154
return
11151155
}
11161156

1157+
// Record the api call in our backend database
1158+
com.ApiCallLog(loggedInUser, dbOwner, dbName, "tags", r.Header.Get("User-Agent"))
1159+
11171160
// If the database is a live database, we return an error message
11181161
isLive, _, err := com.CheckDBLive(dbOwner, dbName)
11191162
if err != nil {
@@ -1263,6 +1306,9 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
12631306
return
12641307
}
12651308

1309+
// Record the api call in our backend database
1310+
com.ApiCallLog(loggedInUser, loggedInUser, dbName, "upload", r.Header.Get("User-Agent"))
1311+
12661312
// Check if the database exists already
12671313
exists, err := com.CheckDBExists(loggedInUser, dbName)
12681314
if err != nil {
@@ -1366,6 +1412,9 @@ func viewsHandler(w http.ResponseWriter, r *http.Request) {
13661412
return
13671413
}
13681414

1415+
// Record the api call in our backend database
1416+
com.ApiCallLog(loggedInUser, loggedInUser, dbName, "views", r.Header.Get("User-Agent"))
1417+
13691418
// Check if the database is a live database, and get the node/queue to send the request to
13701419
isLive, liveNode, err := com.CheckDBLive(dbOwner, dbName)
13711420
if err != nil {
@@ -1441,12 +1490,15 @@ func viewsHandler(w http.ResponseWriter, r *http.Request) {
14411490
// * "dbname" is the name of the database being queried
14421491
func webpageHandler(w http.ResponseWriter, r *http.Request) {
14431492
// Authenticate user and collect requested database details
1444-
_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
1493+
loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
14451494
if err != nil {
14461495
jsonErr(w, err.Error(), httpStatus)
14471496
return
14481497
}
14491498

1499+
// Record the api call in our backend database
1500+
com.ApiCallLog(loggedInUser, dbOwner, dbName, "views", r.Header.Get("User-Agent"))
1501+
14501502
// Return the database webUI URL to the user
14511503
var z com.WebpageResponseContainer
14521504
z.WebPage = "https://" + com.Conf.Web.ServerName + "/" + dbOwner + "/" + dbName

Diff for: ‎api/main.go

+24-30
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,6 @@ package main
33
// TODO: API functions that still need updating for Live databases
44
// * diff - already updated to just return an error for live databases. needs testing though
55

6-
// FIXME: Update the documented Upload() function return values on the API doc page. Currently it talks about
7-
// returning the commit ID for the upload. We'll probably return that field with a blank value for live
8-
// databases though. TBD.
9-
106
// FIXME: After the API and webui pieces are done, figure out how the DB4S end
117
// point and dio should be updated to use live databases too
128

@@ -113,29 +109,29 @@ func main() {
113109
}
114110

115111
// Our pages
116-
http.Handle("/", gz.GzipHandler(handleWrapper(rootHandler)))
117-
http.Handle("/changelog", gz.GzipHandler(handleWrapper(changeLogHandler)))
118-
http.Handle("/changelog.html", gz.GzipHandler(handleWrapper(changeLogHandler)))
119-
http.Handle("/v1/branches", gz.GzipHandler(handleWrapper(branchesHandler)))
120-
http.Handle("/v1/columns", gz.GzipHandler(handleWrapper(columnsHandler)))
121-
http.Handle("/v1/commits", gz.GzipHandler(handleWrapper(commitsHandler)))
122-
http.Handle("/v1/databases", gz.GzipHandler(handleWrapper(databasesHandler)))
123-
http.Handle("/v1/delete", gz.GzipHandler(handleWrapper(deleteHandler)))
124-
http.Handle("/v1/diff", gz.GzipHandler(handleWrapper(diffHandler)))
125-
http.Handle("/v1/download", gz.GzipHandler(handleWrapper(downloadHandler)))
126-
http.Handle("/v1/execute", gz.GzipHandler(handleWrapper(executeHandler)))
127-
http.Handle("/v1/indexes", gz.GzipHandler(handleWrapper(indexesHandler)))
128-
http.Handle("/v1/metadata", gz.GzipHandler(handleWrapper(metadataHandler)))
129-
http.Handle("/v1/query", gz.GzipHandler(handleWrapper(queryHandler)))
130-
http.Handle("/v1/releases", gz.GzipHandler(handleWrapper(releasesHandler)))
131-
http.Handle("/v1/tables", gz.GzipHandler(handleWrapper(tablesHandler)))
132-
http.Handle("/v1/tags", gz.GzipHandler(handleWrapper(tagsHandler)))
133-
http.Handle("/v1/upload", gz.GzipHandler(handleWrapper(uploadHandler)))
134-
http.Handle("/v1/views", gz.GzipHandler(handleWrapper(viewsHandler)))
135-
http.Handle("/v1/webpage", gz.GzipHandler(handleWrapper(webpageHandler)))
112+
http.Handle("/", gz.GzipHandler(corsWrapper(rootHandler)))
113+
http.Handle("/changelog", gz.GzipHandler(corsWrapper(changeLogHandler)))
114+
http.Handle("/changelog.html", gz.GzipHandler(corsWrapper(changeLogHandler)))
115+
http.Handle("/v1/branches", gz.GzipHandler(corsWrapper(branchesHandler)))
116+
http.Handle("/v1/columns", gz.GzipHandler(corsWrapper(columnsHandler)))
117+
http.Handle("/v1/commits", gz.GzipHandler(corsWrapper(commitsHandler)))
118+
http.Handle("/v1/databases", gz.GzipHandler(corsWrapper(databasesHandler)))
119+
http.Handle("/v1/delete", gz.GzipHandler(corsWrapper(deleteHandler)))
120+
http.Handle("/v1/diff", gz.GzipHandler(corsWrapper(diffHandler)))
121+
http.Handle("/v1/download", gz.GzipHandler(corsWrapper(downloadHandler)))
122+
http.Handle("/v1/execute", gz.GzipHandler(corsWrapper(executeHandler)))
123+
http.Handle("/v1/indexes", gz.GzipHandler(corsWrapper(indexesHandler)))
124+
http.Handle("/v1/metadata", gz.GzipHandler(corsWrapper(metadataHandler)))
125+
http.Handle("/v1/query", gz.GzipHandler(corsWrapper(queryHandler)))
126+
http.Handle("/v1/releases", gz.GzipHandler(corsWrapper(releasesHandler)))
127+
http.Handle("/v1/tables", gz.GzipHandler(corsWrapper(tablesHandler)))
128+
http.Handle("/v1/tags", gz.GzipHandler(corsWrapper(tagsHandler)))
129+
http.Handle("/v1/upload", gz.GzipHandler(corsWrapper(uploadHandler)))
130+
http.Handle("/v1/views", gz.GzipHandler(corsWrapper(viewsHandler)))
131+
http.Handle("/v1/webpage", gz.GzipHandler(corsWrapper(webpageHandler)))
136132

137133
// favicon.ico
138-
http.Handle("/favicon.ico", gz.GzipHandler(handleWrapper(func(w http.ResponseWriter, r *http.Request) {
134+
http.Handle("/favicon.ico", gz.GzipHandler(corsWrapper(func(w http.ResponseWriter, r *http.Request) {
139135
logReq(r, "-")
140136
http.ServeFile(w, r, filepath.Join(com.Conf.Web.BaseDir, "webui", "favicon.ico"))
141137
})))
@@ -329,9 +325,8 @@ func extractUserFromClientCert(w http.ResponseWriter, r *http.Request) (userAcc
329325
return
330326
}
331327

332-
// handleWrapper does nothing useful except interface between types
333-
// TODO: Get rid of this, as it shouldn't be needed
334-
func handleWrapper(fn http.HandlerFunc) http.HandlerFunc {
328+
// corsWrapper sets a general allow for all our api calls
329+
func corsWrapper(fn http.HandlerFunc) http.HandlerFunc {
335330
return func(w http.ResponseWriter, r *http.Request) {
336331
// Enable CORS (https://enable-cors.org)
337332
w.Header().Set("Access-Control-Allow-Origin", "*")
@@ -361,6 +356,5 @@ func jsonErr(w http.ResponseWriter, msg string, statusCode int) {
361356
// logReq writes an entry for the incoming request to the request log
362357
func logReq(r *http.Request, loggedInUser string) {
363358
fmt.Fprintf(reqLog, "%v - %s [%s] \"%s %s %s\" \"-\" \"-\" \"%s\" \"%s\"\n", r.RemoteAddr,
364-
loggedInUser, time.Now().Format(time.RFC3339Nano), r.Method, r.URL, r.Proto,
365-
r.Referer(), r.Header.Get("User-Agent"))
359+
loggedInUser, time.Now().Format(time.RFC3339Nano), r.Method, r.URL, r.Proto, r.Referer(), r.Header.Get("User-Agent"))
366360
}

Diff for: ‎common/postgresql.go

+45
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,51 @@ func AddUser(auth0ID, userName, password, email, displayName, avatarURL string)
9696
return nil
9797
}
9898

99+
// ApiCallLog records an API call operation. Database name is optional, as not all API calls operate on a
100+
// database. If a database name is provided however, then the database owner name *must* also be provided
101+
func ApiCallLog(loggedInUser, dbOwner, dbName, operation, callerSw string) {
102+
var dbQuery string
103+
var err error
104+
var commandTag pgx.CommandTag
105+
if dbName != "" {
106+
dbQuery = `
107+
WITH loggedIn AS (
108+
SELECT user_id
109+
FROM users
110+
WHERE lower(user_name) = lower($1)
111+
),
112+
WITH owner AS (
113+
SELECT user_id
114+
FROM users
115+
WHERE lower(user_name) = lower($2)
116+
), d AS (
117+
SELECT db.db_id
118+
FROM sqlite_databases AS db, owner
119+
WHERE db.user_id = owner.user_id
120+
AND db.db_name = $3)
121+
INSERT INTO api_call_log (caller_id, db_owner_id, db_id, api_operation, api_caller_sw)
122+
VALUES ((SELECT user_id FROM loggedIn), (SELECT user_id FROM owner), (SELECT db_id FROM d), $4, $5)`
123+
commandTag, err = pdb.Exec(dbQuery, loggedInUser, dbOwner, dbName, operation, callerSw)
124+
} else {
125+
dbQuery = `
126+
WITH loggedIn AS (
127+
SELECT user_id
128+
FROM users
129+
WHERE lower(user_name) = lower($1)
130+
)
131+
INSERT INTO api_call_log (caller_id, api_operation, api_caller_sw)
132+
VALUES ((SELECT user_id FROM loggedIn), $2, $3)`
133+
commandTag, err = pdb.Exec(dbQuery, loggedInUser, operation, callerSw)
134+
}
135+
if err != nil {
136+
log.Printf("Adding api call log entry failed: %s", err)
137+
return
138+
}
139+
if numRows := commandTag.RowsAffected(); numRows != 1 {
140+
log.Printf("Wrong number of rows (%d) affected when adding api call entry for user '%s'", numRows, SanitiseLogString(loggedInUser))
141+
}
142+
}
143+
99144
// APIKeySave saves a new API key to the PostgreSQL database
100145
func APIKeySave(key, loggedInUser string, dateCreated time.Time) error {
101146
// Make sure the API key isn't already in the database

0 commit comments

Comments
 (0)