From c0d5fc52b7bbb2c9675cbfb4bc7ebabe4d8696a7 Mon Sep 17 00:00:00 2001 From: "Brian \"Moses\" Hall" Date: Thu, 15 Jan 2026 12:38:00 -0500 Subject: [PATCH] ETT-1251 OTIS download stats should only track when users have switched to ATRS or RS roles - Check `access_type` in addition to `role` when determining if a log entry is relevant for the database. - Update spec fixtures with some red herrings but expect the test counts to stay the same -- the red herrings are ignored. - No changes to database schema. --- app/lib/otis/log_importer.rb | 14 ++++++++++++-- spec/fixtures/ulib-logs/README.txt | 5 +++-- .../access-imgsrv_downloads.log-20250901 | 3 ++- .../access-imgsrv_downloads.log-20250902.gz | Bin 1263 -> 1315 bytes 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/app/lib/otis/log_importer.rb b/app/lib/otis/log_importer.rb index 785b7cc6..27cf971e 100644 --- a/app/lib/otis/log_importer.rb +++ b/app/lib/otis/log_importer.rb @@ -131,17 +131,27 @@ def extract_pages_from_seq(seq) private # Returns true iff all of these are true: - # role is "ssdproxy" or "resource_sharing" + # activated role is ATRS or RS # access is "success" # mode is "download" # datetime is on or after last import def relevant_log_entry?(entry) - (entry["role"] == "ssdproxy" || entry["role"] == "resource_sharing") && + (atrs?(entry) || rs?(entry)) && entry["access"] == "success" && entry["mode"] == "download" && Time.parse(entry["datetime"]) >= last_import end + # Is activated role ATRS? (role is ssdproxy and access_type is ssd_proxy_user) + def atrs?(entry) + entry["role"] == "ssdproxy" && entry["access_type"] == "ssd_proxy_user" + end + + # Is RS role activated? (role is "resource_sharing" and access_type is "resource_sharing_user") + def rs?(entry) + entry["role"] == "resource_sharing" && entry["access_type"] == "resource_sharing_user" + end + def create_report(entry) datetime = Time.parse entry["datetime"] report = HTDownload.new( diff --git a/spec/fixtures/ulib-logs/README.txt b/spec/fixtures/ulib-logs/README.txt index 5a168212..212d1242 100644 --- a/spec/fixtures/ulib-logs/README.txt +++ b/spec/fixtures/ulib-logs/README.txt @@ -1,5 +1,6 @@ For use by spec/lib/otis/log_importer_spec.rb -access-imgsrv_downloads.log-20250901 - all well-formed JSON, one qualifying ssdproxy entry +access-imgsrv_downloads.log-20250901 - all well-formed JSON, one qualifying and one non-activated ssdproxy entry access-imgsrv_downloads.log-20250902.gz - first line ill-formed JSON, one qualifying ssdproxy - entry and one qualifying resource_sharing entry + entry and one qualifying resource_sharing entry, + and one non-activated entry for each {ATRS, RS} diff --git a/spec/fixtures/ulib-logs/archive/macc-ht-web-189.umdl.umich.edu/var/log/babel/access-imgsrv_downloads.log-20250901 b/spec/fixtures/ulib-logs/archive/macc-ht-web-189.umdl.umich.edu/var/log/babel/access-imgsrv_downloads.log-20250901 index 76b65165..d4ff8890 100644 --- a/spec/fixtures/ulib-logs/archive/macc-ht-web-189.umdl.umich.edu/var/log/babel/access-imgsrv_downloads.log-20250901 +++ b/spec/fixtures/ulib-logs/archive/macc-ht-web-189.umdl.umich.edu/var/log/babel/access-imgsrv_downloads.log-20250901 @@ -1,4 +1,5 @@ -{"datetime":"2025-09-01 15:27:23","session":"xxx","app":"imgsrv","id":"mdp.39015033899868","attr":1,"ic":0,"access":"success","access_type":"ht_affiliate","access_type_by_attr":"allow","digitization_source":"google","collection_source":"umich","remote_addr":"192.0.2.0","proxied_addr":null,"remote_user_processed":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","remote_user_from_env":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","affiliation":"employee@default.invalid;member@default.invalid;staff@default.invalid","eppn":"example@default.invalid","display_name":"Example, Bob","entityID":"https://login.default.invalid/cas/idp/metadata","persistent_id":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","usertype":"external","role":"ssdproxy","sdrinst":"example","auth_type":"shibboleth","http_host":"babel.hathitrust.org","server_addr":"141.213.128.138","sdrlib":null,"http_referer":"https://babel.hathitrust.org/cgi/pt?id=mdp.39015033899868&seq=456","request_uri":"/cgi/imgsrv/download/pdf?id=mdp.39015033899868;seq=415-456;marker=xxx;attachment=1","user_agent":"sample user agent","inst_code":"default","inst_code_mapped":"default","geo_code":"US","geo_code_proxy":null,"mode":"download","is_partial":1,"seq":"415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456","content_length":19144826} +{"datetime":"2025-09-01 15:27:23","session":"xxx","app":"imgsrv","id":"mdp.39015033899868","attr":1,"ic":0,"access":"success","access_type":"ssd_proxy_user","access_type_by_attr":"allow","digitization_source":"google","collection_source":"umich","remote_addr":"192.0.2.0","proxied_addr":null,"remote_user_processed":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","remote_user_from_env":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","affiliation":"employee@default.invalid;member@default.invalid;staff@default.invalid","eppn":"example@default.invalid","display_name":"Example, Bob","entityID":"https://login.default.invalid/cas/idp/metadata","persistent_id":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","usertype":"external","role":"ssdproxy","sdrinst":"example","auth_type":"shibboleth","http_host":"babel.hathitrust.org","server_addr":"141.213.128.138","sdrlib":null,"http_referer":"https://babel.hathitrust.org/cgi/pt?id=mdp.39015033899868&seq=456","request_uri":"/cgi/imgsrv/download/pdf?id=mdp.39015033899868;seq=415-456;marker=xxx;attachment=1","user_agent":"sample user agent","inst_code":"default","inst_code_mapped":"default","geo_code":"US","geo_code_proxy":null,"mode":"download","is_partial":1,"seq":"415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456","content_length":19144826} +{"datetime":"2025-09-01 15:28:00","session":"xxx","app":"imgsrv","id":"mdp.30000080041746","attr":9,"ic":0,"access":"success","access_type":"ht_affiliate","access_type_by_attr":"allow","digitization_source":"google","collection_source":"umich","remote_addr":"192.0.2.0","proxied_addr":null,"remote_user_processed":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","remote_user_from_env":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","affiliation":"employee@default.invalid;member@default.invalid;staff@default.invalid","eppn":"example@default.invalid","display_name":"Example, Bob","entityID":"https://login.default.invalid/cas/idp/metadata","persistent_id":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","usertype":"external","role":"ssdproxy","sdrinst":"example","auth_type":"shibboleth","http_host":"babel.hathitrust.org","server_addr":"141.213.128.138","sdrlib":null,"http_referer":"https://babel.hathitrust.org/cgi/pt?id=mdp.39015033899868&seq=456","request_uri":"/cgi/imgsrv/download/pdf?id=mdp.39015033899868;seq=415-456;marker=xxx;attachment=1","user_agent":"sample user agent","inst_code":"default","inst_code_mapped":"default","geo_code":"US","geo_code_proxy":null,"mode":"download","is_partial":1,"seq":"415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456","content_length":19144826} {"datetime":"2025-09-01 15:28:52","session":"yyy","app":"imgsrv","id":"inu.30000080041746","attr":9,"ic":0,"access":"success","access_type":"ordinary_user","access_type_by_attr":"allow_by_us_geo_ipaddr","digitization_source":"google","collection_source":"iu","remote_addr":"2001:db8::/32","proxied_addr":null,"remote_user_processed":null,"remote_user_from_env":null,"affiliation":null,"eppn":null,"display_name":null,"entityID":"notset","persistent_id":"notset","usertype":null,"role":null,"sdrinst":null,"auth_type":null,"http_host":"babel.hathitrust.org","server_addr":"141.213.128.138","sdrlib":null,"http_referer":"https://babel.hathitrust.org/cgi/pt?id=inu.30000080041746&seq=7&q1=Mujaes","request_uri":"/cgi/imgsrv/download/pdf?id=inu.30000080041746&attachment=1&tracker=D1&seq=7","user_agent":"a user agent","inst_code":null,"inst_code_mapped":null,"geo_code":"US","geo_code_proxy":null,"mode":"streaming","is_partial":1,"seq":"7","content_length":273894} {"datetime":"2025-09-01 15:29:03","session":"yyy","app":"imgsrv","id":"inu.30000080041746","attr":9,"ic":0,"access":"success","access_type":"ordinary_user","access_type_by_attr":"allow_by_us_geo_ipaddr","digitization_source":"google","collection_source":"iu","remote_addr":"2001:db8::/32","proxied_addr":null,"remote_user_processed":null,"remote_user_from_env":null,"affiliation":null,"eppn":null,"display_name":null,"entityID":"notset","persistent_id":"notset","usertype":null,"role":null,"sdrinst":null,"auth_type":null,"http_host":"babel.hathitrust.org","server_addr":"141.213.128.138","sdrlib":null,"http_referer":"https://babel.hathitrust.org/cgi/pt?id=inu.30000080041746&seq=8&q1=Mujaes","request_uri":"/cgi/imgsrv/download/pdf?id=inu.30000080041746&attachment=1&tracker=D2&seq=8","user_agent":"a user agent","inst_code":null,"inst_code_mapped":null,"geo_code":"US","geo_code_proxy":null,"mode":"streaming","is_partial":1,"seq":"8","content_length":93412} {"datetime":"2025-09-01 15:29:11","session":"yyy","app":"imgsrv","id":"inu.30000080041746","attr":9,"ic":0,"access":"success","access_type":"ordinary_user","access_type_by_attr":"allow_by_us_geo_ipaddr","digitization_source":"google","collection_source":"iu","remote_addr":"2001:db8::/32","proxied_addr":null,"remote_user_processed":null,"remote_user_from_env":null,"affiliation":null,"eppn":null,"display_name":null,"entityID":"notset","persistent_id":"notset","usertype":null,"role":null,"sdrinst":null,"auth_type":null,"http_host":"babel.hathitrust.org","server_addr":"141.213.128.138","sdrlib":null,"http_referer":"https://babel.hathitrust.org/cgi/pt?id=inu.30000080041746&seq=9&q1=Mujaes","request_uri":"/cgi/imgsrv/download/pdf?id=inu.30000080041746&attachment=1&tracker=D3&seq=9","user_agent":"a user agent","inst_code":null,"inst_code_mapped":null,"geo_code":"US","geo_code_proxy":null,"mode":"streaming","is_partial":1,"seq":"9","content_length":160982} diff --git a/spec/fixtures/ulib-logs/archive/macc-ht-web-189.umdl.umich.edu/var/log/babel/access-imgsrv_downloads.log-20250902.gz b/spec/fixtures/ulib-logs/archive/macc-ht-web-189.umdl.umich.edu/var/log/babel/access-imgsrv_downloads.log-20250902.gz index d37714e1ac4919bf845efe13f86655f11a8b83f5..7aa25a0779177fb86f92653d04b5fd664600a4d4 100644 GIT binary patch literal 1315 zcmV+;1>E`{iwFP!000006YW}EbE7sCeebVe{g9``)&~$Elj+y)L)&RP(@dWk8Ofj) z5G#o!&QAY(uK>q!>`mgNNxdyQs~M0ESK{7#&c%1~$+Lwt9{gt>7p0sF&8#SwUd76V z(v{#|DQn@Wh4hr5b5QyL>2AWEY+j>52zD#>_6IPb$3RbzyrIk0#!r4-zTAeu)xl(!Q9?n!*ifnVStz=ff zniiF^B4wOAgE%HrG=%{xjaJuE@L{QL%kmc88lh8IcV`m3JFRU@GMkkum-Up31#3$? zmGvbnC7)%?%p`ASm9Pw4%-%UTps%j3rUipAS=|~tRXU%ULeA%^6t*}q&Aa7tIoaHF zp;eWN`tr5+T=~8awy2s?Ers~z-Y#dAs35Plrj3Q?@6Uo&(KMa8YX%0ltaX7jP05yN z&AKG~&;yUX@6_BqUR!CGKfK@jfm!W7v;T^boA7hkR+>Q_VJWpI>jWKVjA`P zfe@^cxihH6LTI7ycx0FDS(eLLW51UCe4E}!MtnT?gRo20kF78^ZM6i8-BZ0d%(%L$ zOU3xC;fvj7&N?$VH~~{<71MtTeGcwC1Fo~Is33CZc<^n?a=3L4>b&N;Ij^4sg#nMI znc~iYLtO99rWJ@*N80sbF4XFY-!Jamr+uIXe^rCc;Z2-TlQv9S3E_3x1}^~X@CwI1 z4q=?ah+xFWK0z=r3<86~@L>Rs0FD5T0FD5T0*(TX0*(TX0*(TX0*(TX0*(UC2b>Q$ zA8{8$9wgyor6362 zv=jhg`apCNI2MDk7>vbWECypSID}#lB>~we27_YkC+pWq#5W)$cM@cfHWWVaLy|`BKer-c-`>9?k`|C|r z9M;!bSSxFU-Jx-}3u`?Z&eADSe|@V#!%^1qb4Mlqec9PgrA~HEk3Qn_pW8o}Fnh_* z4s&;P=E!Q6IRSc)`)%$E5Bsd<^e?$BH~suxN@J`Rtde#9IjK3_L1J!Xa|L3>SOX4gE1Z%JTmwo7#zPggK59bVKO-0^(7smIC`AI z!4oOG{;>p(ir|Mr{G6uB9u)p3dnzMo59J5@DtS6Kh3J{T*Ke&n^ugR&-y|#ev;elg zn6lr0`(ny=r@PNtdP*V`Jjhv!5{x!DdvkO1cq0@AD8>{$r|fFcvnw?PgKW#6Q+`vW z(=9syDpU!AkelwrDLASSr8_zP4|i1ic)e{$b;yx}=&1g7;eEJmHGTp8)ZTXw_EYbt z_8a$A=^dHwsPp2s>L3@Lj^gJwRR<%hMk@IUhf|EXFMLL$c@Tz-VJv7Ukk7M#^3V?v zCyR*2a|Yio@q%N-ctq)3#8DV7VxN*I{!*H;Xh&a!`Vk@|dQ9NS{sO=CQMQM`@AXFy ZqfC!7J<9Yb)Bg*Z{u}7!ecH|@008hlfFS?? literal 1263 zcmV&vZ_FRXo>DVM?-(~Y$=Tg|IFi}l5?S%71h$KS+!8Q z7Tl|3BRsY6emVdBH}BJ%j}s0ED{C=HCj^o31SKbkcsNYSEG2YuJTbx;sT#0fUthyL zYg@RMbz$@++)562HE(^IARHn}<0Ofr*a5B8lN19apQH%x^Bi0O+H}L!YAUnK)~#RK zj4c+jlCY&+V>VxAgS!cYp4YRPf%&1Q1&PAbAQMre6RaGM2HFmYkOR#BC zD=RX_xpRmU;v*jhSX!;FrQoBf>8fgt?u^hG*xjB4?{{lklTN3VDrDnxv0zH zO7dyW%vADrS_{h{#O$qu1N!Rf$}btD$?DEnU+H3MN;#jaO4#zmv~QQo2a~1m`*`jVMwG`s}TdSPbqK3NKn>H4nzdZ|HMcel7t{FJo^VTQQv=v)s4eN{W z{Qx}nepGY!cw?nq{`~I2FYmG`w2{Vwi%dRa#s*rrRE>2kuC34wt6UxdH<=EA9d~ z=5>2Et3kAS(r%1}P^%~YJm0#{hD43=sz;yWn>eQ?YniqZ((AMhUI6Uy3daGCV9a2| zFp}edAQ%`1fk9yeFaSpYM*v3vM*v3wM*&9xM*&9xM*&9xM*&9xM*$ZAE&yBrxBzeg z-~zw}fC~T@04@Mr2)Gb%A>cy5g@6kI*JmqNO;3nQGzFAciW3YElIZTEAWlQFa}Kkja%8s z&z4I3_p-BJO1Pz zN@J`Rtd>pjIjNZ)ATgPNo(6YQI7tzGrSL0-Uj&8Y=cX_jmU)>Jjt@{cp#diMF&N`l z2EQ`+MKCydZU)m~nU~4nys(_(+^LkY`?qPoTW1oqwr47Qj}t}!`Yjg zoBP|KC`1XS=rPBx76ZF7Q!=Qw;xWf>rgXOF2SBB&KoAPk?>K`%m7;Px$N%Gw>X5Im z+ff~BJ$HrhDo