diff --git a/app/lib/otis/log_importer.rb b/app/lib/otis/log_importer.rb index 785b7cc6..27cf971e 100644 --- a/app/lib/otis/log_importer.rb +++ b/app/lib/otis/log_importer.rb @@ -131,17 +131,27 @@ def extract_pages_from_seq(seq) private # Returns true iff all of these are true: - # role is "ssdproxy" or "resource_sharing" + # activated role is ATRS or RS # access is "success" # mode is "download" # datetime is on or after last import def relevant_log_entry?(entry) - (entry["role"] == "ssdproxy" || entry["role"] == "resource_sharing") && + (atrs?(entry) || rs?(entry)) && entry["access"] == "success" && entry["mode"] == "download" && Time.parse(entry["datetime"]) >= last_import end + # Is activated role ATRS? (role is ssdproxy and access_type is ssd_proxy_user) + def atrs?(entry) + entry["role"] == "ssdproxy" && entry["access_type"] == "ssd_proxy_user" + end + + # Is RS role activated? (role is "resource_sharing" and access_type is "resource_sharing_user") + def rs?(entry) + entry["role"] == "resource_sharing" && entry["access_type"] == "resource_sharing_user" + end + def create_report(entry) datetime = Time.parse entry["datetime"] report = HTDownload.new( diff --git a/spec/fixtures/ulib-logs/README.txt b/spec/fixtures/ulib-logs/README.txt index 5a168212..212d1242 100644 --- a/spec/fixtures/ulib-logs/README.txt +++ b/spec/fixtures/ulib-logs/README.txt @@ -1,5 +1,6 @@ For use by spec/lib/otis/log_importer_spec.rb -access-imgsrv_downloads.log-20250901 - all well-formed JSON, one qualifying ssdproxy entry +access-imgsrv_downloads.log-20250901 - all well-formed JSON, one qualifying and one non-activated ssdproxy entry access-imgsrv_downloads.log-20250902.gz - first line ill-formed JSON, one qualifying ssdproxy - entry and one qualifying resource_sharing entry + entry and one qualifying resource_sharing entry, + and one non-activated entry for each {ATRS, RS} diff --git a/spec/fixtures/ulib-logs/archive/macc-ht-web-189.umdl.umich.edu/var/log/babel/access-imgsrv_downloads.log-20250901 b/spec/fixtures/ulib-logs/archive/macc-ht-web-189.umdl.umich.edu/var/log/babel/access-imgsrv_downloads.log-20250901 index 76b65165..d4ff8890 100644 --- a/spec/fixtures/ulib-logs/archive/macc-ht-web-189.umdl.umich.edu/var/log/babel/access-imgsrv_downloads.log-20250901 +++ b/spec/fixtures/ulib-logs/archive/macc-ht-web-189.umdl.umich.edu/var/log/babel/access-imgsrv_downloads.log-20250901 @@ -1,4 +1,5 @@ -{"datetime":"2025-09-01 15:27:23","session":"xxx","app":"imgsrv","id":"mdp.39015033899868","attr":1,"ic":0,"access":"success","access_type":"ht_affiliate","access_type_by_attr":"allow","digitization_source":"google","collection_source":"umich","remote_addr":"192.0.2.0","proxied_addr":null,"remote_user_processed":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","remote_user_from_env":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","affiliation":"employee@default.invalid;member@default.invalid;staff@default.invalid","eppn":"example@default.invalid","display_name":"Example, Bob","entityID":"https://login.default.invalid/cas/idp/metadata","persistent_id":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","usertype":"external","role":"ssdproxy","sdrinst":"example","auth_type":"shibboleth","http_host":"babel.hathitrust.org","server_addr":"141.213.128.138","sdrlib":null,"http_referer":"https://babel.hathitrust.org/cgi/pt?id=mdp.39015033899868&seq=456","request_uri":"/cgi/imgsrv/download/pdf?id=mdp.39015033899868;seq=415-456;marker=xxx;attachment=1","user_agent":"sample user agent","inst_code":"default","inst_code_mapped":"default","geo_code":"US","geo_code_proxy":null,"mode":"download","is_partial":1,"seq":"415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456","content_length":19144826} +{"datetime":"2025-09-01 15:27:23","session":"xxx","app":"imgsrv","id":"mdp.39015033899868","attr":1,"ic":0,"access":"success","access_type":"ssd_proxy_user","access_type_by_attr":"allow","digitization_source":"google","collection_source":"umich","remote_addr":"192.0.2.0","proxied_addr":null,"remote_user_processed":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","remote_user_from_env":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","affiliation":"employee@default.invalid;member@default.invalid;staff@default.invalid","eppn":"example@default.invalid","display_name":"Example, Bob","entityID":"https://login.default.invalid/cas/idp/metadata","persistent_id":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","usertype":"external","role":"ssdproxy","sdrinst":"example","auth_type":"shibboleth","http_host":"babel.hathitrust.org","server_addr":"141.213.128.138","sdrlib":null,"http_referer":"https://babel.hathitrust.org/cgi/pt?id=mdp.39015033899868&seq=456","request_uri":"/cgi/imgsrv/download/pdf?id=mdp.39015033899868;seq=415-456;marker=xxx;attachment=1","user_agent":"sample user agent","inst_code":"default","inst_code_mapped":"default","geo_code":"US","geo_code_proxy":null,"mode":"download","is_partial":1,"seq":"415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456","content_length":19144826} +{"datetime":"2025-09-01 15:28:00","session":"xxx","app":"imgsrv","id":"mdp.30000080041746","attr":9,"ic":0,"access":"success","access_type":"ht_affiliate","access_type_by_attr":"allow","digitization_source":"google","collection_source":"umich","remote_addr":"192.0.2.0","proxied_addr":null,"remote_user_processed":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","remote_user_from_env":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","affiliation":"employee@default.invalid;member@default.invalid;staff@default.invalid","eppn":"example@default.invalid","display_name":"Example, Bob","entityID":"https://login.default.invalid/cas/idp/metadata","persistent_id":"https://login.default.invalid/cas/idp/metadata!http://www.hathitrust.org/shibboleth-sp!xxx","usertype":"external","role":"ssdproxy","sdrinst":"example","auth_type":"shibboleth","http_host":"babel.hathitrust.org","server_addr":"141.213.128.138","sdrlib":null,"http_referer":"https://babel.hathitrust.org/cgi/pt?id=mdp.39015033899868&seq=456","request_uri":"/cgi/imgsrv/download/pdf?id=mdp.39015033899868;seq=415-456;marker=xxx;attachment=1","user_agent":"sample user agent","inst_code":"default","inst_code_mapped":"default","geo_code":"US","geo_code_proxy":null,"mode":"download","is_partial":1,"seq":"415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456","content_length":19144826} {"datetime":"2025-09-01 15:28:52","session":"yyy","app":"imgsrv","id":"inu.30000080041746","attr":9,"ic":0,"access":"success","access_type":"ordinary_user","access_type_by_attr":"allow_by_us_geo_ipaddr","digitization_source":"google","collection_source":"iu","remote_addr":"2001:db8::/32","proxied_addr":null,"remote_user_processed":null,"remote_user_from_env":null,"affiliation":null,"eppn":null,"display_name":null,"entityID":"notset","persistent_id":"notset","usertype":null,"role":null,"sdrinst":null,"auth_type":null,"http_host":"babel.hathitrust.org","server_addr":"141.213.128.138","sdrlib":null,"http_referer":"https://babel.hathitrust.org/cgi/pt?id=inu.30000080041746&seq=7&q1=Mujaes","request_uri":"/cgi/imgsrv/download/pdf?id=inu.30000080041746&attachment=1&tracker=D1&seq=7","user_agent":"a user agent","inst_code":null,"inst_code_mapped":null,"geo_code":"US","geo_code_proxy":null,"mode":"streaming","is_partial":1,"seq":"7","content_length":273894} {"datetime":"2025-09-01 15:29:03","session":"yyy","app":"imgsrv","id":"inu.30000080041746","attr":9,"ic":0,"access":"success","access_type":"ordinary_user","access_type_by_attr":"allow_by_us_geo_ipaddr","digitization_source":"google","collection_source":"iu","remote_addr":"2001:db8::/32","proxied_addr":null,"remote_user_processed":null,"remote_user_from_env":null,"affiliation":null,"eppn":null,"display_name":null,"entityID":"notset","persistent_id":"notset","usertype":null,"role":null,"sdrinst":null,"auth_type":null,"http_host":"babel.hathitrust.org","server_addr":"141.213.128.138","sdrlib":null,"http_referer":"https://babel.hathitrust.org/cgi/pt?id=inu.30000080041746&seq=8&q1=Mujaes","request_uri":"/cgi/imgsrv/download/pdf?id=inu.30000080041746&attachment=1&tracker=D2&seq=8","user_agent":"a user agent","inst_code":null,"inst_code_mapped":null,"geo_code":"US","geo_code_proxy":null,"mode":"streaming","is_partial":1,"seq":"8","content_length":93412} {"datetime":"2025-09-01 15:29:11","session":"yyy","app":"imgsrv","id":"inu.30000080041746","attr":9,"ic":0,"access":"success","access_type":"ordinary_user","access_type_by_attr":"allow_by_us_geo_ipaddr","digitization_source":"google","collection_source":"iu","remote_addr":"2001:db8::/32","proxied_addr":null,"remote_user_processed":null,"remote_user_from_env":null,"affiliation":null,"eppn":null,"display_name":null,"entityID":"notset","persistent_id":"notset","usertype":null,"role":null,"sdrinst":null,"auth_type":null,"http_host":"babel.hathitrust.org","server_addr":"141.213.128.138","sdrlib":null,"http_referer":"https://babel.hathitrust.org/cgi/pt?id=inu.30000080041746&seq=9&q1=Mujaes","request_uri":"/cgi/imgsrv/download/pdf?id=inu.30000080041746&attachment=1&tracker=D3&seq=9","user_agent":"a user agent","inst_code":null,"inst_code_mapped":null,"geo_code":"US","geo_code_proxy":null,"mode":"streaming","is_partial":1,"seq":"9","content_length":160982} diff --git a/spec/fixtures/ulib-logs/archive/macc-ht-web-189.umdl.umich.edu/var/log/babel/access-imgsrv_downloads.log-20250902.gz b/spec/fixtures/ulib-logs/archive/macc-ht-web-189.umdl.umich.edu/var/log/babel/access-imgsrv_downloads.log-20250902.gz index d37714e1..7aa25a07 100644 Binary files a/spec/fixtures/ulib-logs/archive/macc-ht-web-189.umdl.umich.edu/var/log/babel/access-imgsrv_downloads.log-20250902.gz and b/spec/fixtures/ulib-logs/archive/macc-ht-web-189.umdl.umich.edu/var/log/babel/access-imgsrv_downloads.log-20250902.gz differ