Merge pull request #1 from aws-samples/release_1_02

Pause/Resume data collection

Setup.md

@@ -35,7 +35,7 @@
    │   ├── Home.py
    │   ├── pages
    │   │   ├── 1_CIFS_and_NFS_Sessions_Report.py
-   │   │   └── 99_Add_storage.py
+   │   │   └── 99_Manage_storage_systems.py
    │   └── requirements.txt
    ├── collector.env
    ├── commons
@@ -85,15 +85,18 @@
 11. Start a web browser and connect to this linux host to port 8080. On first lauch you will see empty tables as shown in this figure below:  
    <img src='./images/first_launch.jpg' width=800>  
 
-12. Now click on Add Storage as highlighted to add new NetApp storage systems.  
+12. Now click on *`Manage storage systems`* as highlighted to add new NetApp storage systems.  
 
 13. Fill the form with NetApp Storage Details - **Name**, **IP Address**, **Username** and **Password**.  
     <img src='./images/add_storage.jpg' width=800>  
 
 14. Navigate to **Home** page to verify the data collection was successful.  
     <img src='./images/home_page.jpg' width=800>  
 
-15. Add storage systems as needed.  
+15. You can pause data collection for any storage system added to the data collector by navigating to `*Manage storage systems*` page and clicking on the toggle switch next to the **Storage Name**.
+    <img src='./images/manage_storage_data_collection.jpg' width=800>
+
+16. Add storage systems as needed.  
 
 ---
 # Stopping data collection

app/Home.py

@@ -49,7 +49,8 @@ def main():
 
     ## Show the storge systems configured
     with st.sidebar.container(border=True):
-        st.dataframe(stContainersDf.get_configured_storage(cursor=cursor)[['Name','StorageIP']], hide_index=True, use_container_width=True)
+        sidebar_storage_df = stContainersDf.get_configured_storage(cursor=cursor)[['Name', 'StorageIP', 'CollectData']]
+        st.dataframe(sidebar_storage_df, hide_index=True, use_container_width=True)
 
     ## Show the range of timestamps of the data collected.
     time_last, time_first, sessionserver_count = stContainersDf.get_sessions_details(cursor=cursor)[0]

app/pages/99_Add_storage.py → app/pages/99_Manage_storage_systems.py

@@ -29,12 +29,12 @@
 
 
 st.set_page_config(
-    page_title="Add NetApp Storage system",
+    page_title="Manage Storage systems",
     page_icon="🏠",
     initial_sidebar_state="expanded",
     layout='wide'
     )
-st.title("Add NetApp Storage system to data collector")
+st.title("Manage Storage systems data collection")
 
 
 def verify_add_storage_form(fernet_key, data):
@@ -77,10 +77,10 @@ def main():
 
     # Show Configured storage systems in Sidebar
     with st.sidebar.container(border=True):
-        sidebar_storage_df = stContainersDf.get_configured_storage(cursor=cursor)[['Name', 'StorageIP']]
+        sidebar_storage_df = stContainersDf.get_configured_storage(cursor=cursor)[['Name', 'StorageIP', 'CollectData']]
         st.dataframe(sidebar_storage_df, hide_index=True, use_container_width=True)
 
-    col11, col12, col13 = st.columns([5, 10, 5])
+    col11, col12, col13, col14 = st.columns([1, 8, 5, 4])
     with col11:
         st.empty()
 
@@ -102,7 +102,8 @@ def main():
                                 "storage_name":storage_name, 
                                 "storage_ip":storage_ip, 
                                 "storage_user":storage_user, 
-                                "storage_password":fernet_key.encrypt(storage_password.encode())
+                                "storage_password":fernet_key.encrypt(storage_password.encode()),
+                                "collectdata" : True
                             }
                             if verify_add_storage_form(data=formData, fernet_key=fernet_key):
                                 pgDb.store_storage_config(conn=conn, cursor=cursor, data=formData)
@@ -114,9 +115,32 @@ def main():
                         except (pg.errors.UniqueViolation, pg.errors.IntegrityError) as e:
                             st.error(e.pgerror.split('DETAIL:  Key ')[1])
 
+    def update_storage_collection(conn, cursor, storage):
+        data = {
+            'collectdata' : st.session_state[storage['Name']],
+            'storagename' : storage['Name']
+        }
+        pgDb.update_storage_collection(conn, cursor, data)
+        return storage
     with col13:
+        with st.container(border=True):
+            st.subheader("Update data collection")
+            for storage in  sidebar_storage_df.iterrows():
+                if storage[1]['CollectData']:
+                    collection_label = f":green[{storage[1]['Name'].upper()}] data collection :green[ENABLED]"
+                else:
+                    collection_label = f":orange[{storage[1]['Name'].upper()}] data collection :orange[DISABLED]"
+
+                st.toggle(
+                    collection_label, 
+                    key=storage[1]['Name'], 
+                    value=storage[1]['CollectData'],
+                    on_change=update_storage_collection,
+                    kwargs={"conn":conn, "cursor":cursor, 'storage':storage[1]}
+                )
+
+    with col14:
         st.empty()
-
 if __name__ == "__main__":
     main()
 

commons/database.py

@@ -46,14 +46,28 @@ def store_storage_config(conn, cursor, data):
             None
         """
         cursor.execute(f"""
-            INSERT INTO public.storageconfigs (storagename, storageip, storageuser, storagepassword)
+            INSERT INTO public.storageconfigs (storagename, storageip, storageuser, storagepassword, collectdata)
             VALUES (
                 '{data['storage_name']}', 
                 '{data['storage_ip']}', 
                 '{data['storage_user']}', 
-                {pg.Binary(data['storage_password'])}
+                {pg.Binary(data['storage_password'])},
+                {data['collectdata']}
             )
         """)
         conn.commit()
 
 
+    def update_storage_collection(conn, cursor, data):
+        query=f"""
+            UPDATE 
+                public.storageconfigs 
+            SET 
+                collectdata={data['collectdata']} 
+            WHERE 
+                storagename='{data['storagename']}';
+        """
+        print(query)
+        cursor.execute(query)
+        conn.commit()
+

commons/netappCollector.py

@@ -237,13 +237,14 @@ def main():
 
         # Collect data for each Storage configured
         for index, storage in storage_list_df.iterrows():
-            storage_system = {'Name':storage['Name'], 'Address':storage['StorageIP'], 'Credentials':[storage['StorageUser'], fernet_key.decrypt(storage['StoragePassEnc'].tobytes()).decode()]}
-            storage_system['netapp_storage'] = get_cluster_information(storage_system, SSL_VERIFY)
-            storage_system['get_cifs_sessions_data'] = threading.Thread(target=get_cifs_sessions_data, args=(conn, cursor, storage_system, SSL_VERIFY,))
-            storage_system['get_nfs_clients_data'] = threading.Thread(target=get_nfs_clients_data, args=(conn, cursor, storage_system, SSL_VERIFY,))
-            storage_system['get_cifs_sessions_data'].start()
-            storage_system['get_nfs_clients_data'].start()
-            storage_list.append(storage_system)
+            if storage['CollectData']:
+                storage_system = {'Name':storage['Name'], 'Address':storage['StorageIP'], 'Credentials':[storage['StorageUser'], fernet_key.decrypt(storage['StoragePassEnc'].tobytes()).decode()], 'CollectData':storage['CollectData']}
+                storage_system['netapp_storage'] = get_cluster_information(storage_system, SSL_VERIFY)
+                storage_system['get_cifs_sessions_data'] = threading.Thread(target=get_cifs_sessions_data, args=(conn, cursor, storage_system, SSL_VERIFY,))
+                storage_system['get_nfs_clients_data'] = threading.Thread(target=get_nfs_clients_data, args=(conn, cursor, storage_system, SSL_VERIFY,))
+                storage_system['get_cifs_sessions_data'].start()
+                storage_system['get_nfs_clients_data'].start()
+                storage_list.append(storage_system)
 
         sleep(POLL_INTERVAL)
 

commons/setupDb.py

@@ -2,7 +2,7 @@
 import sys
 sys.path.append(os.environ['PROJECT_HOME'])
 
-from sqlalchemy import create_engine, Column, String, TIMESTAMP, Table, Index, MetaData, LargeBinary
+from sqlalchemy import create_engine, Column, String, TIMESTAMP, Table, Index, MetaData, LargeBinary, Boolean
 from sqlalchemy.orm import declarative_base, Session
 from sqlalchemy.exc import ProgrammingError
 from urllib.parse import quote_plus
@@ -42,7 +42,8 @@ def create_tables(engine):
             Column('storagename', String()),
             Column('storageip', String()),
             Column('storageuser', String()),
-            Column('storagepassword', LargeBinary)
+            Column('storagepassword', LargeBinary),
+            Column('collectdata', Boolean, default=True, nullable=False)
         ).create(bind=engine)
     except ProgrammingError as e:
         if "already exists" not in str(e):

commons/streamlitDfs.py

@@ -9,7 +9,7 @@ def get_configured_storage(cursor):
                 SELECT * FROM storageconfigs s ORDER by s.storagename;
             """)
             storage_list = cursor.fetchall()
-            storage_df = pd.DataFrame(storage_list, columns=['Name', 'StorageIP', 'StorageUser', 'StoragePassEnc'])
+            storage_df = pd.DataFrame(storage_list, columns=['Name', 'StorageIP', 'StorageUser', 'StoragePassEnc', 'CollectData'])
             return storage_df
         except Exception as e:
             return traceback.format_exc()