From e976b55a5c2a83f4f2ab1221ea228c9b8737db4f Mon Sep 17 00:00:00 2001 From: Simeon Wong Date: Thu, 14 Nov 2024 20:32:09 -0500 Subject: [PATCH 01/11] delete ip logs... WARNING UNTESTED! --- 02_activities/assignments/assignment.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index d81e9a77b..001291867 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -33,6 +33,7 @@ unzip rawdata.zip # 6. Repeat the above step for user logs and event logs # 7. For user privacy, remove all files containing IP addresses (files with "ipaddr" in the filename) from ./data/raw and ./data/processed/user_logs +rf -rf ./data # 8. Create a file named ./data/inventory.txt that lists all the files in the subfolders of ./data/processed From ea20676d33161a6f4d0fcd3c4f7aa5360f0f4309 Mon Sep 17 00:00:00 2001 From: Simeon Wong Date: Thu, 14 Nov 2024 20:55:44 -0500 Subject: [PATCH 02/11] initialize README file with company name --- 02_activities/assignments/assignment.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 001291867..f2bfd22bd 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -11,6 +11,7 @@ set -x mkdir analysis output touch README.md +echo "# Project Name: DSI Consulting Inc." > README.md touch analysis/main.py # download client data From af70500ade444b59dd47c16675ac28476e19dee6 Mon Sep 17 00:00:00 2001 From: Rashida Brown Date: Fri, 6 Feb 2026 13:27:55 -0500 Subject: [PATCH 03/11] made my submission part 1 --- 02_activities/assignments/assignment.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 4b2baa7ae..810214873 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -28,6 +28,7 @@ unzip -q rawdata.zip # Complete assignment here # 1. Create a directory named data +mkdir data # 2. Move the ./rawdata directory to ./data/raw From d6a689e8ec7ee23be4dd7588fa743ba083e9f601 Mon Sep 17 00:00:00 2001 From: Rashida Brown Date: Fri, 6 Feb 2026 15:24:03 -0500 Subject: [PATCH 04/11] Second set of changes to assignment --- 02_activities/assignments/assignment.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 810214873..0f490f134 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -29,12 +29,23 @@ unzip -q rawdata.zip # 1. Create a directory named data mkdir data +cd data # 2. Move the ./rawdata directory to ./data/raw +mkdir raw +cd .. +mv rawdata data/raw # 3. List the contents of the ./data/raw directory +echo "Listing contents of raw directory" +ls data/raw # 4. In ./data/processed, create the following directories: server_logs, user_logs, and event_logs +cd data +mkdir processed + +cd processed +mkdir server_logs user_logs event_logs # 5. Copy all server log files (files with "server" in the name AND a .log extension) from ./data/raw to ./data/processed/server_logs From 3fed0fe5e424e16372ceff85b5a4e2ff1491d262 Mon Sep 17 00:00:00 2001 From: Rashida Brown Date: Tue, 10 Feb 2026 11:04:46 -0500 Subject: [PATCH 05/11] assignment update permissions issue --- 02_activities/assignments/assignment.sh | 30 +++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 0f490f134..576da00c8 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -34,7 +34,7 @@ cd data # 2. Move the ./rawdata directory to ./data/raw mkdir raw cd .. -mv rawdata data/raw +sudo mv rawdata data/raw # 3. List the contents of the ./data/raw directory echo "Listing contents of raw directory" @@ -45,12 +45,38 @@ cd data mkdir processed cd processed -mkdir server_logs user_logs event_logs +#mkdir server_logs user_logs event_logs # 5. Copy all server log files (files with "server" in the name AND a .log extension) from ./data/raw to ./data/processed/server_logs +# Define search parameters +sourceFolder="/data/raw" +#destinationFolder="./server_logs" +#extension=".log" + +#mkdir -p $destinationFolder + +#mv "$sourceFolder"/*server*"$extension" "destinationFolder" # 6. Repeat the above step for user logs and event logs +#user logs +destinationFolder="user_logs" +#extension=".log" + +mkdir $destinationFolder + +cd $sourceFolder + +mv "$sourceFolder/"user_log_1.log "destinationFolder" + +#event logs +#destinationFolder="./event_logs" +#extension=".log" + +#mkdir -p $destinationFolder + +#mv "$sourceFolder"/*event*"$extension" "destinationFolder" + # 7. For user privacy, remove all files containing IP addresses (files with "ipaddr" in the filename) from ./data/raw and ./data/processed/user_logs # 8. Create a file named ./data/inventory.txt that lists all the files in the subfolders of ./data/processed From 8ec95a5c456bf71d7729f7211465c8426a6430d8 Mon Sep 17 00:00:00 2001 From: Rashida Brown Date: Tue, 10 Feb 2026 12:52:52 -0500 Subject: [PATCH 06/11] final project script --- 02_activities/assignments/assignment.sh | 44 +++++++++++++++---------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 576da00c8..5c37bbb7d 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -34,7 +34,7 @@ cd data # 2. Move the ./rawdata directory to ./data/raw mkdir raw cd .. -sudo mv rawdata data/raw +mv rawdata data/raw # 3. List the contents of the ./data/raw directory echo "Listing contents of raw directory" @@ -44,43 +44,53 @@ ls data/raw cd data mkdir processed -cd processed -#mkdir server_logs user_logs event_logs +mkdir -p processed/server_logs processed/user_logs processed/event_logs # 5. Copy all server log files (files with "server" in the name AND a .log extension) from ./data/raw to ./data/processed/server_logs # Define search parameters -sourceFolder="/data/raw" -#destinationFolder="./server_logs" -#extension=".log" +sourceFolder="raw/rawdata" -#mkdir -p $destinationFolder +#server logs +destinationFolder="../../processed/server_logs" -#mv "$sourceFolder"/*server*"$extension" "destinationFolder" +cd "$sourceFolder"/ +pwd + +mv *server*.log "$destinationFolder"/ +pwd # 6. Repeat the above step for user logs and event logs #user logs -destinationFolder="user_logs" -#extension=".log" +destinationFolder="../../processed/user_logs" -mkdir $destinationFolder +pwd -cd $sourceFolder +mv *user*.log "$destinationFolder"/ -mv "$sourceFolder/"user_log_1.log "destinationFolder" #event logs -#destinationFolder="./event_logs" -#extension=".log" +destinationFolder="../../processed/event_logs" + +pwd -#mkdir -p $destinationFolder +mv *event*.log "$destinationFolder"/ -#mv "$sourceFolder"/*event*"$extension" "destinationFolder" # 7. For user privacy, remove all files containing IP addresses (files with "ipaddr" in the filename) from ./data/raw and ./data/processed/user_logs +pwd +rm -rf *ipaddr* + +pwd +rm -rf ../../processed/user_logs/*ipaddr* + # 8. Create a file named ./data/inventory.txt that lists all the files in the subfolders of ./data/processed +cd ../.. +pwd + +find processed/ -type f > inventory.txt ########################################### From db435d02333fc1f874fc15e2e42ba337b9917af0 Mon Sep 17 00:00:00 2001 From: Rashida Brown Date: Tue, 10 Feb 2026 13:35:24 -0500 Subject: [PATCH 07/11] additional folder change issue --- 02_activities/assignments/assignment.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 5c37bbb7d..892c57545 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -34,7 +34,7 @@ cd data # 2. Move the ./rawdata directory to ./data/raw mkdir raw cd .. -mv rawdata data/raw +mv rawdata/* data/raw # 3. List the contents of the ./data/raw directory echo "Listing contents of raw directory" @@ -48,10 +48,10 @@ mkdir -p processed/server_logs processed/user_logs processed/event_logs # 5. Copy all server log files (files with "server" in the name AND a .log extension) from ./data/raw to ./data/processed/server_logs # Define search parameters -sourceFolder="raw/rawdata" +sourceFolder="raw" #server logs -destinationFolder="../../processed/server_logs" +destinationFolder="../processed/server_logs" cd "$sourceFolder"/ pwd @@ -62,7 +62,7 @@ pwd # 6. Repeat the above step for user logs and event logs #user logs -destinationFolder="../../processed/user_logs" +destinationFolder="../processed/user_logs" pwd @@ -70,7 +70,7 @@ mv *user*.log "$destinationFolder"/ #event logs -destinationFolder="../../processed/event_logs" +destinationFolder="../processed/event_logs" pwd @@ -83,11 +83,11 @@ pwd rm -rf *ipaddr* pwd -rm -rf ../../processed/user_logs/*ipaddr* +rm -rf ../processed/user_logs/*ipaddr* # 8. Create a file named ./data/inventory.txt that lists all the files in the subfolders of ./data/processed -cd ../.. +cd .. pwd find processed/ -type f > inventory.txt From e25f1d760694d8513b71a1db4c0a71c668e67b97 Mon Sep 17 00:00:00 2001 From: Rashida Brown Date: Tue, 10 Feb 2026 13:43:20 -0500 Subject: [PATCH 08/11] fixing folder embedding again --- 02_activities/assignments/assignment.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 892c57545..a6533f29c 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -34,7 +34,7 @@ cd data # 2. Move the ./rawdata directory to ./data/raw mkdir raw cd .. -mv rawdata/* data/raw +mv rawdata/ data/raw # 3. List the contents of the ./data/raw directory echo "Listing contents of raw directory" @@ -48,10 +48,10 @@ mkdir -p processed/server_logs processed/user_logs processed/event_logs # 5. Copy all server log files (files with "server" in the name AND a .log extension) from ./data/raw to ./data/processed/server_logs # Define search parameters -sourceFolder="raw" +sourceFolder="raw/rawdata" #server logs -destinationFolder="../processed/server_logs" +destinationFolder="../../processed/server_logs" cd "$sourceFolder"/ pwd @@ -62,7 +62,7 @@ pwd # 6. Repeat the above step for user logs and event logs #user logs -destinationFolder="../processed/user_logs" +destinationFolder="../../processed/user_logs" pwd @@ -70,7 +70,7 @@ mv *user*.log "$destinationFolder"/ #event logs -destinationFolder="../processed/event_logs" +destinationFolder="../../processed/event_logs" pwd @@ -83,7 +83,7 @@ pwd rm -rf *ipaddr* pwd -rm -rf ../processed/user_logs/*ipaddr* +rm -rf ../../processed/user_logs/*ipaddr* # 8. Create a file named ./data/inventory.txt that lists all the files in the subfolders of ./data/processed From afdd4ef9950fa4c07a67255ec6191365e5624ccf Mon Sep 17 00:00:00 2001 From: Rashida Brown Date: Tue, 10 Feb 2026 13:49:41 -0500 Subject: [PATCH 09/11] fixing folder embedding with rawdata files in raw --- 02_activities/assignments/assignment.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index a6533f29c..04c29c98a 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -87,11 +87,14 @@ rm -rf ../../processed/user_logs/*ipaddr* # 8. Create a file named ./data/inventory.txt that lists all the files in the subfolders of ./data/processed +cd .. cd .. pwd find processed/ -type f > inventory.txt +mv raw/rawdata/* raw/ + ########################################### echo "Project setup is complete!" From 88cc0798a6d9b94338e190ca8a515fb0aea708b6 Mon Sep 17 00:00:00 2001 From: Rashida Brown Date: Tue, 10 Feb 2026 13:52:01 -0500 Subject: [PATCH 10/11] changed mv to cp --- 02_activities/assignments/assignment.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 04c29c98a..afc7d3696 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -56,7 +56,7 @@ destinationFolder="../../processed/server_logs" cd "$sourceFolder"/ pwd -mv *server*.log "$destinationFolder"/ +cp *server*.log "$destinationFolder"/ pwd # 6. Repeat the above step for user logs and event logs @@ -66,7 +66,7 @@ destinationFolder="../../processed/user_logs" pwd -mv *user*.log "$destinationFolder"/ +cp *user*.log "$destinationFolder"/ #event logs From 186f86cd9d8644198da53d148f2f9a65725804a8 Mon Sep 17 00:00:00 2001 From: Rashida Brown Date: Tue, 10 Feb 2026 13:52:55 -0500 Subject: [PATCH 11/11] changed mv to cp again --- 02_activities/assignments/assignment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index afc7d3696..98a67eb15 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -74,7 +74,7 @@ destinationFolder="../../processed/event_logs" pwd -mv *event*.log "$destinationFolder"/ +cp *event*.log "$destinationFolder"/ # 7. For user privacy, remove all files containing IP addresses (files with "ipaddr" in the filename) from ./data/raw and ./data/processed/user_logs