From e976b55a5c2a83f4f2ab1221ea228c9b8737db4f Mon Sep 17 00:00:00 2001 From: Simeon Wong Date: Thu, 14 Nov 2024 20:32:09 -0500 Subject: [PATCH 01/14] delete ip logs... WARNING UNTESTED! --- 02_activities/assignments/assignment.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index d81e9a77b..001291867 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -33,6 +33,7 @@ unzip rawdata.zip # 6. Repeat the above step for user logs and event logs # 7. For user privacy, remove all files containing IP addresses (files with "ipaddr" in the filename) from ./data/raw and ./data/processed/user_logs +rf -rf ./data # 8. Create a file named ./data/inventory.txt that lists all the files in the subfolders of ./data/processed From ea20676d33161a6f4d0fcd3c4f7aa5360f0f4309 Mon Sep 17 00:00:00 2001 From: Simeon Wong Date: Thu, 14 Nov 2024 20:55:44 -0500 Subject: [PATCH 02/14] initialize README file with company name --- 02_activities/assignments/assignment.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 001291867..f2bfd22bd 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -11,6 +11,7 @@ set -x mkdir analysis output touch README.md +echo "# Project Name: DSI Consulting Inc." > README.md touch analysis/main.py # download client data From 59b73114322ae33a77f783a1b57cb813ffe316ae Mon Sep 17 00:00:00 2001 From: OlgaNazarenko Date: Wed, 30 Jul 2025 19:47:18 -0400 Subject: [PATCH 03/14] Made some changes in the file --- 02_activities/assignments/assignment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 8af8dc092..14e939eb1 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -28,7 +28,7 @@ unzip -q rawdata.zip # Complete assignment here # 1. Create a directory named data - +mkdir data # 2. Move the ./rawdata directory to ./data/raw # 3. List the contents of the ./data/raw directory From ebc996183253a6c8da9a3da801831aa8c362ddd5 Mon Sep 17 00:00:00 2001 From: OlgaNazarenko Date: Wed, 30 Jul 2025 23:21:20 -0400 Subject: [PATCH 04/14] Modified the file to include the remaining required tasks. --- 02_activities/assignments/assignment.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 14e939eb1..a5a76da02 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x ############################################ # DSI CONSULTING INC. Project setup script # @@ -30,18 +29,30 @@ unzip -q rawdata.zip # 1. Create a directory named data mkdir data # 2. Move the ./rawdata directory to ./data/raw +mv ./rawdata ./data/raw # 3. List the contents of the ./data/raw directory +ls ./data/raw # 4. In ./data/processed, create the following directories: server_logs, user_logs, and event_logs +mkdir ./data/processed/server_logs ./data/processed/user_logs ./data/processed/event_logs # 5. Copy all server log files (files with "server" in the name AND a .log extension) from ./data/raw to ./data/processed/server_logs +cp ./data/raw/*server*.log ./data/processed/server_logs # 6. Repeat the above step for user logs and event logs +cp ./data/raw/*event*.log ./data/processed/event_logs +cp ./data/raw/*user*.log ./data/processed/user_logs # 7. For user privacy, remove all files containing IP addresses (files with "ipaddr" in the filename) from ./data/raw and ./data/processed/user_logs +rm ./data/raw/*ipaddr* +rm ./data/processed/user_logs/*ipaddr* + # 8. Create a file named ./data/inventory.txt that lists all the files in the subfolders of ./data/processed +touch ./data/inventory.txt + +ls ./data/processed/* > ./data/inventory.txt ########################################### From 48d602bcc5f29230c411916b3bc62ee784d567ea Mon Sep 17 00:00:00 2001 From: OlgaNazarenko Date: Wed, 30 Jul 2025 23:26:11 -0400 Subject: [PATCH 05/14] Made some modifications in the file. --- 02_activities/assignments/assignment.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index a5a76da02..8eec4b679 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -35,7 +35,9 @@ mv ./rawdata ./data/raw ls ./data/raw # 4. In ./data/processed, create the following directories: server_logs, user_logs, and event_logs -mkdir ./data/processed/server_logs ./data/processed/user_logs ./data/processed/event_logs +mkdir ./data/processed/server_logs +mkdir ./data/processed/user_logs +mkdir ./data/processed/event_logs # 5. Copy all server log files (files with "server" in the name AND a .log extension) from ./data/raw to ./data/processed/server_logs cp ./data/raw/*server*.log ./data/processed/server_logs From a3397003448f4f932f8648f53817b37f154edc15 Mon Sep 17 00:00:00 2001 From: OlgaNazarenko Date: Thu, 31 Jul 2025 20:58:27 -0400 Subject: [PATCH 06/14] Made some changes in the file, ls command. --- 02_activities/assignments/assignment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 8eec4b679..543a983ca 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -32,7 +32,7 @@ mkdir data mv ./rawdata ./data/raw # 3. List the contents of the ./data/raw directory -ls ./data/raw +ls # 4. In ./data/processed, create the following directories: server_logs, user_logs, and event_logs mkdir ./data/processed/server_logs From 87d98d9fbe9eed7d911f534d757bbf526832c6e6 Mon Sep 17 00:00:00 2001 From: OlgaNazarenko Date: Thu, 31 Jul 2025 21:13:12 -0400 Subject: [PATCH 07/14] Added some changes in the commands. --- 02_activities/assignments/assignment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 543a983ca..8eec4b679 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -32,7 +32,7 @@ mkdir data mv ./rawdata ./data/raw # 3. List the contents of the ./data/raw directory -ls +ls ./data/raw # 4. In ./data/processed, create the following directories: server_logs, user_logs, and event_logs mkdir ./data/processed/server_logs From 3181bb346f25222c158fc38eafc0d4139dd4d3af Mon Sep 17 00:00:00 2001 From: OlgaNazarenko Date: Thu, 31 Jul 2025 21:17:44 -0400 Subject: [PATCH 08/14] Changes --- 02_activities/assignments/assignment.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 8eec4b679..0bec92f0a 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -35,9 +35,12 @@ mv ./rawdata ./data/raw ls ./data/raw # 4. In ./data/processed, create the following directories: server_logs, user_logs, and event_logs -mkdir ./data/processed/server_logs -mkdir ./data/processed/user_logs -mkdir ./data/processed/event_logs +cd data +mkdir processed +cd processed +mkdir server_logs +mkdir user_logs +mkdir event_logs # 5. Copy all server log files (files with "server" in the name AND a .log extension) from ./data/raw to ./data/processed/server_logs cp ./data/raw/*server*.log ./data/processed/server_logs From 2b1f6a28936d2a755d26d949c9b1dce03889d46d Mon Sep 17 00:00:00 2001 From: OlgaNazarenko Date: Thu, 31 Jul 2025 21:33:20 -0400 Subject: [PATCH 09/14] Additional changes in the code --- 02_activities/assignments/assignment.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 0bec92f0a..ace4ea6ca 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -32,7 +32,7 @@ mkdir data mv ./rawdata ./data/raw # 3. List the contents of the ./data/raw directory -ls ./data/raw +ls # 4. In ./data/processed, create the following directories: server_logs, user_logs, and event_logs cd data @@ -43,21 +43,24 @@ mkdir user_logs mkdir event_logs # 5. Copy all server log files (files with "server" in the name AND a .log extension) from ./data/raw to ./data/processed/server_logs -cp ./data/raw/*server*.log ./data/processed/server_logs +cd .. +ls +cp raw/rawdata/*server*.log processed/server_logs/ # 6. Repeat the above step for user logs and event logs -cp ./data/raw/*event*.log ./data/processed/event_logs -cp ./data/raw/*user*.log ./data/processed/user_logs +cp raw/rawdata/*user*.log processed/user_logs +cp raw/rawdata/*event*.log processed/event_logs # 7. For user privacy, remove all files containing IP addresses (files with "ipaddr" in the filename) from ./data/raw and ./data/processed/user_logs -rm ./data/raw/*ipaddr* -rm ./data/processed/user_logs/*ipaddr* +rm raw/rawdata/*ipaddr* +rm processed/user_logs/*ipaddr* # 8. Create a file named ./data/inventory.txt that lists all the files in the subfolders of ./data/processed -touch ./data/inventory.txt +touch inventory.txt + +ls processed/* > inventory.txt -ls ./data/processed/* > ./data/inventory.txt ########################################### From 0160e4a54dec87ebc4f14ea7d8a3291cca1dcaad Mon Sep 17 00:00:00 2001 From: OlgaNazarenko Date: Thu, 31 Jul 2025 21:40:47 -0400 Subject: [PATCH 10/14] Other changes --- 02_activities/assignments/assignment.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index ace4ea6ca..d6ccfceab 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -32,7 +32,7 @@ mkdir data mv ./rawdata ./data/raw # 3. List the contents of the ./data/raw directory -ls +ls ./data/raw # 4. In ./data/processed, create the following directories: server_logs, user_logs, and event_logs cd data @@ -43,24 +43,27 @@ mkdir user_logs mkdir event_logs # 5. Copy all server log files (files with "server" in the name AND a .log extension) from ./data/raw to ./data/processed/server_logs -cd .. -ls cp raw/rawdata/*server*.log processed/server_logs/ # 6. Repeat the above step for user logs and event logs cp raw/rawdata/*user*.log processed/user_logs cp raw/rawdata/*event*.log processed/event_logs +cp ./data/raw/*user*.log ./data/processed/user_logs +cp ./data/raw/*event*.log ./data/processed/event_logs + # 7. For user privacy, remove all files containing IP addresses (files with "ipaddr" in the filename) from ./data/raw and ./data/processed/user_logs -rm raw/rawdata/*ipaddr* -rm processed/user_logs/*ipaddr* + ## rm raw/rawdata/*ipaddr* + ## rm processed/user_logs/*ipaddr* +rm ./data/raw/*ipaddr* +rm ./data/processed/user_logs/*ipaddr* # 8. Create a file named ./data/inventory.txt that lists all the files in the subfolders of ./data/processed -touch inventory.txt - -ls processed/* > inventory.txt + ## touch inventory.txt + ## ls processed/* > inventory.txt +find ./data/processed/ -type f > ./data/inventory.txt ########################################### From 090e9b9d62dadca83efc017a120fab7621f123ba Mon Sep 17 00:00:00 2001 From: OlgaNazarenko Date: Thu, 31 Jul 2025 21:52:09 -0400 Subject: [PATCH 11/14] Other changes --- 02_activities/assignments/assignment.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index d6ccfceab..d04a85844 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -33,6 +33,8 @@ mv ./rawdata ./data/raw # 3. List the contents of the ./data/raw directory ls ./data/raw + ## cd .. + ## ls ./data/raw # 4. In ./data/processed, create the following directories: server_logs, user_logs, and event_logs cd data From cb9c400bc111ef5d174d67c942e0e5ad82057e3e Mon Sep 17 00:00:00 2001 From: OlgaNazarenko Date: Thu, 31 Jul 2025 21:54:07 -0400 Subject: [PATCH 12/14] Other changes --- 02_activities/assignments/assignment.sh | 32 ++++--------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index d04a85844..0376f7d31 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -x ############################################ # DSI CONSULTING INC. Project setup script # @@ -30,44 +31,21 @@ unzip -q rawdata.zip mkdir data # 2. Move the ./rawdata directory to ./data/raw mv ./rawdata ./data/raw - # 3. List the contents of the ./data/raw directory ls ./data/raw - ## cd .. - ## ls ./data/raw - # 4. In ./data/processed, create the following directories: server_logs, user_logs, and event_logs -cd data -mkdir processed -cd processed -mkdir server_logs -mkdir user_logs -mkdir event_logs - +mkdir ./data/processed +mkdir ./data/processed/server_logs ./data/processed/user_logs ./data/processed/event_logs # 5. Copy all server log files (files with "server" in the name AND a .log extension) from ./data/raw to ./data/processed/server_logs -cp raw/rawdata/*server*.log processed/server_logs/ - +cp ./data/raw/*server*.log ./data/processed/server_logs # 6. Repeat the above step for user logs and event logs -cp raw/rawdata/*user*.log processed/user_logs -cp raw/rawdata/*event*.log processed/event_logs - cp ./data/raw/*user*.log ./data/processed/user_logs cp ./data/raw/*event*.log ./data/processed/event_logs - # 7. For user privacy, remove all files containing IP addresses (files with "ipaddr" in the filename) from ./data/raw and ./data/processed/user_logs - ## rm raw/rawdata/*ipaddr* - ## rm processed/user_logs/*ipaddr* - rm ./data/raw/*ipaddr* rm ./data/processed/user_logs/*ipaddr* - # 8. Create a file named ./data/inventory.txt that lists all the files in the subfolders of ./data/processed - ## touch inventory.txt - ## ls processed/* > inventory.txt - find ./data/processed/ -type f > ./data/inventory.txt - - ########################################### -echo "Project setup is complete!" +echo "Project setup is complete!" \ No newline at end of file From fcee092c9bf27322ccc6da3fff1c1c99019443e9 Mon Sep 17 00:00:00 2001 From: OlgaNazarenko Date: Thu, 31 Jul 2025 22:08:45 -0400 Subject: [PATCH 13/14] Other changes --- 02_activities/assignments/assignment.sh | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 0376f7d31..540b1760f 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -29,23 +29,30 @@ unzip -q rawdata.zip # 1. Create a directory named data mkdir data + # 2. Move the ./rawdata directory to ./data/raw mv ./rawdata ./data/raw + # 3. List the contents of the ./data/raw directory -ls ./data/raw +ls -lrt ./data/raw + # 4. In ./data/processed, create the following directories: server_logs, user_logs, and event_logs mkdir ./data/processed mkdir ./data/processed/server_logs ./data/processed/user_logs ./data/processed/event_logs + # 5. Copy all server log files (files with "server" in the name AND a .log extension) from ./data/raw to ./data/processed/server_logs -cp ./data/raw/*server*.log ./data/processed/server_logs +cp -p ./data/raw/*server*.log ./data/processed/server_logs + # 6. Repeat the above step for user logs and event logs -cp ./data/raw/*user*.log ./data/processed/user_logs -cp ./data/raw/*event*.log ./data/processed/event_logs +cp -p ./data/raw/*user*.log ./data/processed/user_logs +cp -p ./data/raw/*event*.log ./data/processed/event_logs + # 7. For user privacy, remove all files containing IP addresses (files with "ipaddr" in the filename) from ./data/raw and ./data/processed/user_logs -rm ./data/raw/*ipaddr* -rm ./data/processed/user_logs/*ipaddr* +rm ./data/raw/*ipaddr* ./data/processed/user_logs/*ipaddr* + # 8. Create a file named ./data/inventory.txt that lists all the files in the subfolders of ./data/processed -find ./data/processed/ -type f > ./data/inventory.txt +ls ./data/processed/* > ./data/inventory.txt + ########################################### echo "Project setup is complete!" \ No newline at end of file From 741d63c3f9915d709a7fb898b374abc40a606f90 Mon Sep 17 00:00:00 2001 From: OlgaNazarenko Date: Thu, 31 Jul 2025 22:11:39 -0400 Subject: [PATCH 14/14] Other changes --- 02_activities/assignments/assignment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/02_activities/assignments/assignment.sh b/02_activities/assignments/assignment.sh index 540b1760f..5c4ec6eca 100644 --- a/02_activities/assignments/assignment.sh +++ b/02_activities/assignments/assignment.sh @@ -34,7 +34,7 @@ mkdir data mv ./rawdata ./data/raw # 3. List the contents of the ./data/raw directory -ls -lrt ./data/raw +ls ./data/raw # 4. In ./data/processed, create the following directories: server_logs, user_logs, and event_logs mkdir ./data/processed