From c5c0866068c8513e534d9d6378ba6c7ae4516a92 Mon Sep 17 00:00:00 2001 From: gauravramachandra Date: Fri, 25 Jul 2025 15:11:27 +0530 Subject: [PATCH 1/3] Updated README.md to implement email notifications from jenksins and clear debugging for all issues --- README.md | 316 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 206 insertions(+), 110 deletions(-) diff --git a/README.md b/README.md index 17faac6e..741b8245 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,11 @@ # **Youtube Video for step by step Demonstration!** -[![Video Tutorial](https://img.youtube.com/vi/g8X5AoqCJHc/0.jpg)](https://youtu.be/g8X5AoqCJHc) +[![Video Tutorial](https://img.youtube.com/vi/g8X5AoqCJHc/0.jpg)](https://youtu.be/g8X5AoqCJHc) ## Susbcribe: + [https://www.youtube.com/@cloudchamp? ](https://www.youtube.com/@cloudchamp?sub_confirmation=1) @@ -35,35 +36,33 @@ - Update all the packages and then clone the code. - Clone your application's code repository onto the EC2 instance: - - ```bash - git clone https://github.com/N4si/DevSecOps-Project.git - ``` - + ```bash + git clone https://github.com/N4si/DevSecOps-Project.git + ``` **Step 3: Install Docker and Run the App Using a Container:** - Set up Docker on the EC2 instance: - - ```bash - - sudo apt-get update - sudo apt-get install docker.io -y - sudo usermod -aG docker $USER # Replace with your system's username, e.g., 'ubuntu' - newgrp docker - sudo chmod 777 /var/run/docker.sock - ``` - + + ```bash + + sudo apt-get update + sudo apt-get install docker.io -y + sudo usermod -aG docker $USER # Replace with your system's username, e.g., 'ubuntu' (command not compulsory) + newgrp docker + sudo chmod 777 /var/run/docker.sock + ``` + - Build and run your application using Docker containers: - - ```bash - docker build -t netflix . - docker run -d --name netflix -p 8081:80 netflix:latest - - #to delete - docker stop - docker rmi -f netflix - ``` + + ```bash + docker build -t netflix . + docker run -d --name netflix -p 8081:80 netflix:latest + + #to delete + docker stop + docker rmi -f netflix + ``` It will show an error cause you need API key @@ -78,6 +77,7 @@ It will show an error cause you need API key - You will receive your TMDB API key. Now recreate the Docker image with your api key: + ``` docker build --build-arg TMDB_V3_API_KEY= -t netflix . ``` @@ -85,67 +85,59 @@ docker build --build-arg TMDB_V3_API_KEY= -t netflix . **Phase 2: Security** 1. **Install SonarQube and Trivy:** - - Install SonarQube and Trivy on the EC2 instance to scan for vulnerabilities. - - sonarqube - ``` - docker run -d --name sonar -p 9000:9000 sonarqube:lts-community - ``` - - - To access: - - publicIP:9000 (by default username & password is admin) - - To install Trivy: - ``` - sudo apt-get install wget apt-transport-https gnupg lsb-release - wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - - echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list - sudo apt-get update - sudo apt-get install trivy - ``` - - to scan image using trivy - ``` - trivy image - ``` - - + - Install SonarQube and Trivy on the EC2 instance to scan for vulnerabilities. + sonarqube + ``` + docker run -d --name sonar -p 9000:9000 sonarqube:lts-community + ``` + To access: + publicIP:9000 (by default username & password is admin) + To install Trivy: + ``` + sudo apt-get install wget apt-transport-https gnupg lsb-release + wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - + echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list + sudo apt-get update + sudo apt-get install trivy + ``` + to scan image using trivy + ``` + trivy image + ``` 2. **Integrate SonarQube and Configure:** - - Integrate SonarQube with your CI/CD pipeline. - - Configure SonarQube to analyze code for quality and security issues. + - Integrate SonarQube with your CI/CD pipeline. + - Configure SonarQube to analyze code for quality and security issues. **Phase 3: CI/CD Setup** 1. **Install Jenkins for Automation:** - - Install Jenkins on the EC2 instance to automate deployment: - Install Java - - ```bash - sudo apt update - sudo apt install fontconfig openjdk-17-jre - java -version - openjdk version "17.0.8" 2023-07-18 - OpenJDK Runtime Environment (build 17.0.8+7-Debian-1deb12u1) - OpenJDK 64-Bit Server VM (build 17.0.8+7-Debian-1deb12u1, mixed mode, sharing) - - #jenkins - sudo wget -O /usr/share/keyrings/jenkins-keyring.asc \ - https://pkg.jenkins.io/debian-stable/jenkins.io-2023.key - echo deb [signed-by=/usr/share/keyrings/jenkins-keyring.asc] \ - https://pkg.jenkins.io/debian-stable binary/ | sudo tee \ - /etc/apt/sources.list.d/jenkins.list > /dev/null - sudo apt-get update - sudo apt-get install jenkins - sudo systemctl start jenkins - sudo systemctl enable jenkins - ``` - - - Access Jenkins in a web browser using the public IP of your EC2 instance. - - publicIp:8080 - + + - Install Jenkins on the EC2 instance to automate deployment: + Install Java + + ```bash + sudo apt update + sudo apt install fontconfig openjdk-17-jre + java -version + openjdk version "17.0.8" 2023-07-18 + OpenJDK Runtime Environment (build 17.0.8+7-Debian-1deb12u1) + OpenJDK 64-Bit Server VM (build 17.0.8+7-Debian-1deb12u1, mixed mode, sharing) + + #jenkins + sudo wget -O /usr/share/keyrings/jenkins-keyring.asc \ + https://pkg.jenkins.io/debian-stable/jenkins.io-2023.key + echo deb [signed-by=/usr/share/keyrings/jenkins-keyring.asc] \ + https://pkg.jenkins.io/debian-stable binary/ | sudo tee \ + /etc/apt/sources.list.d/jenkins.list > /dev/null + sudo apt-get update + sudo apt-get install jenkins + sudo systemctl start jenkins + sudo systemctl enable jenkins + ``` + + - Access Jenkins in a web browser using the public IP of your EC2 instance. + publicIp:8080 + 2. **Install Necessary Plugins in Jenkins:** Goto Manage Jenkins →Plugins → Available Plugins → @@ -164,7 +156,6 @@ Install below plugins Goto Manage Jenkins → Tools → Install JDK(17) and NodeJs(16)→ Click on Apply and Save - ### SonarQube Create the token @@ -184,6 +175,7 @@ We will install a sonar scanner in the tools. Create a Jenkins webhook 1. **Configure CI/CD Pipeline in Jenkins:** + - Create a CI/CD pipeline in Jenkins to automate your application deployment. ```groovy @@ -308,9 +300,9 @@ pipeline{ stage("quality gate"){ steps { script { - waitForQualityGate abortPipeline: false, credentialsId: 'Sonar-token' + waitForQualityGate abortPipeline: false, credentialsId: 'Sonar-token' } - } + } } stage('Install Dependencies') { steps { @@ -331,7 +323,7 @@ pipeline{ stage("Docker Build & Push"){ steps{ script{ - withDockerRegistry(credentialsId: 'docker', toolName: 'docker'){ + withDockerRegistry(credentialsId: 'docker', toolName: 'docker'){ sh "docker build --build-arg TMDB_V3_API_KEY= -t netflix ." sh "docker tag netflix nasi101/netflix:latest " sh "docker push nasi101/netflix:latest " @@ -341,7 +333,7 @@ pipeline{ } stage("TRIVY"){ steps{ - sh "trivy image nasi101/netflix:latest > trivyimage.txt" + sh "trivy image nasi101/netflix:latest > trivyimage.txt" } } stage('Deploy to container'){ @@ -351,15 +343,27 @@ pipeline{ } } } +``` +> **Note:** +> If you encounter a "port already allocated" error during the final container deployment stage, remove the '--name netflix' part in the last stage: +> +> ```groovy +> stage('Deploy to container') { +> steps { +> sh 'docker run -d -p 8081:80 nasi101/netflix:latest' +> } +> } +> ``` +> +> This will allocate random name to container and does not conflict with existing containers. + +If you get docker login failed error, then execute below commands in ec2 jenkins terminal: -If you get docker login failed errorr - +```bash sudo su sudo usermod -aG docker jenkins sudo systemctl restart jenkins - - ``` **Phase 4: Monitoring** @@ -532,14 +536,14 @@ sudo systemctl restart jenkins scrape_interval: 15s scrape_configs: - - job_name: 'node_exporter' + - job_name: "node_exporter" static_configs: - - targets: ['localhost:9100'] + - targets: ["localhost:9100"] - - job_name: 'jenkins' - metrics_path: '/prometheus' + - job_name: "jenkins" + metrics_path: "/prometheus" static_configs: - - targets: [':'] + - targets: [":"] ``` Make sure to replace `` and `` with the appropriate values for your Jenkins setup. @@ -560,7 +564,6 @@ sudo systemctl restart jenkins `http://:9090/targets` - ####Grafana **Install Grafana on Ubuntu 22.04 and Set it up to Work with Prometheus** @@ -674,13 +677,106 @@ Grafana is a powerful tool for creating visualizations and dashboards, and you c That's it! You've successfully installed and set up Grafana to work with Prometheus for monitoring and visualization. 2. **Configure Prometheus Plugin Integration:** - - Integrate Jenkins with Prometheus to monitor the CI/CD pipeline. - + - Integrate Jenkins with Prometheus to monitor the CI/CD pipeline. **Phase 5: Notification** 1. **Implement Notification Services:** - - Set up email notifications in Jenkins or other notification mechanisms. + - Set up email notifications in Jenkins or other notification mechanisms. + +```groovy + +pipeline{ + agent any + tools{ + jdk 'jdk17' + nodejs 'node16' + } + environment { + SCANNER_HOME=tool 'sonar-scanner' + } + stages { + stage('clean workspace'){ + steps{ + cleanWs() + } + } + stage('Checkout from Git'){ + steps{ + git branch: 'main', url: 'https://github.com/N4si/DevSecOps-Project.git' + } + } + stage("Sonarqube Analysis "){ + steps{ + withSonarQubeEnv('sonar-server') { + sh ''' $SCANNER_HOME/bin/sonar-scanner -Dsonar.projectName=Netflix \ + -Dsonar.projectKey=Netflix ''' + } + } + } + stage("quality gate"){ + steps { + script { + waitForQualityGate abortPipeline: false, credentialsId: 'Sonar-token' + } + } + } + stage('Install Dependencies') { + steps { + sh "npm install" + } + } + stage('OWASP FS SCAN') { + steps { + dependencyCheck additionalArguments: '--scan ./ --disableYarnAudit --disableNodeAudit', odcInstallation: 'DP-Check' + dependencyCheckPublisher pattern: '**/dependency-check-report.xml' + } + } + stage('TRIVY FS SCAN') { + steps { + sh "trivy fs . > trivyfs.txt" + } + } + stage("Docker Build & Push"){ + steps{ + script{ + withDockerRegistry(credentialsId: 'docker', toolName: 'docker'){ + sh "docker build --build-arg TMDB_V3_API_KEY= -t netflix ." + sh "docker tag netflix nasi101/netflix:latest " + sh "docker push nasi101/netflix:latest " + } + } + } + } + stage("TRIVY"){ + steps{ + sh "trivy image nasi101/netflix:latest > trivyimage.txt" + } + } + stage('Deploy to container'){ + steps{ + sh 'docker run -d --name netflix -p 8081:80 nasi101/netflix:latest' + } + } + } + post { + always { + emailext attachLog: true, + subject: "'${currentBuild.result}'", + body: """ + Project: ${env.JOB_NAME}
+ Build Number: ${env.BUILD_NUMBER}
+ URL: ${env.BUILD_URL}
+ """, + to: '', + attachmentsPattern: 'trivyfs.txt,trivyimage.txt' + } + } +} +``` + +> **Note:** If you encounter any errors during pipeline execution, please refer to the troubleshooting section in [Phase 3 Pipeline Notes](#install-dependency-check-and-docker-tools-in-jenkins) for solutions. + # Phase 6: Kubernetes @@ -698,27 +794,26 @@ To begin monitoring your Kubernetes cluster, you'll install the Prometheus Node 1. Add the Prometheus Community Helm repository: - ```bash - helm repo add prometheus-community https://prometheus-community.github.io/helm-charts - ``` + ```bash + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + ``` 2. Create a Kubernetes namespace for the Node Exporter: - ```bash - kubectl create namespace prometheus-node-exporter - ``` + ```bash + kubectl create namespace prometheus-node-exporter + ``` 3. Install the Node Exporter using Helm: - ```bash - helm install prometheus-node-exporter prometheus-community/prometheus-node-exporter --namespace prometheus-node-exporter - ``` + ```bash + helm install prometheus-node-exporter prometheus-community/prometheus-node-exporter --namespace prometheus-node-exporter + ``` Add a Job to Scrape Metrics on nodeip:9001/metrics in prometheus.yml: Update your Prometheus configuration (prometheus.yml) to add a new job for scraping metrics from nodeip:9001/metrics. You can do this by adding the following configuration to your prometheus.yml file: - ``` - job_name: 'Netflix' metrics_path: '/metrics' @@ -743,6 +838,7 @@ To deploy an application with ArgoCD, you can follow these steps, which I'll out After installing ArgoCD, you need to set up your GitHub repository as a source for your application deployment. This typically involves configuring the connection to your repository and defining the source for your ArgoCD application. The specific steps will depend on your setup and requirements. 3. **Create an ArgoCD Application:** + - `name`: Set the name for your application. - `destination`: Define the destination where your application should be deployed. - `project`: Specify the project the application belongs to. @@ -755,4 +851,4 @@ To deploy an application with ArgoCD, you can follow these steps, which I'll out **Phase 7: Cleanup** 1. **Cleanup AWS EC2 Instances:** - - Terminate AWS EC2 instances that are no longer needed. \ No newline at end of file + - Terminate AWS EC2 instances that are no longer needed. From 6952f349143935a01032805c0048c8aabd85366e Mon Sep 17 00:00:00 2001 From: gauravramachandra Date: Fri, 25 Jul 2025 16:58:29 +0530 Subject: [PATCH 2/3] Updated README.md and fixed movie deduplication to prevent duplicate entries --- README.md | 4 +++- src/store/slices/discover.ts | 43 ++++++++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 741b8245..4008ad22 100644 --- a/README.md +++ b/README.md @@ -345,6 +345,8 @@ pipeline{ } ``` +### Troubleshooting Pipeline Issues + > **Note:** > If you encounter a "port already allocated" error during the final container deployment stage, remove the '--name netflix' part in the last stage: > @@ -775,7 +777,7 @@ pipeline{ } ``` -> **Note:** If you encounter any errors during pipeline execution, please refer to the troubleshooting section in [Phase 3 Pipeline Notes](#install-dependency-check-and-docker-tools-in-jenkins) for solutions. +> **Note:** If you encounter any errors during pipeline execution, please refer to the troubleshooting section in [Troubleshooting Pipeline Issues](#troubleshooting-pipeline-issues) for solutions. # Phase 6: Kubernetes diff --git a/src/store/slices/discover.ts b/src/store/slices/discover.ts index cf96476f..525e8f60 100644 --- a/src/store/slices/discover.ts +++ b/src/store/slices/discover.ts @@ -2,7 +2,28 @@ import { TMDB_V3_API_KEY } from "src/constant"; import { tmdbApi } from "./apiSlice"; import { MEDIA_TYPE, PaginatedMovieResult } from "src/types/Common"; import { MovieDetail } from "src/types/Movie"; -import { createSlice, isAnyOf } from "@reduxjs/toolkit"; +import { createSlice, isAnyOf, PayloadAction } from "@reduxjs/toolkit"; +import type { Movie } from "src/types/Movie"; + +// Define proper types for actions +interface SetNextPageAction { + mediaType: string; + itemKey: string | number; +} + +interface InitiateItemAction { + mediaType: string; + itemKey: string | number; +} + +interface FulfilledAction { + page: number; + results: Movie[]; + total_pages: number; + total_results: number; + mediaType: string; + itemKey: string | number; +} const initialState: Record> = {}; export const initialItemState: PaginatedMovieResult = { @@ -16,27 +37,27 @@ const discoverSlice = createSlice({ name: "discover", initialState, reducers: { - setNextPage: (state, action) => { + setNextPage: (state, action: PayloadAction) => { const { mediaType, itemKey } = action.payload; state[mediaType][itemKey].page += 1; }, - initiateItem: (state, action) => { + initiateItem: (state, action: PayloadAction) => { const { mediaType, itemKey } = action.payload; if (!state[mediaType]) { state[mediaType] = {}; } if (!state[mediaType][itemKey]) { - state[mediaType][itemKey] = initialItemState; + state[mediaType][itemKey] = { ...initialItemState }; } }, }, - extraReducers(builder) { + extraReducers: (builder) => { builder.addMatcher( isAnyOf( extendedApi.endpoints.getVideosByMediaTypeAndCustomGenre.matchFulfilled, extendedApi.endpoints.getVideosByMediaTypeAndGenreId.matchFulfilled ), - (state, action) => { + (state, action: PayloadAction) => { const { page, results, @@ -46,7 +67,15 @@ const discoverSlice = createSlice({ itemKey, } = action.payload; state[mediaType][itemKey].page = page; - state[mediaType][itemKey].results.push(...results); + + // Get existing movie IDs to prevent duplicates + const existingIds = new Set(state[mediaType][itemKey].results.map((movie: Movie) => movie.id)); + + // Filter out movies that already exist + const newResults = results.filter((movie: Movie) => !existingIds.has(movie.id)); + + // Only add new unique movies + state[mediaType][itemKey].results.push(...newResults); state[mediaType][itemKey].total_pages = total_pages; state[mediaType][itemKey].total_results = total_results; } From fa35a9d896a27ba4145e7913e0280b78110635bd Mon Sep 17 00:00:00 2001 From: Gaurav Ramachandra <156442048+gauravramachandra@users.noreply.github.com> Date: Sat, 26 Jul 2025 16:45:29 +0530 Subject: [PATCH 3/3] Create Jenkinsfile created jenkins file in git repo for pipeline execution through github --- Jenkinsfile | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..11dc61e4 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,88 @@ + +pipeline{ + agent any + tools{ + jdk 'jdk17' + nodejs 'node16' + } + environment { + SCANNER_HOME=tool 'sonar-scanner' + } + stages { + stage('clean workspace'){ + steps{ + cleanWs() + } + } + stage('Checkout from Git'){ + steps{ + git branch: 'main', url: 'https://github.com/N4si/DevSecOps-Project.git' + } + } + stage("Sonarqube Analysis "){ + steps{ + withSonarQubeEnv('sonar-server') { + sh ''' $SCANNER_HOME/bin/sonar-scanner -Dsonar.projectName=Netflix \ + -Dsonar.projectKey=Netflix ''' + } + } + } + stage("quality gate"){ + steps { + script { + waitForQualityGate abortPipeline: false, credentialsId: 'Sonar-token' + } + } + } + stage('Install Dependencies') { + steps { + sh "npm install" + } + } + stage('OWASP FS SCAN') { + steps { + dependencyCheck additionalArguments: '--scan ./ --disableYarnAudit --disableNodeAudit', odcInstallation: 'DP-Check' + dependencyCheckPublisher pattern: '**/dependency-check-report.xml' + } + } + stage('TRIVY FS SCAN') { + steps { + sh "trivy fs . > trivyfs.txt" + } + } + stage("Docker Build & Push"){ + steps{ + script{ + withDockerRegistry(credentialsId: 'docker', toolName: 'docker'){ + sh "docker build --build-arg TMDB_V3_API_KEY= -t netflix ." + sh "docker tag netflix nasi101/netflix:latest " + sh "docker push nasi101/netflix:latest " + } + } + } + } + stage("TRIVY"){ + steps{ + sh "trivy image nasi101/netflix:latest > trivyimage.txt" + } + } + stage('Deploy to container'){ + steps{ + sh 'docker run -d --name netflix -p 8081:80 nasi101/netflix:latest' + } + } + } + post { + always { + emailext attachLog: true, + subject: "'${currentBuild.result}'", + body: """ + Project: ${env.JOB_NAME}
+ Build Number: ${env.BUILD_NUMBER}
+ URL: ${env.BUILD_URL}
+ """, + to: '', + attachmentsPattern: 'trivyfs.txt,trivyimage.txt' + } + } +}