From c230419e2288e37bdaeb6625127bb033bc811019 Mon Sep 17 00:00:00 2001 From: Harshan07-web Date: Sat, 16 May 2026 13:56:29 +0530 Subject: [PATCH 1/2] data : add new API ELT pipeline data engineering project --- data/projects.json | 37 +++++++++++++++++++++++++ starter_code/api_data_pipeline.py | 46 +++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 starter_code/api_data_pipeline.py diff --git a/data/projects.json b/data/projects.json index 1bad3d31..63bbcd94 100644 --- a/data/projects.json +++ b/data/projects.json @@ -247,6 +247,43 @@ "OWASP password guidelines: https://owasp.org" ], "starter_code": "starter_code/password_checker.py" + }, + { + "id": 9, + "title": "API ETL pipeline", + "skills": ["Python","pandas","requests","os module"], + "level": "Intermediate", + "interest": "Data", + "time": "Medium", + "description": "Upload a API url, which then fetches data from the API, and automatically transform it into a CSV file.", + "features": [ + "Fetch requests from API", + "handle missing values", + "Normalize nested JSON", + "Generate summary statistics", + "Export the processed CSV for any other Analytics projects" + ], + "tech_stack": ["Python", "pandas","requests","JSON","os module"], + "roadmap": [ + "Step 1: Install required modules via pip", + "Step 2: Find a public API key for this project", + "Step 3: Fetch the data from the API using requests", + "Step 4: Validate the reponse you just fetched From the API", + "Step 5: Normalize the nested JSON data by flattening it", + "Step 6: Use the fetched data to build a pandas dataframe", + "Step 7: Handle missing values or duplicate values", + "Step 8: Export the cleaned datatable to a CSV format", + "Step 9: Generate a summary for the newly created CSV dataset", + "Step 10: Test the file with atleast two different public API keys" + ], + "resources": [ + "pandas docs: https://pandas.pydata.org/docs", + "requests docs: https://requests.readthedocs.io/en/latest/", + "JSON handling in Python: https://docs.python.org/3/library/json.html", + "REST API tutorial: https://restfulapi.net/", + "Real Python API guide: https://realpython.com/api-integration-in-python/" + ], + "starter_code": "starter_code/api_data_pipeline.py" } ] diff --git a/starter_code/api_data_pipeline.py b/starter_code/api_data_pipeline.py new file mode 100644 index 00000000..38f28527 --- /dev/null +++ b/starter_code/api_data_pipeline.py @@ -0,0 +1,46 @@ +""" +api_etl_pipeline.py +=================== +Project: API ETL Pipeline +Difficulty: Intermediate +Skills: Python, requests, pandas, JSON handling, os module +Time: Medium (2–5 days) + +What you will build: + Fetch data from a public API and automatically transform it into + a clean, structured CSV dataset. The pipeline will validate API + responses, normalize nested JSON data, handle missing values, + generate summary statistics, and export the processed dataset + for analytics, dashboards, or machine learning workflows. + + The final cleaned dataset and summary report are exported as files + that can be reused for further data analysis projects. + +How to run: + pip install pandas requests + python api_etl_pipeline.py + +When prompted, enter a public API URL that returns JSON data. +A sample API endpoint is included at the bottom of this file +to help you test the project quickly without searching for APIs. + +Learning goals: + - Working with REST APIs using requests + - Understanding JSON response structures + - Extracting and normalizing nested API data + - Converting JSON data into pandas DataFrames + - Cleaning and transforming datasets + - Exporting structured datasets to CSV files + - Generating reusable summary reports + +Roadmap: + Step 1: Run the script using the sample API endpoint + Step 2: Complete fetch_api_data() to retrieve JSON data safely + Step 3: Complete validate_response() to verify API responses + Step 4: Complete normalize_json() to flatten nested JSON data + Step 5: Complete create_dataframe() to build a pandas DataFrame + Step 6: Complete clean_data() to handle missing values and duplicates + Step 7: Complete export_csv() to save the processed dataset + Step 8: Complete generate_summary_report() to export statistics + Step 9: Test the pipeline with at least two different public APIs +""" \ No newline at end of file From bb0adc8c08c1252eab966509ec374bee4a7ae863 Mon Sep 17 00:00:00 2001 From: Harshan07-web Date: Sun, 17 May 2026 08:56:56 +0530 Subject: [PATCH 2/2] data : add starter code for api etl pipeline --- data/projects.json | 16 +- starter_code/api_data_pipeline.py | 279 +++++++++++++++++++++++++++++- 2 files changed, 286 insertions(+), 9 deletions(-) diff --git a/data/projects.json b/data/projects.json index 63bbcd94..452f6c0e 100644 --- a/data/projects.json +++ b/data/projects.json @@ -250,31 +250,31 @@ }, { "id": 9, - "title": "API ETL pipeline", - "skills": ["Python","pandas","requests","os module"], + "title": "API ETL Pipeline", + "skills": ["Python","pandas","requests"], "level": "Intermediate", "interest": "Data", "time": "Medium", - "description": "Upload a API url, which then fetches data from the API, and automatically transform it into a CSV file.", + "description": "Enter a public API URL to fetch data and automatically transform it into a structured CSV dataset.", "features": [ - "Fetch requests from API", + "Fetch data from public APIs", "handle missing values", "Normalize nested JSON", "Generate summary statistics", "Export the processed CSV for any other Analytics projects" ], - "tech_stack": ["Python", "pandas","requests","JSON","os module"], + "tech_stack": ["Python", "pandas","requests","JSON"], "roadmap": [ "Step 1: Install required modules via pip", "Step 2: Find a public API key for this project", "Step 3: Fetch the data from the API using requests", - "Step 4: Validate the reponse you just fetched From the API", + "Step 4: Validate the response you just fetched From the API", "Step 5: Normalize the nested JSON data by flattening it", "Step 6: Use the fetched data to build a pandas dataframe", "Step 7: Handle missing values or duplicate values", - "Step 8: Export the cleaned datatable to a CSV format", + "Step 8: Export the cleaned dataset to CSV format", "Step 9: Generate a summary for the newly created CSV dataset", - "Step 10: Test the file with atleast two different public API keys" + "Step 10: Test the file with at least two different public APIs" ], "resources": [ "pandas docs: https://pandas.pydata.org/docs", diff --git a/starter_code/api_data_pipeline.py b/starter_code/api_data_pipeline.py index 38f28527..3ea5a75d 100644 --- a/starter_code/api_data_pipeline.py +++ b/starter_code/api_data_pipeline.py @@ -43,4 +43,281 @@ Step 7: Complete export_csv() to save the processed dataset Step 8: Complete generate_summary_report() to export statistics Step 9: Test the pipeline with at least two different public APIs -""" \ No newline at end of file +""" +import os +import sys + +try: + import pandas as pd + import requests +except ImportError: + print("Required packages not installed.") + print("Run: pip install pandas requests") + sys.exit(1) + + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +OUTPUT_DIR = "output" + + +# --------------------------------------------------------------------------- +# Setup helpers — already complete +# --------------------------------------------------------------------------- + +def ensure_output_directory(): + """Create output directory if it does not already exist.""" + os.makedirs(OUTPUT_DIR, exist_ok=True) + + +# --------------------------------------------------------------------------- +# Core functions — complete the TODOs below +# --------------------------------------------------------------------------- + +def fetch_api_data(url): + """ + Fetch JSON data from a public API endpoint. + + Args: + url (str): The API endpoint URL. + + Returns: + dict | list | None: + Parsed JSON response if successful, otherwise None. + + TODO: + 1. Validate that the URL starts with http:// or https:// + Example APIs: + https://jsonplaceholder.typicode.com/users + https://dummyjson.com/products + https://dummyjson.com/users + 2. Send a GET request using: + requests.get(url, timeout=10) + 3. Use response.raise_for_status() to detect HTTP errors + such as 404 or 500 status codes. + 4. Convert the API response into JSON format using: + response.json() + 5. Handle request exceptions using try/except blocks. + 6. Print a success message showing that the data was fetched. + 7. Return the parsed JSON response. + """ + # --- Write your code here --- + + return None + + +def validate_response(data): + """ + Validate the API response before processing. + + Args: + data (dict | list): The parsed API response. + + Returns: + bool: + True if the response is valid, otherwise False. + + TODO: + 1. Check whether the response is empty. + 2. Ensure the response is either: + - a dictionary + - or a list + 3. Print a helpful validation message. + 4. Return True if the response is valid. + Otherwise return False. + """ + # --- Write your code here --- + + return False + + +def normalize_json(data): + """ + Normalize and flatten nested JSON data. + + Args: + data (dict | list): Raw JSON response. + + Returns: + pd.DataFrame: + Flattened DataFrame generated from JSON data. + + TODO: + 1. Use pd.json_normalize(data) to flatten nested JSON fields. + 2. Example: + { + "name": "John", + "address": { + "city": "London" + } + } + becomes: + name | address.city + 3. Handle both dictionary and list responses. + 4. Print the resulting column names. + 5. Return the normalized DataFrame. + """ + # --- Write your code here --- + + return pd.DataFrame() + + +def create_dataframe(data): + """ + Convert normalized JSON data into a pandas DataFrame. + + Args: + data (dict | list): API response data. + + Returns: + pd.DataFrame: + Constructed pandas DataFrame. + + TODO: + 1. Create a DataFrame using pandas. + 2. Example: + df = pd.DataFrame(data) + 3. Print dataset shape information using: + df.shape + 4. Display the first few rows using: + df.head() + 5. Return the DataFrame. + """ + # --- Write your code here --- + + return pd.DataFrame() + + +def clean_data(df): + """ + Clean the dataset by handling missing values and duplicates. + + Args: + df (pd.DataFrame): Raw DataFrame. + + Returns: + pd.DataFrame: + Cleaned DataFrame. + + TODO: + 1. Identify missing values using: + df.isnull().sum() + 2. Remove duplicate rows using: + df.drop_duplicates() + 3. Handle missing values using: + df.fillna() + 4. Standardize column names if necessary. + 5. Print dataset shape after cleaning. + 6. Return the cleaned DataFrame. + """ + # --- Write your code here --- + + return df + + +def export_csv(df, filename="processed_data.csv"): + """ + Export the processed dataset to a CSV file. + + Args: + df (pd.DataFrame): Cleaned dataset. + filename (str): Output CSV filename. + + Returns: + str: + Filepath of the exported CSV file. + + TODO: + 1. Build the full output path using: + os.path.join() + 2. Export the dataset using: + df.to_csv(output_path, index=False) + 3. Use index=False to prevent pandas from adding row numbers. + 4. Print a confirmation message after export. + 5. Return the output filepath. + """ + # --- Write your code here --- + + return "" + + +def generate_summary_report(df): + """ + Generate a summary report for the dataset. + + Args: + df (pd.DataFrame): Cleaned dataset. + + TODO: + 1. Print dataset shape: + df.shape + 2. Display column names and data types: + df.dtypes + 3. Show missing value counts: + df.isnull().sum() + 4. Generate descriptive statistics: + df.describe() + 5. Display: + - total rows + - total columns + - column names + - missing value counts + - numeric statistics + 6. Print the summary report clearly. + """ + # --- Write your code here --- + + pass + + +# ----------------------------------------------------------------------------------- +# Main pipeline — already complete and ready to run after completing core functions +# ----------------------------------------------------------------------------------- + +def run_pipeline(api_url): + """ + Run the complete API ETL pipeline. + + Args: + api_url (str): Public API endpoint URL. + """ + ensure_output_directory() + + data = fetch_api_data(api_url) #fetch the data from the given url + if not validate_response(data): #validate the response + return + + normalized_df = normalize_json(data) # normalize nested json responses + + df = create_dataframe(normalized_df) #create a new dataframe with the normalized json + + cleaned_df = clean_data(df) #clean the created data table + + export_path = export_csv(cleaned_df) #export it to csv + + print(f"\nCSV exported successfully: {export_path}") + + generate_summary_report(cleaned_df) #generate summary for the exported csv + + +def main(): + """main function to test the pipeline if run directly""" + print("\n" + "=" * 50) + print("DevPath — API ETL Pipeline") + print("=" * 50) + + api_url = input("\nEnter a public API URL: ").strip() + + if not api_url: + print("No API URL provided. Exiting.") + return + + run_pipeline(api_url) + + print("\nPipeline execution complete.\n") + + +if __name__ == "__main__": + main() \ No newline at end of file