diff --git a/Build and Optimize Data Warehouses with BigQuery: Challenge Lab b/Build and Optimize Data Warehouses with BigQuery: Challenge Lab new file mode 100644 index 0000000..839876a --- /dev/null +++ b/Build and Optimize Data Warehouses with BigQuery: Challenge Lab @@ -0,0 +1,101 @@ + +CREATE OR REPLACE TABLE . +PARTITION BY date +OPTIONS( +partition_expiration_days=90, +description="oxford_policy_tracker table in the COVID 19 Government Response public dataset with an expiry time set to 90 days." +) AS +SELECT + * +FROM + `bigquery-public-data.covid19_govt_response.oxford_policy_tracker` +WHERE + alpha_3_code NOT IN ('GBR', 'USA') + + + + +ALTER TABLE . +ADD COLUMN population INT64, +ADD COLUMN country_area FLOAT64, +ADD COLUMN mobility STRUCT< + avg_retail FLOAT64, + avg_grocery FLOAT64, + avg_parks FLOAT64, + avg_transit FLOAT64, + avg_workplace FLOAT64, + avg_residential FLOAT64 + > + + +UPDATE + `.` t0 +SET + population = t1.population +FROM + `bigquery-public-data.covid19_ecdc.covid_19_geographic_distribution_worldwide` t1 +WHERE + CONCAT(t0.alpha_3_code, t0.date) = CONCAT(t1.country_territory_code, t1.date); + + + +UPDATE + `.` t0 +SET + t0.country_area = t1.country_area +FROM + `bigquery-public-data.census_bureau_international.country_names_area` t1 +WHERE + t0.country_name = t1.country_name + + + +UPDATE + `.` t0 +SET + t0.mobility.avg_retail = t1.avg_retail + t0.mobility.avg_grocery = t1.avg_grocery + t0.mobility.avg_parks = t1.avg_parks + t0.mobility.avg_transit = t1.avg_transit + t0.mobility.avg_workplace = t1.avg_workplace + t0.mobility.avg_residential = t1.avg_residential +FROM + ( SELECT country_region, date, + AVG(retail_and_recreation_percent_change_from_baseline) as avg_retail, + AVG(grocery_and_pharmacy_percent_change_from_baseline) as avg_grocery, + AVG(parks_percent_change_from_baseline) as avg_parks, + AVG(transit_stations_percent_change_from_baseline) as avg_transit, + AVG(workplaces_percent_change_from_baseline) as avg_workplace, + AVG(residential_percent_change_from_baseline) as avg_residential + FROM `bigquery-public-data.covid19_google_mobility.mobility_report` + GROUP BY country_region, date + ) AS t1 +WHERE + CONCAT(t0.country_name, t0.date) = CONCAT(t1.country_region, t1.date) + + + + +SELECT country_name, population +FROM `.` +WHERE population is NULL + + + +SELECT country_name, country_area +FROM `.` +WHERE WHERE country_area IS NULL + + +SELECT DISTINCT country_name +FROM `.` +WHERE population is NULL +UNION ALL +SELECT DISTINCT country_name +FROM `.` +WHERE WHERE country_area IS NULL +ORDER BY country_name ASC + + + + diff --git a/Build and Secure Networks in Google Cloud: Challenge Lab b/Build and Secure Networks in Google Cloud: Challenge Lab index 552a751..81fd7c2 100644 --- a/Build and Secure Networks in Google Cloud: Challenge Lab +++ b/Build and Secure Networks in Google Cloud: Challenge Lab @@ -1,24 +1,24 @@ #WATCH FULL LAB ON : https://www.youtube.com/watch?v=22TczCBXyys -gcloud compute firewall-rules delete open-access +Step 1: gcloud compute firewall-rules delete open-access -gcloud compute instances start bastion +step 2: gcloud compute instances start bastion -gcloud compute firewall-rules create ssh-ingress --allow=tcp:22 --source-ranges 35.235.240.0/20 --target-tags ssh-ingress --network acme-vpc +step 3: gcloud compute firewall-rules create ssh-ingress --allow=tcp:22 --source-ranges 35.235.240.0/20 --target-tags ssh-ingress --network acme-vpc gcloud compute instances add-tags bastion --tags=ssh-ingress --zone=us-central1-b -gcloud compute firewall-rules create http-ingress --allow=tcp:80 --source-ranges 0.0.0.0/0 --target-tags http-ingress --network acme-vpc +step 4: gcloud compute firewall-rules create http-ingress --allow=tcp:80 --source-ranges 0.0.0.0/0 --target-tags http-ingress --network acme-vpc -gcloud compute instances add-tags juice-shop --tags=http-ingress --zone=us-central1-b +step 5: gcloud compute instances add-tags juice-shop --tags=http-ingress --zone=us-central1-b -gcloud compute firewall-rules create internal-ssh-ingress --allow=tcp:22 --source-ranges 192.168.10.0/24 --target-tags internal-ssh-ingress --network acme-vpc +step 6: gcloud compute firewall-rules create internal-ssh-ingress --allow=tcp:22 --source-ranges 192.168.10.0/24 --target-tags internal-ssh-ingress --network acme-vpc -gcloud compute instances add-tags juice-shop --tags=internal-ssh-ingress --zone=us-central1-b +step 7 :gcloud compute instances add-tags juice-shop --tags=internal-ssh-ingress --zone=us-central1-b diff --git a/Create ML Models with BigQuery ML: Challenge Lab b/Create ML Models with BigQuery ML: Challenge Lab new file mode 100644 index 0000000..0d606c1 --- /dev/null +++ b/Create ML Models with BigQuery ML: Challenge Lab @@ -0,0 +1,150 @@ +Complete each of the sets of steps below to prepare the lab for the activity tracking assessment for each task in the lab. + +Task 1: Create a dataset to store your machine learning models +It can be called any name to pass the test but for the remaining instructions to work use `austin` as the dataset name. + +bq mk austin + + + +Task 2: Create a forecasting BigQuery machine learning model. +Create the first ML model using a JOIN between two bike share tables. Again any names will work but keep them as ‘austin_1’ and ‘austin_2’ for the remaining instructions to work. + +BigQuery Console Query Editor + +CREATE OR REPLACE MODEL austin.location_model +OPTIONS + (model_type='linear_reg', labels=['duration_minutes']) AS +SELECT + start_station_name, + EXTRACT(HOUR FROM start_time) AS start_hour, + EXTRACT(DAYOFWEEK FROM start_time) AS day_of_week, + duration_minutes, + address as location +FROM + `bigquery-public-data.austin_bikeshare.bikeshare_trips` AS trips +JOIN + `bigquery-public-data.austin_bikeshare.bikeshare_stations` AS stations +ON + trips.start_station_name = stations.name +WHERE + EXTRACT(YEAR FROM start_time) = 2018 + AND duration_minutes > 0 + + + + + + + +Task 3: Create the second machine learning model. + +BigQuery Console Query Editor +CREATE OR REPLACE MODEL austin.subscriber_model +OPTIONS + (model_type='linear_reg', labels=['duration_minutes']) AS +SELECT + start_station_name, + EXTRACT(HOUR FROM start_time) AS start_hour, + subscriber_type, + duration_minutes +FROM `bigquery-public-data.austin_bikeshare.bikeshare_trips` AS trips +WHERE EXTRACT(YEAR FROM start_time) = 2018 + + + + + +Task 4: Evaluate the two machine learning models. + +BigQuery Console Query Editor +Query 1 + +-- Evaluation metrics for location_model +SELECT + SQRT(mean_squared_error) AS rmse, + mean_absolute_error +FROM + ML.EVALUATE(MODEL austin.location_model, ( + SELECT + start_station_name, + EXTRACT(HOUR FROM start_time) AS start_hour, + EXTRACT(DAYOFWEEK FROM start_time) AS day_of_week, + duration_minutes, + address as location + FROM + `bigquery-public-data.austin_bikeshare.bikeshare_trips` AS trips + JOIN + `bigquery-public-data.austin_bikeshare.bikeshare_stations` AS stations + ON + trips.start_station_name = stations.name + WHERE EXTRACT(YEAR FROM start_time) = 2019) +) + + + + +Query 2 +-- Evaluation metrics for subscriber_model +SELECT + SQRT(mean_squared_error) AS rmse, + mean_absolute_error +FROM + ML.EVALUATE(MODEL austin.subscriber_model, ( + SELECT + start_station_name, + EXTRACT(HOUR FROM start_time) AS start_hour, + subscriber_type, + duration_minutes + FROM + `bigquery-public-data.austin_bikeshare.bikeshare_trips` AS trips + WHERE + EXTRACT(YEAR FROM start_time) = 2019) +) + + + + + + + + +Task 5: Use the subscriber type machine learning model to predict average trip durations +Use the second model, that model (model austin_2 in this case) to to predict the average duration length of all trips from the busiest rental station in 2019 (based on the number of rentals per station in 2019) where the subscriber_type=’Single Trip’. + + +The following query will list busiest stations in descending order. The busiest station for 2019 was “21st & Speedway @PCL”. + +BigQuery Console Query Editor +SELECT + start_station_name, + COUNT(*) AS trips +FROM + `bigquery-public-data.austin_bikeshare.bikeshare_trips` +WHERE + EXTRACT(YEAR FROM start_time) = 2019 +GROUP BY + start_station_name +ORDER BY + trips DESC + + + + + +Then predict trip length. + +BigQuery Console Query Editor +SELECT AVG(predicted_duration_minutes) AS average_predicted_trip_length +FROM ML.predict(MODEL austin.subscriber_model, ( +SELECT + start_station_name, + EXTRACT(HOUR FROM start_time) AS start_hour, + subscriber_type, + duration_minutes +FROM + `bigquery-public-data.austin_bikeshare.bikeshare_trips` +WHERE + EXTRACT(YEAR FROM start_time) = 2019 + AND subscriber_type = 'Single Trip' + AND start_station_name = '21st & Speedway @PCL')) diff --git a/Ensure Access & Identity in Google Cloud: Challenge Lab b/Ensure Access & Identity in Google Cloud: Challenge Lab new file mode 100644 index 0000000..9fae9e0 --- /dev/null +++ b/Ensure Access & Identity in Google Cloud: Challenge Lab @@ -0,0 +1,53 @@ + +gcloud config set compute/zone us-east1-b + +nano role-definition.yaml + +title: "Edirca Storage Update" +description: "Add and update objects in Google Cloud Storage buckets" +includedPermissions: +- storage.buckets.get +- storage.objects.get +- storage.objects.list +- storage.objects.update +- storage.objects.create + + +gcloud iam roles create orca_storage_update \ + --project $DEVSHELL_PROJECT_ID \ + --file role-definition.yaml + + +gcloud iam service-accounts create orca-private-cluster-sa \ + --display-name "Orca Private Cluster Service Account" + +gcloud projects add-iam-policy-binding $DEVSHELL_PROJECT_ID \ + --member serviceAccount:orca-private-cluster-sa@$DEVSHELL_PROJECT_ID.iam.gserviceaccount.com --role roles/monitoring.viewer + +gcloud projects add-iam-policy-binding $DEVSHELL_PROJECT_ID \ + --member serviceAccount:orca-private-cluster-sa@$DEVSHELL_PROJECT_ID.iam.gserviceaccount.com --role roles/monitoring.metricWriter + +gcloud projects add-iam-policy-binding $DEVSHELL_PROJECT_ID \ + --member serviceAccount:orca-private-cluster-sa@$DEVSHELL_PROJECT_ID.iam.gserviceaccount.com --role roles/logging.logWriter + + + + + + + +gcloud projects add-iam-policy-binding $DEVSHELL_PROJECT_ID \ + --member serviceAccount:orca-private-cluster-sa@$DEVSHELL_PROJECT_ID.iam.gserviceaccount.com --role projects/$DEVSHELL_PROJECT_ID/roles/orca_storage_update + + + + + + + +gcloud container clusters create orca-test-cluster --network orca-build-vpc --subnetwork orca-build-subnet --service-account orca-private-cluster-sa@qwiklabs-gcp-01-73bc421e624d.iam.gserviceaccount.com --enable-master-authorized-networks --master-authorized-networks 192.168.10.2/32 --enable-ip-alias --enable-private-nodes --master-ipv4-cidr 10.142.0.0/28 --enable-private-endpoint + + + +gcloud container clusters get-credentials orca-test-cluster --internal-ip --zone=us-east1-b +kubectl create deployment hello-server --image=gcr.io/google-samples/hello-app:1.0 diff --git a/Exploring the Public Cryptocurrency Datasets Available in BigQuery b/Exploring the Public Cryptocurrency Datasets Available in BigQuery index dc86e04..ff8ba23 100644 --- a/Exploring the Public Cryptocurrency Datasets Available in BigQuery +++ b/Exploring the Public Cryptocurrency Datasets Available in BigQuery @@ -5,7 +5,6 @@ where outputs.output_satoshis = 19499300000000 --- SQL source from https://cloud.google.com/blog/product... CREATE OR REPLACE TABLE lab.52 (balance NUMERIC) as WITH double_entry_book AS ( -- debits @@ -19,7 +18,6 @@ WITH double_entry_book AS ( array_to_string(outputs.addresses, ",") as address , outputs.value as value FROM `bigquery-public-data.crypto_bitcoin.outputs` as outputs - ) SELECT sum(value) as balance diff --git a/Getting Started: Create and Manage Cloud Resources: Challenge b/Getting Started: Create and Manage Cloud Resources: Challenge index 436611e..5ecf392 100644 --- a/Getting Started: Create and Manage Cloud Resources: Challenge +++ b/Getting Started: Create and Manage Cloud Resources: Challenge @@ -25,7 +25,7 @@ --port 8080 -Step 3:kubect1 get pods +Step 3:kubectl get pods Step 4:cat << EOF > startup.sh #! /bin/bash diff --git a/Insights from Data with BigQuery: Challenge Lab b/Insights from Data with BigQuery: Challenge Lab(Updated task 9) similarity index 100% rename from Insights from Data with BigQuery: Challenge Lab rename to Insights from Data with BigQuery: Challenge Lab(Updated task 9) diff --git a/analyze-images.py b/analyze-images.py index b222571..2fbbc6c 100644 --- a/analyze-images.py +++ b/analyze-images.py @@ -52,7 +52,7 @@ file_content = file.download_as_string() # TBD: Create a Vision API image object called image_object - image_object = vision.types.Image(content=file_content) + image_object = vision.Image(content=file_content) # Ref: https://googleapis.dev/python/vision/latest/gapic/v1/types.html#google.cloud.vision_v1.types.Image