hackforla · codemamma · Nov 30, 2021 · Apr 14, 2022 · May 4, 2022 · Jun 27, 2022
diff --git a/311-data/CSV_files/DataLoading_Script.ipynb b/311-data/CSV_files/DataLoading_Script.ipynb
diff --git a/311-data/CSV_files/Data_csvfiles b/311-data/CSV_files/Data_csvfiles
@@ -0,0 +1 @@
+
diff --git a/311-data/CSV_files/Docs/CleaningRules.txt b/311-data/CSV_files/Docs/CleaningRules.txt
diff --git a/311-data/webscraping/scrape_more_info.py b/311-data/webscraping/scrape_more_info.py
@@ -0,0 +1,62 @@
+"""Scrapes extra info for desired tech categories
+
+Takes tech_table produced by Rajinders' scrape.py script
+
+Extra info includes: url of tech found on builtwith, text description of tech, url for tech website, subcategories listed on builtwith,
+number of live sites that use technology, list of top 5 competitors of tech
+"""
+
+import requests
+from bs4 import BeautifulSoup, element
+import pandas as pd
+import numpy as np
+
+def get_extra_info(url_list):
+    columns = ['builtwith_tech_url', 'tech_description', 'tech_website', 'subcategories','num_live_sites','competitors']
+    extra_info = pd.DataFrame(columns=columns)
+
+    for url in url_list:
+        print(url)
+        res = requests.get(url)
+        soup = BeautifulSoup(res.content, "html.parser")
+
+        #first card search for tech description, tech website, subcategories
+        div = soup.findAll("div", {"class": "col-9 col-md-10"})
+
+        ls = [url]
+        for d in div:
+            info = d.findAll("p")
+            for i in info:
+                ls.append(i.text)
+
+        #second search, for top competitors
+        div2 = soup.find("div", {"class": "list-group small"})
+        comp = div2.findAll("a", href=True)
+        links = []
+
+        try:
+            for i in range(5): #get top 5 competitors
+                if 'trends' in str(comp[i]): #some sites don't have competitors listed
+                        links.append(comp[i]["href"][2:])
+                else:
+                    continue
+        except:
+            continue
+
+        try:
+            ls.append(soup.find("dd", {"class": "col-6"}).text) #get number of live websites
+        except:
+            ls.append(np.NaN)#some urls don't have this info yet
+
+        ls.append(links)
+        extra_info.loc[len(extra_info)] = ls
+
+    return extra_info
+
+def main():
+    tech = pd.read_csv('tech_table.csv')
+    more_info = tech[tech['category'].isin(['widgets', 'analytics', 'cms','copyright','framework','link','mobile','payment','ssl','widgets'])] #choose which tech categories to collect more data
+    urls = list(more_info['URL of tech'])
+    info = get_extra_info(urls)
+    info['subcategories'] = info['subcategories'].str.replace(' · ', ', ') #formatting for subcategories column
+    info.to_csv('techtable_extrainfo.csv', index=False)
diff --git a/...idents_vs_crosswalks_-_QGIS_-_7:25:21.png → ...idents_vs_crosswalks_-_QGIS_-_7-25-21.png b/...idents_vs_crosswalks_-_QGIS_-_7:25:21.png → ...idents_vs_crosswalks_-_QGIS_-_7-25-21.png
diff --git a/README.md b/README.md
@@ -7,18 +7,16 @@ Welcome to the Data Science Community of Practice. We are happy you are here!
 If you have not read the [Guide for New Volunteers](https://www.hackforla.org/getting-started), please do so.  
 
 1. Join the [#data-science](https://hackforla.slack.com/archives/CGRATJCCF) Slack channel and introduce yourself.
-1. Slack one of our Data Science Community of Practice leads [Ryan](https://hackforla.slack.com/team/UPB2FHJCX) or [Sophia](https://hackforla.slack.com/team/UN7V7L934) with your Gmail address.
-1. Accept your Google Drive invite to access the [shared folder](https://drive.google.com/drive/u/0/folders/17VuPq--bK2RvBiAG87C0Vo1oM7nluuS7).
-1. Add yourself to the [Team Roster](https://docs.google.com/spreadsheets/d/1QJltNh1gOybfebe-RkT-xS7m4OtxbuFfaJ4OujeA4h0/edit) and inform Ryan or Sophia after you have done so.
-1. Join our meetings Thursday at 8 pm PST via [Zoom](https://us02web.zoom.us/j/81067015817?pwd=M3l6a0tQTWhLbnlTbEZNOWJ5UXN3QT09).
-1. Check out the [open Data Science roles](https://github.com/hackforla/data-science/projects/2) we have available.
+1. Come to one of the Data Science Community of Practice meetings and provide your gmail address to on of the leads so that they can add you to the drive. 
+1. After you have been added to the drive, Add yourself to the [Team Roster] and the leads know you have added yourself
+1. Join our meetings Monday at 7- 8 pm PST via [Zoom] link in the Slack channel. We do not meet during holidays or on the first Monday of the month.
 
 The Data Science Community of Practice is one of many.  [See all our Communities of Practices](https://github.com/hackforla/communities-of-practice/blob/main/README.md)
 
 
 ## Focus
 
-The Hack For LA Data Science team is a group within the LA brigade seeking to make analytical and machine learning services available to local communities and organizations. We have three main goals as a team:
+The Hack For LA Data Science team is a group within Hack for LA seeking to make analytical and machine learning services available to local communities and organizations. We have three main goals as a team:
 
 1. To provide data science services to communities in the Los Angeles area. Please [contact us](mailto:[email protected]) if you have a proposal for a data science project. We have many talented data scientists with track records in industry and academia, and we are excited to be able to support our communities.