Skip to content

Commit f112e5e

Browse files
naeemaa00naeemaa00
committed
updatd chaneg
1 parent a60ad28 commit f112e5e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+4565
-0
lines changed

‎AWS_ETL_S3_to_Glue_RDSAuroraDB/MySQLSQL.py

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
import logging
2+
import os
3+
import boto3
4+
import csv
5+
import pymysql
6+
from io import StringIO
7+
8+
# Set up logging
9+
logger = logging.getLogger()
10+
logger.setLevel(logging.INFO)
11+
12+
# Initialize the S3 client
13+
s3_client = boto3.client('s3')
14+
15+
def lambda_handler(event, context):
16+
# Retrieve database connection parameters from environment variables
17+
host = os.getenv('DB_HOST')
18+
dbname = os.getenv('DB_NAME')
19+
user = os.getenv('DB_USER')
20+
password = os.getenv('DB_PASSWORD')
21+
port = int(os.getenv('DB_PORT')) # Default MySQL port is 3306
22+
23+
# Get the S3 bucket name and file name from the event
24+
bucket_name = event['Records'][0]['s3']['bucket']['name']
25+
file_key = event['Records'][0]['s3']['object']['key']
26+
27+
# Download the CSV file from S3
28+
try:
29+
response = s3_client.get_object(Bucket=bucket_name, Key=file_key)
30+
csv_content = response['Body'].read().decode('utf-8')
31+
logger.info(f"CSV file {file_key} successfully fetched from S3.")
32+
except Exception as e:
33+
logger.error(f"Error downloading file from S3: {e}")
34+
return {'statusCode': 500, 'body': f"Error downloading file from S3: {e}"}
35+
36+
# Parse CSV file content
37+
try:
38+
csv_file = StringIO(csv_content)
39+
csv_reader = csv.DictReader(csv_file)
40+
rows = [row for row in csv_reader]
41+
logger.info(f"CSV file parsed successfully. {len(rows)} records found.")
42+
except Exception as e:
43+
logger.error(f"Error parsing CSV file: {e}")
44+
return {'statusCode': 500, 'body': f"Error parsing CSV file: {e}"}
45+
46+
# Establish the connection to the MySQL database
47+
try:
48+
conn = pymysql.connect(
49+
host=host,
50+
user=user,
51+
password=password,
52+
database=dbname,
53+
port=port,
54+
cursorclass=pymysql.cursors.DictCursor
55+
)
56+
logger.info("Connection to database successful!")
57+
except pymysql.MySQLError as e:
58+
logger.error(f"Error connecting to database: {e}")
59+
return {'statusCode': 500, 'body': f"Error connecting to database: {e}"}
60+
61+
# Create a cursor object to interact with the database
62+
try:
63+
with conn.cursor() as cursor:
64+
# Step 1: Create table if it doesn't exist
65+
create_table_query = '''
66+
CREATE TABLE IF NOT EXISTS demo.employees (
67+
id INT AUTO_INCREMENT PRIMARY KEY,
68+
first_name VARCHAR(50),
69+
last_name VARCHAR(50),
70+
email VARCHAR(100),
71+
hire_date DATE,
72+
salary DECIMAL(10,2)
73+
);
74+
'''
75+
cursor.execute(create_table_query)
76+
conn.commit()
77+
logger.info("Table created successfully!")
78+
79+
# Step 2: Insert records from CSV into the table
80+
insert_record_query = '''
81+
INSERT INTO demo.employees (first_name, last_name, email, hire_date, salary)
82+
VALUES (%s, %s, %s, %s, %s);
83+
'''
84+
for row in rows:
85+
cursor.execute(insert_record_query, (row['first_name'], row['last_name'], row['email'], row['hire_date'], row['salary']))
86+
conn.commit()
87+
logger.info(f"{len(rows)} records inserted successfully!")
88+
89+
# Step 3: Fetch and print all data from the table
90+
select_query = 'SELECT * FROM demo.employees;'
91+
cursor.execute(select_query)
92+
all_records = cursor.fetchall()
93+
logger.info("Data from demo.employees table:")
94+
for record in all_records:
95+
logger.info(f"Row: {record}")
96+
97+
except Exception as e:
98+
logger.error(f"Error during database operations: {e}")
99+
conn.rollback()
100+
return {'statusCode': 500, 'body': f"Error during database operations: {e}"}
101+
finally:
102+
if conn:
103+
conn.close()
104+
logger.info("Database connection closed.")
105+
106+
return {'statusCode': 200, 'body': 'Database operations completed successfully'}

‎AWS_ETL_S3_to_Glue_RDSAuroraDB/mysqlquery.sql

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
CREATE TABLE demo.employees (
2+
id INT AUTO_INCREMENT PRIMARY KEY,
3+
first_name VARCHAR(50) NOT NULL,
4+
last_name VARCHAR(50) NOT NULL,
5+
email VARCHAR(100) NOT NULL UNIQUE,
6+
hire_date DATE NOT NULL,
7+
salary DECIMAL(10, 2) NOT NULL
8+
);
9+
10+
11+
INSERT INTO demo.employees (first_name, last_name, email, hire_date, salary) VALUES
12+
('John', 'Doe', '[email protected]', '2023-01-15', 60000.00),
13+
('Jane', 'Smith', '[email protected]', '2023-02-20', 75000.00),
14+
('Alice', 'Johnson', '[email protected]', '2023-03-05', 80000.00);

‎AWS_ETL_S3_to_Glue_RDSAuroraDB/python.zip

107 KB
Binary file not shown.

‎AWS_ETL_S3_to_Glue_RDSAuroraDB/python/PyMySQL-1.1.1.dist-info/INSTALLER

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
pip

‎AWS_ETL_S3_to_Glue_RDSAuroraDB/python/PyMySQL-1.1.1.dist-info/LICENSE

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
Copyright (c) 2010, 2013 PyMySQL contributors
2+
3+
Permission is hereby granted, free of charge, to any person obtaining a copy
4+
of this software and associated documentation files (the "Software"), to deal
5+
in the Software without restriction, including without limitation the rights
6+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7+
copies of the Software, and to permit persons to whom the Software is
8+
furnished to do so, subject to the following conditions:
9+
10+
The above copyright notice and this permission notice shall be included in
11+
all copies or substantial portions of the Software.
12+
13+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19+
THE SOFTWARE.

‎AWS_ETL_S3_to_Glue_RDSAuroraDB/python/PyMySQL-1.1.1.dist-info/METADATA

+135
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
Metadata-Version: 2.1
2+
Name: PyMySQL
3+
Version: 1.1.1
4+
Summary: Pure Python MySQL Driver
5+
Author-email: Inada Naoki <[email protected]>, Yutaka Matsubara <[email protected]>
6+
License: MIT License
7+
Project-URL: Project, https://github.com/PyMySQL/PyMySQL
8+
Project-URL: Documentation, https://pymysql.readthedocs.io/
9+
Keywords: MySQL
10+
Classifier: Development Status :: 5 - Production/Stable
11+
Classifier: Programming Language :: Python :: 3
12+
Classifier: Programming Language :: Python :: 3.7
13+
Classifier: Programming Language :: Python :: 3.8
14+
Classifier: Programming Language :: Python :: 3.9
15+
Classifier: Programming Language :: Python :: 3.10
16+
Classifier: Programming Language :: Python :: 3.11
17+
Classifier: Programming Language :: Python :: 3.12
18+
Classifier: Programming Language :: Python :: Implementation :: CPython
19+
Classifier: Programming Language :: Python :: Implementation :: PyPy
20+
Classifier: Intended Audience :: Developers
21+
Classifier: License :: OSI Approved :: MIT License
22+
Classifier: Topic :: Database
23+
Requires-Python: >=3.7
24+
Description-Content-Type: text/markdown
25+
License-File: LICENSE
26+
Provides-Extra: ed25519
27+
Requires-Dist: PyNaCl >=1.4.0 ; extra == 'ed25519'
28+
Provides-Extra: rsa
29+
Requires-Dist: cryptography ; extra == 'rsa'
30+
31+
[![Documentation Status](https://readthedocs.org/projects/pymysql/badge/?version=latest)](https://pymysql.readthedocs.io/)
32+
[![codecov](https://codecov.io/gh/PyMySQL/PyMySQL/branch/main/graph/badge.svg?token=ppEuaNXBW4)](https://codecov.io/gh/PyMySQL/PyMySQL)
33+
34+
# PyMySQL
35+
36+
This package contains a pure-Python MySQL client library, based on [PEP
37+
249](https://www.python.org/dev/peps/pep-0249/).
38+
39+
## Requirements
40+
41+
- Python -- one of the following:
42+
- [CPython](https://www.python.org/) : 3.7 and newer
43+
- [PyPy](https://pypy.org/) : Latest 3.x version
44+
- MySQL Server -- one of the following:
45+
- [MySQL](https://www.mysql.com/) \>= 5.7
46+
- [MariaDB](https://mariadb.org/) \>= 10.4
47+
48+
## Installation
49+
50+
Package is uploaded on [PyPI](https://pypi.org/project/PyMySQL).
51+
52+
You can install it with pip:
53+
54+
$ python3 -m pip install PyMySQL
55+
56+
To use "sha256_password" or "caching_sha2_password" for authenticate,
57+
you need to install additional dependency:
58+
59+
$ python3 -m pip install PyMySQL[rsa]
60+
61+
To use MariaDB's "ed25519" authentication method, you need to install
62+
additional dependency:
63+
64+
$ python3 -m pip install PyMySQL[ed25519]
65+
66+
## Documentation
67+
68+
Documentation is available online: <https://pymysql.readthedocs.io/>
69+
70+
For support, please refer to the
71+
[StackOverflow](https://stackoverflow.com/questions/tagged/pymysql).
72+
73+
## Example
74+
75+
The following examples make use of a simple table
76+
77+
``` sql
78+
CREATE TABLE `users` (
79+
`id` int(11) NOT NULL AUTO_INCREMENT,
80+
`email` varchar(255) COLLATE utf8_bin NOT NULL,
81+
`password` varchar(255) COLLATE utf8_bin NOT NULL,
82+
PRIMARY KEY (`id`)
83+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin
84+
AUTO_INCREMENT=1 ;
85+
```
86+
87+
``` python
88+
import pymysql.cursors
89+
90+
# Connect to the database
91+
connection = pymysql.connect(host='localhost',
92+
user='user',
93+
password='passwd',
94+
database='db',
95+
cursorclass=pymysql.cursors.DictCursor)
96+
97+
with connection:
98+
with connection.cursor() as cursor:
99+
# Create a new record
100+
sql = "INSERT INTO `users` (`email`, `password`) VALUES (%s, %s)"
101+
cursor.execute(sql, ('[email protected]', 'very-secret'))
102+
103+
# connection is not autocommit by default. So you must commit to save
104+
# your changes.
105+
connection.commit()
106+
107+
with connection.cursor() as cursor:
108+
# Read a single record
109+
sql = "SELECT `id`, `password` FROM `users` WHERE `email`=%s"
110+
cursor.execute(sql, ('[email protected]',))
111+
result = cursor.fetchone()
112+
print(result)
113+
```
114+
115+
This example will print:
116+
117+
``` python
118+
{'password': 'very-secret', 'id': 1}
119+
```
120+
121+
## Resources
122+
123+
- DB-API 2.0: <https://www.python.org/dev/peps/pep-0249/>
124+
- MySQL Reference Manuals: <https://dev.mysql.com/doc/>
125+
- MySQL client/server protocol:
126+
<https://dev.mysql.com/doc/internals/en/client-server-protocol.html>
127+
- "Connector" channel in MySQL Community Slack:
128+
<https://lefred.be/mysql-community-on-slack/>
129+
- PyMySQL mailing list:
130+
<https://groups.google.com/forum/#!forum/pymysql-users>
131+
132+
## License
133+
134+
PyMySQL is released under the MIT License. See LICENSE for more
135+
information.

‎AWS_ETL_S3_to_Glue_RDSAuroraDB/python/PyMySQL-1.1.1.dist-info/RECORD

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
PyMySQL-1.1.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
2+
PyMySQL-1.1.1.dist-info/LICENSE,sha256=MUEg3GXwgA9ziksxQAx27hTezR--d86cNUCkIbhup7Y,1070
3+
PyMySQL-1.1.1.dist-info/METADATA,sha256=9rEWPHhKScrQDgtyF9-myblwCpZVxwoGCXLJMDtxWGQ,4404
4+
PyMySQL-1.1.1.dist-info/RECORD,,
5+
PyMySQL-1.1.1.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
6+
PyMySQL-1.1.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
7+
PyMySQL-1.1.1.dist-info/top_level.txt,sha256=IKlV-f4o90sOdnMd6HBvo0l2nqfJOGUzkwZeaEEGuRg,8
8+
pymysql/__init__.py,sha256=tz3GIFRN1ug3ycSCxhFkPZ_rsVE5IHjuVTpIR8rTJRQ,4265
9+
pymysql/__pycache__/__init__.cpython-310.pyc,,
10+
pymysql/__pycache__/_auth.cpython-310.pyc,,
11+
pymysql/__pycache__/charset.cpython-310.pyc,,
12+
pymysql/__pycache__/connections.cpython-310.pyc,,
13+
pymysql/__pycache__/converters.cpython-310.pyc,,
14+
pymysql/__pycache__/cursors.cpython-310.pyc,,
15+
pymysql/__pycache__/err.cpython-310.pyc,,
16+
pymysql/__pycache__/optionfile.cpython-310.pyc,,
17+
pymysql/__pycache__/protocol.cpython-310.pyc,,
18+
pymysql/__pycache__/times.cpython-310.pyc,,
19+
pymysql/_auth.py,sha256=ytTe6T4_dRKkT4x1gwXJYBXeUKI50sR7_IZj_oWtYY0,7417
20+
pymysql/charset.py,sha256=_f1uIga7AaWoeKLXzA-9Xra9jYPqqgDiT78ikqtn5yE,10238
21+
pymysql/connections.py,sha256=Yvd97VqhGr6QciyZy9lsQeoZFjXdpQ1xaGcWiXMel7c,53684
22+
pymysql/constants/CLIENT.py,sha256=SSvMFPZCTVMU1UWa4zOrfhYMDdR2wG2mS0E5GzJhDsg,878
23+
pymysql/constants/COMMAND.py,sha256=TGITAUcNWlq2Gwg2wv5UK2ykdTd4LYTk_EcJJOCpGIc,679
24+
pymysql/constants/CR.py,sha256=Qk35FWRMxRHd6Sa9CCIATMh7jegR3xnLdrdaBCT0dTQ,2320
25+
pymysql/constants/ER.py,sha256=nwqX_r0o4mmN4Cxm7NVRyJOTVov_5Gbl5peGe6oz5fk,12357
26+
pymysql/constants/FIELD_TYPE.py,sha256=ytFzgAnGmb9hvdsBlnK68qdZv_a6jYFIXT6VSAb60z8,370
27+
pymysql/constants/FLAG.py,sha256=Fy-PrCLnUI7fx_o5WypYnUAzWAM0E9d5yL8fFRVKffY,214
28+
pymysql/constants/SERVER_STATUS.py,sha256=m28Iq5JGCFCWLhafE73-iOvw_9gDGqnytW3NkHpbugA,333
29+
pymysql/constants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
30+
pymysql/constants/__pycache__/CLIENT.cpython-310.pyc,,
31+
pymysql/constants/__pycache__/COMMAND.cpython-310.pyc,,
32+
pymysql/constants/__pycache__/CR.cpython-310.pyc,,
33+
pymysql/constants/__pycache__/ER.cpython-310.pyc,,
34+
pymysql/constants/__pycache__/FIELD_TYPE.cpython-310.pyc,,
35+
pymysql/constants/__pycache__/FLAG.cpython-310.pyc,,
36+
pymysql/constants/__pycache__/SERVER_STATUS.cpython-310.pyc,,
37+
pymysql/constants/__pycache__/__init__.cpython-310.pyc,,
38+
pymysql/converters.py,sha256=8Jl-1K1Nt-ZKAiahBJV4MoSvO1O-PZtu8CfQG9EDftk,9523
39+
pymysql/cursors.py,sha256=a4-JHYP148kx-9qVNRz8vTtlilGlKDbk_QtFlWph5L4,16535
40+
pymysql/err.py,sha256=wLe0af6AmK6z7fq_MnYfgYsc6LnUuMj7EliHPZKquBA,4178
41+
pymysql/optionfile.py,sha256=eQoz6c43yvmHtp5MI9TB2GPRdoggOLemcUWABksfutk,651
42+
pymysql/protocol.py,sha256=aD-PGPRYcwkSI6ZJoJWZVRKn9H_A0f70KfPDu65tq0o,11812
43+
pymysql/times.py,sha256=_qXgDaYwsHntvpIKSKXp1rrYIgtq6Z9pLyLnO2XNoL0,360

‎AWS_ETL_S3_to_Glue_RDSAuroraDB/python/PyMySQL-1.1.1.dist-info/REQUESTED

Whitespace-only changes.

‎AWS_ETL_S3_to_Glue_RDSAuroraDB/python/PyMySQL-1.1.1.dist-info/WHEEL

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Wheel-Version: 1.0
2+
Generator: bdist_wheel (0.43.0)
3+
Root-Is-Purelib: true
4+
Tag: py3-none-any
5+

‎AWS_ETL_S3_to_Glue_RDSAuroraDB/python/PyMySQL-1.1.1.dist-info/top_level.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
pymysql

0 commit comments

Comments
 (0)