updated script to us h3 bucket and env
This commit is contained in:
parent
e280619f2f
commit
9e1dabbc66
2 changed files with 29 additions and 7 deletions
17
.gitignore
vendored
Normal file
17
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
# Local data files
|
||||||
|
files/
|
||||||
|
*.csv
|
||||||
|
*.json
|
||||||
|
*.xlsx
|
||||||
|
|
||||||
|
# Python caches
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
*.pyo
|
||||||
|
*.pyd
|
||||||
|
|
||||||
|
# VS Code settings
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
#Environment files
|
||||||
|
.env
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
from faker import Faker
|
from faker import Faker
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
import os
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import boto3
|
import boto3
|
||||||
|
|
||||||
|
|
@ -6,14 +8,17 @@ import boto3
|
||||||
fake = Faker()
|
fake = Faker()
|
||||||
|
|
||||||
# ---- Hetzner S3 setup ----
|
# ---- Hetzner S3 setup ----
|
||||||
|
# Load .env file
|
||||||
|
load_dotenv()
|
||||||
|
|
||||||
s3 = boto3.resource(
|
s3 = boto3.resource(
|
||||||
's3',
|
's3',
|
||||||
endpoint_url='https://fsn1.your-objectstorage.com',
|
endpoint_url=os.getenv('STORAGE_ENDPOINT'),
|
||||||
aws_access_key_id='LXH38VQ3K0A87TZS0KXP',
|
aws_access_key_id=os.getenv('STORAGE_ACCESS_KEY'),
|
||||||
aws_secret_access_key='tkJt2iNjmEj1KAqtx2Tvb3WqQNkOxgqJzHC7Iq1H'
|
aws_secret_access_key=os.getenv('STORAGE_SECRET_KEY')
|
||||||
)
|
)
|
||||||
|
|
||||||
bucket_name = 'CamDoesData'
|
bucket_name = os.getenv('STORAGE_BUCKET')
|
||||||
s3_key = 'DataLab/branches/branches.csv'
|
s3_key = 'DataLab/branches/branches.csv'
|
||||||
|
|
||||||
# ---- Generate branch data ----
|
# ---- Generate branch data ----
|
||||||
|
|
@ -31,9 +36,9 @@ for i in range(1, 11): # 10 Branches
|
||||||
df = pd.DataFrame(branches)
|
df = pd.DataFrame(branches)
|
||||||
|
|
||||||
# ---- Save locally (optional) ----
|
# ---- Save locally (optional) ----
|
||||||
# local_file = "branches.csv"
|
local_file = "branches.csv"
|
||||||
# df.to_csv(local_file, index=False)
|
df.to_csv(local_file, index=False)
|
||||||
# print("Generated 10 branches locally.")
|
print("Generated 10 branches locally.")
|
||||||
|
|
||||||
# ---- Upload to S3 ----
|
# ---- Upload to S3 ----
|
||||||
s3.Bucket(bucket_name).upload_file(local_file, s3_key)
|
s3.Bucket(bucket_name).upload_file(local_file, s3_key)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue