updated script to us h3 bucket and env

This commit is contained in:
CameronCSS 2025-12-09 15:49:29 -07:00
parent e280619f2f
commit 9e1dabbc66
2 changed files with 29 additions and 7 deletions

17
.gitignore vendored Normal file
View file

@ -0,0 +1,17 @@
# Local data files
files/
*.csv
*.json
*.xlsx
# Python caches
__pycache__/
*.pyc
*.pyo
*.pyd
# VS Code settings
.vscode/
#Environment files
.env

View file

@ -1,4 +1,6 @@
from faker import Faker
from dotenv import load_dotenv
import os
import pandas as pd
import boto3
@ -6,14 +8,17 @@ import boto3
fake = Faker()
# ---- Hetzner S3 setup ----
# Load .env file
load_dotenv()
s3 = boto3.resource(
's3',
endpoint_url='https://fsn1.your-objectstorage.com',
aws_access_key_id='LXH38VQ3K0A87TZS0KXP',
aws_secret_access_key='tkJt2iNjmEj1KAqtx2Tvb3WqQNkOxgqJzHC7Iq1H'
endpoint_url=os.getenv('STORAGE_ENDPOINT'),
aws_access_key_id=os.getenv('STORAGE_ACCESS_KEY'),
aws_secret_access_key=os.getenv('STORAGE_SECRET_KEY')
)
bucket_name = 'CamDoesData'
bucket_name = os.getenv('STORAGE_BUCKET')
s3_key = 'DataLab/branches/branches.csv'
# ---- Generate branch data ----
@ -31,9 +36,9 @@ for i in range(1, 11): # 10 Branches
df = pd.DataFrame(branches)
# ---- Save locally (optional) ----
# local_file = "branches.csv"
# df.to_csv(local_file, index=False)
# print("Generated 10 branches locally.")
local_file = "branches.csv"
df.to_csv(local_file, index=False)
print("Generated 10 branches locally.")
# ---- Upload to S3 ----
s3.Bucket(bucket_name).upload_file(local_file, s3_key)