updated script to us h3 bucket and env
This commit is contained in:
parent
e280619f2f
commit
9e1dabbc66
2 changed files with 29 additions and 7 deletions
17
.gitignore
vendored
Normal file
17
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# Local data files
|
||||
files/
|
||||
*.csv
|
||||
*.json
|
||||
*.xlsx
|
||||
|
||||
# Python caches
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
|
||||
# VS Code settings
|
||||
.vscode/
|
||||
|
||||
#Environment files
|
||||
.env
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
from faker import Faker
|
||||
from dotenv import load_dotenv
|
||||
import os
|
||||
import pandas as pd
|
||||
import boto3
|
||||
|
||||
|
|
@ -6,14 +8,17 @@ import boto3
|
|||
fake = Faker()
|
||||
|
||||
# ---- Hetzner S3 setup ----
|
||||
# Load .env file
|
||||
load_dotenv()
|
||||
|
||||
s3 = boto3.resource(
|
||||
's3',
|
||||
endpoint_url='https://fsn1.your-objectstorage.com',
|
||||
aws_access_key_id='LXH38VQ3K0A87TZS0KXP',
|
||||
aws_secret_access_key='tkJt2iNjmEj1KAqtx2Tvb3WqQNkOxgqJzHC7Iq1H'
|
||||
endpoint_url=os.getenv('STORAGE_ENDPOINT'),
|
||||
aws_access_key_id=os.getenv('STORAGE_ACCESS_KEY'),
|
||||
aws_secret_access_key=os.getenv('STORAGE_SECRET_KEY')
|
||||
)
|
||||
|
||||
bucket_name = 'CamDoesData'
|
||||
bucket_name = os.getenv('STORAGE_BUCKET')
|
||||
s3_key = 'DataLab/branches/branches.csv'
|
||||
|
||||
# ---- Generate branch data ----
|
||||
|
|
@ -31,9 +36,9 @@ for i in range(1, 11): # 10 Branches
|
|||
df = pd.DataFrame(branches)
|
||||
|
||||
# ---- Save locally (optional) ----
|
||||
# local_file = "branches.csv"
|
||||
# df.to_csv(local_file, index=False)
|
||||
# print("Generated 10 branches locally.")
|
||||
local_file = "branches.csv"
|
||||
df.to_csv(local_file, index=False)
|
||||
print("Generated 10 branches locally.")
|
||||
|
||||
# ---- Upload to S3 ----
|
||||
s3.Bucket(bucket_name).upload_file(local_file, s3_key)
|
||||
|
|
|
|||
Loading…
Reference in a new issue