diff --git a/city_scrapers/settings/prod.py b/city_scrapers/settings/prod.py index 64b77e4..7ce09f0 100644 --- a/city_scrapers/settings/prod.py +++ b/city_scrapers/settings/prod.py @@ -11,7 +11,7 @@ # Configure item pipelines ITEM_PIPELINES = { # "city_scrapers_core.pipelines.S3DiffPipeline": 200, - # "city_scrapers_core.pipelines.AzureDiffPipeline": 200, + "city_scrapers_core.pipelines.AzureDiffPipeline": 200, # "city_scrapers_core.pipelines.GCSDiffPipeline": 200, "city_scrapers_core.pipelines.MeetingPipeline": 300, "city_scrapers_core.pipelines.OpenCivicDataPipeline": 400, @@ -28,7 +28,7 @@ SENTRY_DSN = os.getenv("SENTRY_DSN") EXTENSIONS = { - # "city_scrapers_core.extensions.AzureBlobStatusExtension": 100, + "city_scrapers_core.extensions.AzureBlobStatusExtension": 100, # "city_scrapers_core.extensions.S3StatusExtension": 100, # "city_scrapers_core.extensions.GCSStatusExtension": 100, "scrapy_sentry.extensions.Errors": 10, @@ -47,7 +47,7 @@ FEED_STORAGES = { # "s3": "scrapy.extensions.feedexport.S3FeedStorage", - # "azure": "city_scrapers_core.extensions.AzureBlobFeedStorage", + "azure": "city_scrapers_core.extensions.AzureBlobFeedStorage", # "gcs": "scrapy.extensions.feedexport.GCSFeedStorage", } @@ -58,10 +58,10 @@ # S3_BUCKET = os.getenv("S3_BUCKET") # CITY_SCRAPERS_STATUS_CONTAINER = S3_BUCKET -# AZURE_ACCOUNT_NAME = os.getenv("AZURE_ACCOUNT_NAME") -# AZURE_ACCOUNT_KEY = os.getenv("AZURE_ACCOUNT_KEY") -# AZURE_CONTAINER = os.getenv("AZURE_CONTAINER") -# CITY_SCRAPERS_STATUS_CONTAINER = AZURE_CONTAINER +AZURE_ACCOUNT_NAME = os.getenv("AZURE_ACCOUNT_NAME") +AZURE_ACCOUNT_KEY = os.getenv("AZURE_ACCOUNT_KEY") +AZURE_CONTAINER = os.getenv("AZURE_CONTAINER") +CITY_SCRAPERS_STATUS_CONTAINER = AZURE_CONTAINER # GOOGLE_APPLICATION_CREDENTIALS = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") # GCS_BUCKET = os.getenv("GCS_BUCKET") @@ -88,14 +88,14 @@ # bucket=S3_BUCKET # ) -# FEED_URI = ( -# "azure://{account_name}:{account_key}@{container}" -# "/%(year)s/%(month)s/%(day)s/%(hour_min)s/%(name)s.json" -# ).format( -# account_name=AZURE_ACCOUNT_NAME, -# account_key=AZURE_ACCOUNT_KEY, -# container=AZURE_CONTAINER, -# ) +FEED_URI = ( + "azure://{account_name}:{account_key}@{container}" + "/%(year)s/%(month)s/%(day)s/%(hour_min)s/%(name)s.json" +).format( + account_name=AZURE_ACCOUNT_NAME, + account_key=AZURE_ACCOUNT_KEY, + container=AZURE_CONTAINER, +) # FEED_URI = ( # "gs://{bucket}/%(year)s/%(month)s/%(day)s/%(hour_min)s/%(name)s.json"