Azure Auto Purge Storage Content
Limpia automáticamente contenido
Antecedentes
Tengo una cuenta de almacenamiento donde almaceno backups. Si almaceno hasta el infinito, guardaré cosas inservibles. En mi caso, con tener almacenados los últimos 10 backups me basta, por lo que todos los anteriores los puedo borrar.
Script
Primero, creamos un script en Python que se encargue de borrar los objetos más antiguos de la cuenta de almacenamiento.
#!/usr/bin/env python3
## author: github.com/aramcap
import azure.identity
import azure.storage.blob
# Vars
account_url = "https://<nombre_del_cubo>.blob.core.windows.net"
container_name = "backups"
retain_blobs = 10
# Count blobs in the container
def blobs_count(container_client):
blob_list = container_client.list_blobs()
blobs_count = 0
for blob in blob_list:
blobs_count += 1
return blobs_count
def main():
try:
print("Autorotate Blobs")
# Get credential based on managed identity
credential = azure.identity.DefaultAzureCredential()
# Client
blob_service_client = azure.storage.blob.BlobServiceClient(account_url, credential=credential)
container_client = blob_service_client.get_container_client(container=container_name)
blobs_n = blobs_count(container_client)
print("Retain blobs: {0}".format(retain_blobs))
print("Blobs count: {0}".format(blobs_n))
while(blobs_n > retain_blobs):
blob_list = container_client.list_blobs()
blob = blob_list.next()
print("Deleting old backup {}...".format(blob.name))
container_client.delete_blob(blob)
print("Deleted old backup {}".format(blob.name))
blobs_n -= 1
print("Nothing more to do")
except Exception as ex:
print('Exception:')
print(ex)
main()
Terraform
Para automatizar la ejecución del script, creamos una cuenta de automatización de Azure y un runbook que ejecute el script. El runbook se ejecutará diariamente a las 6:00 UTC.
# Automation account
resource "azurerm_automation_account" "account" {
name = "${var.default_prefix}-automation"
location = "${var.rg.location}"
resource_group_name = "${var.rg.name}"
sku_name = "Basic"
tags = "${var.default_tags}"
identity {
type = "SystemAssigned"
}
}
# Python libs
resource "azapi_resource" "azure_core" {
type = "Microsoft.Automation/automationAccounts/python3Packages@2022-08-08"
name = "azure_core"
parent_id = azurerm_automation_account.account.id
body = jsonencode({
properties = {
contentLink = {
contentHash = {
algorithm = "SHA256"
value = "f7bad0a7b4d800d6e73733ea7e13a616016221ac111ff9a344fe4cba41e51bbe"
}
uri = "https://files.pythonhosted.org/packages/73/c6/d938eab39385703d28555872df60b77e11779229ffb611118ac1c8725ca5/azure_core-1.26.3-py3-none-any.whl"
version = "1.26.3"
}
}
})
}
resource "azapi_resource" "azure_identity" {
type = "Microsoft.Automation/automationAccounts/python3Packages@2022-08-08"
name = "azure_identity"
parent_id = azurerm_automation_account.account.id
body = jsonencode({
properties = {
contentLink = {
contentHash = {
algorithm = "SHA256"
value = "2a58ce4a209a013e37eaccfd5937570ab99e9118b3e1acf875eed3a85d541b92"
}
uri = "https://files.pythonhosted.org/packages/ce/96/942f03d8a80e30e2289496c10d99e3a8b71f10c0b70b5337fd8ec2ae85e5/azure_identity-1.12.0-py3-none-any.whl"
version = "1.12.0"
}
}
})
}
resource "azapi_resource" "azure_mgmt_core" {
type = "Microsoft.Automation/automationAccounts/python3Packages@2022-08-08"
name = "azure_mgmt_core"
parent_id = azurerm_automation_account.account.id
body = jsonencode({
properties = {
contentLink = {
contentHash = {
algorithm = "SHA256"
value = "fd829f67086e5cf6f7eb016c9e80bb0fb293cbbbd4d8738dc90af9aa1055fb7b"
}
uri = "https://files.pythonhosted.org/packages/29/99/d06021eb45ff660cd4cf1bf16a60645a8256672edf46ff1976a709a50918/azure_mgmt_core-1.3.2-py3-none-any.whl"
version = "1.3.2"
}
}
})
}
resource "azapi_resource" "azure_mgmt_storage" {
type = "Microsoft.Automation/automationAccounts/python3Packages@2022-08-08"
name = "azure_mgmt_storage"
parent_id = azurerm_automation_account.account.id
body = jsonencode({
properties = {
contentLink = {
contentHash = {
algorithm = "SHA256"
value = "89d644c6192118b0b097deaa9c4925832d8f7ea4693d38d5fce3f0125b43a1c5"
}
uri = "https://files.pythonhosted.org/packages/6a/53/dd7f37c78faf77f15ec0acfea3094b317e7183014414ffe737c2e32e6fa7/azure_mgmt_storage-21.0.0-py3-none-any.whl"
version = "21.0.0"
}
}
})
}
resource "azapi_resource" "azure_storage_blob" {
type = "Microsoft.Automation/automationAccounts/python3Packages@2022-08-08"
name = "azure_storage_blob"
parent_id = azurerm_automation_account.account.id
body = jsonencode({
properties = {
contentLink = {
contentHash = {
algorithm = "SHA256"
value = "08d8807c577c63a436740627927c1a03a97c963efc29af5c818aed906590e1cf"
}
uri = "https://files.pythonhosted.org/packages/46/cf/ef1daa7b7df2b2d72db82fa2a777bf50133f4797b4bdfa6b3bbea09660fe/azure_storage_blob-12.15.0-py3-none-any.whl"
version = "12.15.0"
}
}
})
}
resource "azapi_resource" "msal" {
type = "Microsoft.Automation/automationAccounts/python3Packages@2022-08-08"
name = "msal"
parent_id = azurerm_automation_account.account.id
body = jsonencode({
properties = {
contentLink = {
contentHash = {
algorithm = "SHA256"
value = "e8444617c1eccdff7bb73f5d4f94036002accea4a2c05f8f39c9efb5bd2b0c6a"
}
uri = "https://files.pythonhosted.org/packages/cc/86/9900755d4be6e36b48603f9056ffca341d42139297df6f7e143fa96e6ed7/msal-1.21.0-py2.py3-none-any.whl"
version = "1.21.0"
}
}
})
}
resource "azapi_resource" "typing_extensions" {
type = "Microsoft.Automation/automationAccounts/python3Packages@2022-08-08"
name = "typing_extensions"
parent_id = azurerm_automation_account.account.id
body = jsonencode({
properties = {
contentLink = {
contentHash = {
algorithm = "SHA256"
value = "fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"
}
uri = "https://files.pythonhosted.org/packages/31/25/5abcd82372d3d4a3932e1fa8c3dbf9efac10cc7c0d16e78467460571b404/typing_extensions-4.5.0-py3-none-any.whl"
version = "4.5.0"
}
}
})
}
# Automation account - Get file purgebackups
data "local_file" "purgebackups" {
filename = "purgebackups.py"
}
resource "azurerm_automation_runbook" "runbook" {
name = "purgebackups_sa"
location = "${var.rg.location}"
resource_group_name = "${var.rg.name}"
automation_account_name = azurerm_automation_account.automation.name
log_verbose = "true"
log_progress = "true"
description = "Runbook for purge old backup blobs"
runbook_type = "Python3"
publish_content_link {
uri = "https://github.com/aramcap"
}
content = data.local_file.purgebackups.content
}
resource "azurerm_automation_schedule" "schedule" {
name = "daily_6h"
resource_group_name = "${var.rg.name}"
automation_account_name = azurerm_automation_account.automation.name
frequency = "Day"
interval = 1
timezone = "UTC"
start_time = formatdate("YYYY-MM-DD'T06:00:00+00:00'", timeadd(timestamp(), "24h"))
description = "0 6 * * * UTC"
}
resource "azurerm_automation_job_schedule" "job" {
resource_group_name = "${var.rg.name}"
automation_account_name = azurerm_automation_account.automation.name
schedule_name = azurerm_automation_schedule.schedule.name
runbook_name = azurerm_automation_runbook.runbook.name
parameters = {
storage_account_name = "${var.sa_backups_name}"
storage_account_key = "${var.sa_backups_key}"
container = "${var.ct_backups_cmsbackups_name}"
}
}
Y listo. Cada día a las 6:00 UTC se ejecutará el runbook que borrará los objetos más antiguos de la cuenta de almacenamiento, manteniendo los últimos 10 backups.
