feat: introduce ooni-geoip private bucket for geoip database files

.github/workflows/fmt_terraform.yml

@@ -0,0 +1,48 @@
+name: Terraform Auto Format on Main
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: write
+
+jobs:
+  terraform-fmt:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install Terraform
+        run: |
+          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \
+              | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install terraform
+
+      - name: Terraform fmt
+        id: fmt
+        run: terraform fmt -recursive
+        continue-on-error: true
+
+      - name: Check for changes
+        id: git-check
+        run: |
+          if [[ -n $(git status --porcelain) ]]; then
+            echo "changes=true" >> $GITHUB_OUTPUT
+          else
+            echo "changes=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Commit and push changes
+        if: steps.git-check.outputs.changes == 'true'
+        run: |
+          git config user.name "GitHub Actions"
+          git config user.email "actions@github.com"
+          git add .
+          git commit -m "chore(terraform): auto-format on merge to main"
+          git push

tf/environments/dev/main.tf

@@ -244,7 +244,7 @@ resource "random_id" "artifact_id" {
 }
 
 resource "aws_s3_bucket" "anoncred_manifests" {
-  bucket = "ooni-anoncreds-manifests-dev-${var.aws_region}"
+  bucket              = "ooni-anoncreds-manifests-dev-${var.aws_region}"
   object_lock_enabled = true
   versioning {
     enabled = true
@@ -313,7 +313,7 @@ resource "aws_s3_bucket_acl" "anonc_manifests" {
 # Stored here to be publicly available, verifiable, and version controlled
 resource "aws_s3_object" "manifest" {
   bucket = aws_s3_bucket.anoncred_manifests.id
-  key = "manifest.json"
+  key    = "manifest.json"
   content = jsonencode({
     nym_scope = "ooni.org/{probe_cc}/{probe_asn}"
     submission_policy = {
@@ -326,7 +326,7 @@ resource "aws_s3_object" "manifest" {
 # Test manifest used for integration tests
 resource "aws_s3_object" "test_manifest" {
   bucket = aws_s3_bucket.anoncred_manifests.id
-  key = "test_manifest.json"
+  key    = "test_manifest.json"
   content = jsonencode({
     nym_scope = "ooni.org/{probe_cc}/{probe_asn}"
     submission_policy = {
@@ -401,8 +401,8 @@ module "ooniapi_cluster" {
   vpc_id     = module.network.vpc_id
   subnet_ids = module.network.vpc_subnet_private[*].id
 
-  asg_min     = 2
-  asg_max     = 4
+  asg_min = 2
+  asg_max = 4
 
   instance_type = "t3a.micro"
 
@@ -429,8 +429,8 @@ module "oonitier1plus_cluster" {
   vpc_id     = module.network.vpc_id
   subnet_ids = module.network.vpc_subnet_private[*].id
 
-  asg_min     = 1
-  asg_max     = 4
+  asg_min = 1
+  asg_max = 4
 
   instance_type = "t3a.micro"
 
@@ -547,13 +547,13 @@ module "ooniapi_ooniprobe" {
     # module.ooniapi_cluster.web_security_group_id
   ]
 
-  use_autoscaling = true
+  use_autoscaling       = true
   service_desired_count = 1
-  max_desired_count = 4
+  max_desired_count     = 4
   autoscale_policies = [
     {
-      resource_type = "memory"
-      name = "memory"
+      resource_type     = "memory"
+      name              = "memory"
       scaleout_treshold = 60
     }
   ]
@@ -1070,13 +1070,13 @@ module "ooniapi_oonimeasurements" {
     module.oonitier1plus_cluster.web_security_group_id
   ]
 
-  use_autoscaling = true
+  use_autoscaling       = true
   service_desired_count = 1
-  max_desired_count = 8
+  max_desired_count     = 8
   autoscale_policies = [
     {
-      name = "memory"
-      resource_type = "memory"
+      name              = "memory"
+      resource_type     = "memory"
       scaleout_treshold = 60
     }
   ]

tf/environments/prod/main.tf

@@ -162,10 +162,10 @@ module "oonidevops_github_user" {
 module "oonipg" {
   source = "../../modules/postgresql"
 
-  name                     = "ooni-tier0-postgres"
-  aws_region               = var.aws_region
-  vpc_id                   = module.network.vpc_id
-  subnet_ids               = module.network.vpc_subnet_public[*].id
+  name       = "ooni-tier0-postgres"
+  aws_region = var.aws_region
+  vpc_id     = module.network.vpc_id
+  subnet_ids = module.network.vpc_subnet_public[*].id
 
   # By default, max_connections is computed as:
   # LEAST({DBInstanceClassMemory/9531392}, 5000)
@@ -262,6 +262,16 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" {
   )
 }
 
+module "geoip_bucket" {
+  source = "../../modules/s3_bucket"
+
+  bucket_name         = "ooni-geoip-${var.aws_region}-private-${local.environment}"
+  public_read         = false
+  create_iam_user     = true
+  versioning_enabled  = false
+  object_lock_enabled = false
+}
+
 resource "random_id" "artifact_id" {
   byte_length = 4
 }
@@ -282,8 +292,9 @@ resource "aws_s3_bucket" "ooni_private_config_bucket" {
   bucket = "ooni-config-${var.aws_region}-${random_id.artifact_id.hex}"
 }
 
+
 resource "aws_s3_bucket" "anoncred_manifests" {
-  bucket = "ooni-anoncreds-manifests-${var.aws_region}"
+  bucket              = "ooni-anoncreds-manifests-${var.aws_region}"
   object_lock_enabled = true
   versioning {
     enabled = true
@@ -352,7 +363,7 @@ resource "aws_s3_bucket_acl" "anonc_manifests" {
 # Stored here to be publicly available, verifiable, and version controlled
 resource "aws_s3_object" "manifest" {
   bucket = aws_s3_bucket.anoncred_manifests.id
-  key = "manifest.json"
+  key    = "manifest.json"
   content = jsonencode({
     nym_scope = "ooni.org/{probe_cc}/{probe_asn}"
     submission_policy = {
@@ -484,14 +495,14 @@ module "ooni_clickhouse_proxy" {
     protocol    = "tcp",
     cidr_blocks = ["0.0.0.0/0"],
     }, {
-    from_port   = 9000,
-    to_port     = 9000,
-    protocol    = "tcp",
+    from_port = 9000,
+    to_port   = 9000,
+    protocol  = "tcp",
     cidr_blocks = concat(
       module.network.vpc_subnet_public[*].cidr_block,
       module.network.vpc_subnet_private[*].cidr_block,
       ["${module.ooni_fastpath.aws_instance_private_ip}/32",
-        "${module.ooni_fastpath.aws_instance_public_ip}/32"]
+      "${module.ooni_fastpath.aws_instance_public_ip}/32"]
     ),
     }, {
     // For the prometheus proxy:
@@ -614,8 +625,8 @@ module "ooniapi_cluster" {
   subnet_ids = module.network.vpc_subnet_public[*].id
 
   # You need be careful how these are tweaked.
-  asg_min     = 2
-  asg_max     = 10
+  asg_min = 2
+  asg_max = 10
 
   instance_type = "t3a.medium"
 
@@ -642,8 +653,8 @@ module "oonitier1plus_cluster" {
   vpc_id     = module.network.vpc_id
   subnet_ids = module.network.vpc_subnet_private[*].id
 
-  asg_min     = 2
-  asg_max     = 5
+  asg_min = 2
+  asg_max = 5
 
   instance_type = "t3a.medium"
 
@@ -733,7 +744,7 @@ module "ooniapi_ooniprobe" {
   dns_zone_ooni_io         = local.dns_zone_ooni_io
   key_name                 = module.adm_iam_roles.oonidevops_key_name
   ecs_cluster_id           = module.ooniapi_cluster.cluster_id
-  task_memory = 256
+  task_memory              = 256
 
 
   task_secrets = {
@@ -758,13 +769,13 @@ module "ooniapi_ooniprobe" {
     module.ooniapi_cluster.web_security_group_id
   ]
 
-  use_autoscaling = true
+  use_autoscaling       = true
   service_desired_count = 2
-  max_desired_count = 8
+  max_desired_count     = 8
   autoscale_policies = [
     {
-      resource_type = "memory"
-      name = "memory"
+      resource_type     = "memory"
+      name              = "memory"
       scaleout_treshold = 60
     }
   ]
@@ -1092,13 +1103,13 @@ module "ooniapi_oonimeasurements" {
     module.ooniapi_cluster.web_security_group_id
   ]
 
-  use_autoscaling = true
+  use_autoscaling       = true
   service_desired_count = 4
-  max_desired_count = 32 # 8gb (total mem) / 256mb (mem per task) = 32 tasks
+  max_desired_count     = 32 # 8gb (total mem) / 256mb (mem per task) = 32 tasks
   autoscale_policies = [
     {
-      name = "memory"
-      resource_type = "memory"
+      name              = "memory"
+      resource_type     = "memory"
       scaleout_treshold = 60
     }
   ]

tf/modules/ansible_controller/main.tf

@@ -12,9 +12,9 @@ resource "aws_security_group" "ansible_ctrl_sg" {
   }
 
   ingress {
-    protocol = "tcp"
-    from_port = 9100
-    to_port = 9100
+    protocol        = "tcp"
+    from_port       = 9100
+    to_port         = 9100
     security_groups = var.monitoring_sg_ids
   }
 
@@ -64,7 +64,7 @@ resource "aws_instance" "ansible_controller" {
 
   vpc_security_group_ids = [aws_security_group.ansible_ctrl_sg.id]
 
-  tags = merge({ Name = "ansible-controller", MonitoringActive = var.monitoring_active}, var.tags)
+  tags = merge({ Name = "ansible-controller", MonitoringActive = var.monitoring_active }, var.tags)
 }
 
 resource "aws_route53_record" "oonith_service_alias" {

tf/modules/ansible_controller/variables.tf

@@ -27,12 +27,12 @@ variable "dns_zone_ooni_io" {
 
 variable "monitoring_sg_ids" {
   description = "Ids of the security groups used for monitoring"
-  default = []
-  type = list(string)
+  default     = []
+  type        = list(string)
 }
 
 variable "monitoring_active" {
   description = "If the monitoring system should consider the ansible controller machine. Set it to 'true' to activate it, anything else to deactivate it"
-  default = "true"
-  type = string
+  default     = "true"
+  type        = string
 }

tf/modules/cloudhsm/main.tf

@@ -58,7 +58,7 @@ resource "aws_instance" "codesign_box" {
                 rm cloudhsm-pkcs11.rpm
                 EOF
 
-  tags = merge(var.tags, { Name = "codesign-box" , MonitoringActive = var.monitoring_active})
+  tags = merge(var.tags, { Name = "codesign-box", MonitoringActive = var.monitoring_active })
 
   // NOTE: remove the ignore_changes rule to deploy 
   lifecycle {
@@ -87,7 +87,7 @@ resource "aws_launch_template" "codesign_box_template" {
     resource_type = "instance"
 
     tags = {
-      Name = "codesign-box"
+      Name             = "codesign-box"
       MonitoringActive = var.monitoring_active
     }
   }

tf/modules/cloudhsm/variables.tf

@@ -29,6 +29,6 @@ variable "tags" {
 
 variable "monitoring_active" {
   description = "If the monitoring system should consider the HSM machine. Set it to 'true' to activate it, anything else to deactivate it"
-  default = "true"
-  type = string
+  default     = "true"
+  type        = string
 }

tf/modules/ec2/main.tf

@@ -20,25 +20,25 @@ resource "aws_security_group" "ec2_sg" {
 resource "aws_security_group_rule" "ec2_sg_ingress" {
   count = length(var.ingress_rules)
 
-  type = "ingress"
+  type              = "ingress"
   from_port         = var.ingress_rules[count.index].from_port
   to_port           = var.ingress_rules[count.index].to_port
   protocol          = var.ingress_rules[count.index].protocol
   cidr_blocks       = var.ingress_rules[count.index].cidr_blocks
   ipv6_cidr_blocks  = var.ingress_rules[count.index].ipv6_cidr_blocks
-  security_group_id =  aws_security_group.ec2_sg.id
+  security_group_id = aws_security_group.ec2_sg.id
 }
 
 resource "aws_security_group_rule" "ec2_sg_egress" {
   count = length(var.egress_rules)
 
-  type = "egress"
+  type              = "egress"
   from_port         = var.egress_rules[count.index].from_port
   to_port           = var.egress_rules[count.index].to_port
   protocol          = var.egress_rules[count.index].protocol
   cidr_blocks       = var.egress_rules[count.index].cidr_blocks
   ipv6_cidr_blocks  = var.egress_rules[count.index].ipv6_cidr_blocks
-  security_group_id =  aws_security_group.ec2_sg.id
+  security_group_id = aws_security_group.ec2_sg.id
 }
 
 data "cloudinit_config" "ooni_ec2" {
@@ -47,7 +47,7 @@ data "cloudinit_config" "ooni_ec2" {
   part {
     filename     = "init.cfg"
     content_type = "text/cloud-config"
-    content = templatefile("${path.module}/templates/cloud-init.yml", {})
+    content      = templatefile("${path.module}/templates/cloud-init.yml", {})
   }
 
 }
@@ -87,15 +87,15 @@ resource "aws_instance" "ooni_ec2" {
 
   lifecycle {
     create_before_destroy = true
-    ignore_changes = [ user_data, launch_template ]
+    ignore_changes        = [user_data, launch_template]
   }
 
   root_block_device {
-    volume_size = var.disk_size  # Size in GB
+    volume_size = var.disk_size # Size in GB
     volume_type = "gp2"
   }
 
-  tags = merge(var.tags, {MonitoringActive = var.monitoring_active})
+  tags = merge(var.tags, { MonitoringActive = var.monitoring_active })
 }
 
 resource "aws_alb_target_group" "ooni_ec2" {