Create template files for the setting up a new project on Incubator #49

robinglov · 2024-02-05T01:54:58Z

Overview

Create template files for the setting up a new project on Incubator.

Action Items

Write template versions.tf and main.tf file templates
- Copy text to a comment and ask Judson or other senior engineer to edit
- edit draft comments
- remove label: role: Dev Ops Engineer
- add label role: product
- unassign yourself
- add the issue to the questions/review column of project board
Set up a folder called examples/sample-project as a child folder of https://github.com/hackforla/incubator/tree/main/terraform-incubator
Add tf templates to the sample-project folder

Resources/Instructions

Refer to these files:
https://github.com/hackforla/incubator/blob/main/terraform-incubator/access-the-data/main.tf
https://github.com/hackforla/incubator/blob/main/terraform-incubator/access-the-data/versions.tf

The text was updated successfully, but these errors were encountered:

ExperimentsInHonesty · 2024-02-05T02:16:27Z

Examples from access-the-data

main.tf

locals {
// we use tf to create the zone, but other projects might
// have an existing zone and get it with a data block
zone_id = module.zone.zone_id

envs = {
dev = {
environment = "dev"
host_names = ["dev"]
container_env = {
CKAN_SITE_URL = "https://dev.accessthedata.org"
}
}
}
}

module "zone" {
source = "../../terraform-modules/project-zone"

zone_name = "accessthedata.org"
github_at_apex = true
shared_configuration = local.shared_configuration
}

module "database" {
for_each = local.envs

source = "../../terraform-modules/database"

shared_configuration = local.shared_configuration
environment = each.value.environment
db_name = "accessthedata"
owner_name = "ckan"
}

module "datastore_database" {
for_each = local.envs

source = "../../terraform-modules/database"

shared_configuration = local.shared_configuration
environment = each.value.environment
db_name = "accessthedata_datastore"
owner_name = "ckands"
viewer_name = "ckands_ro"
}

module "secrets" {
for_each = local.envs
source = "../../terraform-modules/cheap-secrets"
scope-name = "ckan-${each.key}"
secret-names = ["csrf", "admin-password"]
}

module "access-the-data" {
for_each = local.envs

source = "../../terraform-modules/multi-container-service"

shared_configuration = local.shared_configuration

region = "us-west-2"
project_name = "access-the-data"
application_type = "fullstack"
environment = each.value.environment
zone_id = local.zone_id

vpc_cidr = "10.10.0.0/16"

containers = {
ckan = {
tag = "latest"
cpu = 256
memory = 512
port = 80

  subdomains    = each.value.host_names
  path_patterns = ["/*"]
  env_vars = merge({
    DATABASE      = "postgres"
    POSTGRES_HOST = module.database[each.key].host
    POSTGRES_PORT = module.database[each.key].port

    // SQLALCHEMY has been set up in the container =
    // we don't know the PG password, so we can't build the URLs

    # Taken verbatim from .env
    CKAN_DB                  = module.database[each.key].database
    CKAN_DB_USER             = module.database[each.key].owner
    CKAN_DATASTORE_DB        = module.datastore_database[each.key].database
    CKAN_DATASTORE_DB_RWUSER = module.datastore_database[each.key].owner
    CKAN_DATASTORE_DB_ROUSER = module.datastore_database[each.key].viewer
    CKAN_VERSION             = "2.10.0"
    CKAN_SITE_ID             = "default"

    CKAN_PORT      = "5000"
    CKAN_PORT_HOST = "5000"

    CKAN_SYSADMIN_NAME  = "ckan_admin"
    CKAN_SYSADMIN_EMAIL = "[email protected]"
    CKAN_STORAGE_PATH   = "/var/lib/ckan"

    CKAN_SMTP_SERVER    = "smtp.hackforla.org:25"
    CKAN_SMTP_STARTTLS  = "True"
    CKAN_SMTP_USER      = "user"
    CKAN_SMTP_PASSWORD  = "pass"
    CKAN_SMTP_MAIL_FROM = "ckan@localhost"

    CKAN_SOLR_URL                       = "http://solr:8983/solr/ckan"
    CKAN_REDIS_URL                      = "redis://redis:6379/1"
    CKAN_DATAPUSHER_URL                 = "http://datapusher:8800"
    CKAN__DATAPUSHER__CALLBACK_URL_BASE = "http://ckan:5000"
    CKAN__HARVEST__MQ__HOSTNAME         = "redis"

    CKAN__PLUGINS               = "envvars image_view text_view recline_view datastore datapusher ckanext_hack4laatd"
    CKAN__HARVEST__MQ__TYPE     = "redis"
    CKAN__HARVEST__MQ__PORT     = "6379"
    CKAN__HARVEST__MQ__REDIS_DB = "1"
    CKAN__FAVICON               = "favicon.png"
  }, lookup(each.value.container_env, "ckan", {}))
  secrets = {
    CKAN_DB_PASSWORD               = module.database[each.key].owner_password_arn
    CKAN_DATASTORE_DB_RWPASSWORD   = module.datastore_database[each.key].owner_password_arn
    CKAN_DATASTORE_DB_ROPASSWORD   = module.datastore_database[each.key].viewer_password_arn
    CKAN___BEAKER__SESSION__SECRET = module.secrets[each.key].arn["csrf"]
    CKAN_SYSADMIN_PASSWORD         = module.secrets[each.key].arn["admin-password"]
  }
}

datapusher = {
  tag    = "latest"
  cpu    = 256
  memory = 512
}

solr = {
  tag    = "latest"
  cpu    = 512
  memory = 4096
}

redis = {
  tag    = "latest"
  cpu    = 256
  memory = 512
}

}
}

versions.tf

// Get configuration from the shared infrastructure
data "terraform_remote_state" "shared" {
backend = "s3"

config = {
bucket = "hlfa-incubator-terragrunt"
dynamodb_table = "terraform-locks"
encrypt = true
key = "terragrunt-states/incubator/./terraform.tfstate"
region = "us-west-2"
}
}

locals {
shared_configuration = data.terraform_remote_state.shared.outputs.configuration
}

provider "aws" {
region = "us-west-2"
}

// Set up Postgres provider to create the database
terraform {
required_providers {
postgresql = {
source = "cyrilgdn/postgresql"
version = "~> 1.21.0"
}
}
}
data "aws_ssm_parameter" "rds_credentials" {
name = "rds_credentials"
}
data "aws_db_instance" "shared" {
db_instance_identifier = local.shared_configuration.db_identifier
}
provider "postgresql" {
host = data.aws_db_instance.shared.address
password = data.aws_ssm_parameter.rds_credentials.value
username = "postgres"
superuser = false
}

ExperimentsInHonesty · 2024-02-05T02:19:49Z

main.tf draft

locals {
// we use tf to create the zone, but other projects might
// have an existing zone and get it with a data block
zone_id = module.zone.zone_id

envs = {
dev = {
environment = "dev"
host_names = ["dev"]
container_env = {
CKAN_SITE_URL = "https://dev.accessthedata.org"
}
}
}
}

module "zone" {
source = "../../terraform-modules/project-zone"

zone_name = "accessthedata.org"
github_at_apex = true
shared_configuration = local.shared_configuration
}

module "database" {
for_each = local.envs

source = "../../terraform-modules/database"

shared_configuration = local.shared_configuration
environment = each.value.environment
db_name = "accessthedata"
owner_name = "ckan"
}

module "datastore_database" {
for_each = local.envs

source = "../../terraform-modules/database"

shared_configuration = local.shared_configuration
environment = each.value.environment
db_name = "accessthedata_datastore"
owner_name = "ckands"
viewer_name = "ckands_ro"
}

module "secrets" {
for_each = local.envs
source = "../../terraform-modules/cheap-secrets"
scope-name = "ckan-${each.key}"
secret-names = ["csrf", "admin-password"]
}

module "access-the-data" {
for_each = local.envs

source = "../../terraform-modules/multi-container-service"

shared_configuration = local.shared_configuration

region = "us-west-2"
project_name = "access-the-data"
application_type = "fullstack"
environment = each.value.environment
zone_id = local.zone_id

vpc_cidr = "10.10.0.0/16"

containers = {
ckan = {
tag = "latest"
cpu = 256
memory = 512
port = 80

  subdomains    = each.value.host_names
  path_patterns = ["/*"]
  env_vars = merge({
    DATABASE      = "postgres"
    POSTGRES_HOST = module.database[each.key].host
    POSTGRES_PORT = module.database[each.key].port

    // SQLALCHEMY has been set up in the container =
    // we don't know the PG password, so we can't build the URLs

    # Taken verbatim from .env
    CKAN_DB                  = module.database[each.key].database
    CKAN_DB_USER             = module.database[each.key].owner
    CKAN_DATASTORE_DB        = module.datastore_database[each.key].database
    CKAN_DATASTORE_DB_RWUSER = module.datastore_database[each.key].owner
    CKAN_DATASTORE_DB_ROUSER = module.datastore_database[each.key].viewer
    CKAN_VERSION             = "2.10.0"
    CKAN_SITE_ID             = "default"

    CKAN_PORT      = "5000"
    CKAN_PORT_HOST = "5000"

    CKAN_SYSADMIN_NAME  = "ckan_admin"
    CKAN_SYSADMIN_EMAIL = "[email protected]"
    CKAN_STORAGE_PATH   = "/var/lib/ckan"

    CKAN_SMTP_SERVER    = "smtp.hackforla.org:25"
    CKAN_SMTP_STARTTLS  = "True"
    CKAN_SMTP_USER      = "user"
    CKAN_SMTP_PASSWORD  = "pass"
    CKAN_SMTP_MAIL_FROM = "ckan@localhost"

    CKAN_SOLR_URL                       = "http://solr:8983/solr/ckan"
    CKAN_REDIS_URL                      = "redis://redis:6379/1"
    CKAN_DATAPUSHER_URL                 = "http://datapusher:8800"
    CKAN__DATAPUSHER__CALLBACK_URL_BASE = "http://ckan:5000"
    CKAN__HARVEST__MQ__HOSTNAME         = "redis"

    CKAN__PLUGINS               = "envvars image_view text_view recline_view datastore datapusher ckanext_hack4laatd"
    CKAN__HARVEST__MQ__TYPE     = "redis"
    CKAN__HARVEST__MQ__PORT     = "6379"
    CKAN__HARVEST__MQ__REDIS_DB = "1"
    CKAN__FAVICON               = "favicon.png"
  }, lookup(each.value.container_env, "ckan", {}))
  secrets = {
    CKAN_DB_PASSWORD               = module.database[each.key].owner_password_arn
    CKAN_DATASTORE_DB_RWPASSWORD   = module.datastore_database[each.key].owner_password_arn
    CKAN_DATASTORE_DB_ROPASSWORD   = module.datastore_database[each.key].viewer_password_arn
    CKAN___BEAKER__SESSION__SECRET = module.secrets[each.key].arn["csrf"]
    CKAN_SYSADMIN_PASSWORD         = module.secrets[each.key].arn["admin-password"]
  }
}

datapusher = {
  tag    = "latest"
  cpu    = 256
  memory = 512
}

solr = {
  tag    = "latest"
  cpu    = 512
  memory = 4096
}

redis = {
  tag    = "latest"
  cpu    = 256
  memory = 512
}

}
}

ExperimentsInHonesty · 2024-02-05T02:21:34Z

version.tf draft

// Get configuration from the shared infrastructure
data "terraform_remote_state" "shared" {
backend = "s3"

config = {
bucket = "hlfa-incubator-terragrunt"
dynamodb_table = "terraform-locks"
encrypt = true
key = "terragrunt-states/incubator/./terraform.tfstate"
region = "us-west-2"
}
}

locals {
shared_configuration = data.terraform_remote_state.shared.outputs.configuration
}

provider "aws" {
region = "us-west-2"
}

// Set up Postgres provider to create the database
terraform {
required_providers {
postgresql = {
source = "cyrilgdn/postgresql"
version = "~> 1.21.0"
}
}
}
data "aws_ssm_parameter" "rds_credentials" {
name = "rds_credentials"
}
data "aws_db_instance" "shared" {
db_instance_identifier = local.shared_configuration.db_identifier
}
provider "postgresql" {
host = data.aws_db_instance.shared.address
password = data.aws_ssm_parameter.rds_credentials.value
username = "postgres"
superuser = false
}

ExperimentsInHonesty · 2024-02-05T02:27:05Z

@nyarly if you have time to edit some draft main.tf and versions.tf files we have set them up for you in this issue. The checkboxes that apply to you in this issue are

edit draft comments

remove label: role: Dev Ops Engineer

add label role: product

unassign yourself

add the issue to the questions/review column of project board

If you don't have time, let us know and we will ask Tyson.

robinglov · 2024-03-02T17:59:30Z

@Tyson-miller would you like to take this on?

robinglov added size: 0.25pt Can be done in 1.5 hours or less feature: missing complexity: small role: Dev Ops Engineer Engineer who maintains and deploys software feature: templates and removed feature: missing labels Feb 5, 2024

robinglov assigned nyarly and unassigned nyarly Feb 5, 2024

ExperimentsInHonesty added this to the 06 - Documentation/Templates milestone Feb 5, 2024

robinglov mentioned this issue Feb 7, 2024

IBT: Leads #22

Open

madcharlie02 added the feature: missing label Feb 11, 2024

ExperimentsInHonesty removed the feature: missing label Feb 26, 2024

ExperimentsInHonesty mentioned this issue Mar 28, 2024

Create a guide for how to set up a new project on Incubator. #50

Open

4 tasks

robinglov mentioned this issue Mar 28, 2024

2024 DevOps Community of Practice Agendas hackforla/ops#100

Open

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Create template files for the setting up a new project on Incubator #49

Create template files for the setting up a new project on Incubator #49

robinglov commented Feb 5, 2024 •

edited by ExperimentsInHonesty

Loading

ExperimentsInHonesty commented Feb 5, 2024 •

edited

Loading

ExperimentsInHonesty commented Feb 5, 2024

ExperimentsInHonesty commented Feb 5, 2024

ExperimentsInHonesty commented Feb 5, 2024

robinglov commented Mar 2, 2024

Create template files for the setting up a new project on Incubator #49

Create template files for the setting up a new project on Incubator #49

Comments

robinglov commented Feb 5, 2024 • edited by ExperimentsInHonesty Loading

Overview

Action Items

Resources/Instructions

ExperimentsInHonesty commented Feb 5, 2024 • edited Loading

ExperimentsInHonesty commented Feb 5, 2024

main.tf draft

ExperimentsInHonesty commented Feb 5, 2024

version.tf draft

ExperimentsInHonesty commented Feb 5, 2024

robinglov commented Mar 2, 2024

robinglov commented Feb 5, 2024 •

edited by ExperimentsInHonesty

Loading

ExperimentsInHonesty commented Feb 5, 2024 •

edited

Loading