workspace "product" "backend" { !identifiers hierarchical model { devopsStaff = person "DevOps Staff" "DevOps staff within Meandair." user = person "Customer" "A customer of Meandair." weatherDataProviders = softwareSystem "Weather Data Providers" "External providers of NWP, observation, forecast, and satellite-derived source data used by the processing pipelines." "External" apiSystem = softwareSystem "API System" "Manages centralized access via REST API." { apigateway = container "apigateway" "Provides a centralized API, authentication, authorization, and documentation." "OpenResty, Lua, Redocly" "Runtime" tlscontroller = container "tlscontroller" "Renews TLS certificates for Kubernetes ingress and updates the TLS secret used by the API entrypoints." "Kubernetes CronJob, certbot, lexicon" "Infrastructure" tlscontroller -> apigateway "Renews TLS material for" "Kubernetes secret" } weatherSystem = softwareSystem "Weather Processing System" "Processes weather data." { airserver = container "airserver" "Provides a centralized access to weather data (map, tile, coverage, spot, METAR, AW-METAR, TAF, AW-TAF)." "Actix Web" "Runtime" nwcgeo = container "nwcgeo" "Generates regional meteorological products from geostationary satellite data." "Ansible-managed NWC/GEO" "Runtime" nwcgeo_aux = container "nwcgeo_aux" "Downloads and prepares auxiliary inputs for NWC/GEO." "Ansible-managed Python, Airflow-era tooling" "Runtime" nwpprocessor = container "nwpprocessor" "Processes NWP datasets and publishes outputs for downstream services." "Python, Argo Workflows" "Runtime" nwcprocessor = container "nwcprocessor" "Processes nowcast datasets and publishes outputs for downstream services." "Python, Argo Workflows" "Runtime" wxprocessor_aux = container "wxprocessor_aux" "Runs the auxiliary wxprocessor deployments with region=aux." "Python, Airflow" "Runtime" { download_gfs = component "download_gfs" "Downloads GFS source files." "Airflow DAG" prepare_gfs = component "prepare_gfs" "Prepares downloaded GFS source files for downstream processing." "Airflow DAG" download_gtg = component "download_gtg" "Downloads GTG turbulence source files." "Airflow DAG" download_icon_eu = component "download_icon_eu" "Downloads ICON-EU source files." "Airflow DAG" download_metar = component "download_metar" "Downloads and stores METAR observations." "Airflow DAG" download_nwcgeo = component "download_nwcgeo" "Downloads and prepares NWC/GEO source files." "Airflow DAG" download_synop = component "download_synop" "Downloads and stores SYNOP observations." "Airflow DAG" download_taf = component "download_taf" "Downloads and stores TAF forecasts." "Airflow DAG" maintenance = component "maintenance" "Performs recurring cleanup and maintenance work." "Airflow DAG" prepare_gfs -> download_gfs "Gets source data from" } wxprocessor_glb = container "wxprocessor_glb" "Runs the global wxprocessor deployments with region=glb." "Python, Airflow" "Runtime" { airports = component "airports" "Builds airport-focused products and layers for the global deployment." "Airflow DAG" air_pressure_qnh_glb = component "air_pressure_qnh_glb" "Builds global QNH pressure products." "Airflow DAG" temperature_2m_agl_glb = component "temperature_2m_agl_glb" "Builds global temperature and dew point products." "Airflow DAG" visibility_glb = component "visibility_glb" "Builds global visibility products." "Airflow DAG" clouds = component "clouds" "Builds global cloud cover and cloud ceiling products." "Airflow DAG" wind = component "wind" "Builds global wind and wind gust products." "Airflow DAG" airports -> air_pressure_qnh_glb "Gets data from" airports -> temperature_2m_agl_glb "Gets data from" airports -> visibility_glb "Gets data from" airports -> clouds "Gets data from" airports -> wind "Gets data from" } wxprocessor = container "wxprocessor" "Runs the regional wxprocessor deployments for non-aux, non-glb regions." "Python, Airflow" "Runtime" { airports = component "airports" "Builds airport-focused products and layers for regional deployments." "Airflow DAG" cloud = component "cloud" "Builds cloud ceiling, cloud top, and related regional cloud products." "Airflow DAG" nowcast = component "nowcast" "Processes NWC/GEO nowcast products." "Airflow DAG" nwp_cloud = component "nwp_cloud" "Builds regional NWP cloud products from GFS and ICON-EU inputs." "Airflow DAG" process_gfs = component "process_gfs" "Processes prepared GFS inputs into regional NWP products." "Airflow DAG" process_gtg = component "process_gtg" "Processes GTG turbulence inputs." "Airflow DAG" process_icon_eu = component "process_icon_eu" "Processes ICON-EU inputs for regional products." "Airflow DAG" rdt = component "rdt" "Processes NWC/GEO thunderstorm products." "Airflow DAG" air_pressure_qnh = component "air_pressure_qnh" "Builds regional QNH pressure products." "Airflow DAG" air_temperature_2m_agl = component "air_temperature_2m_agl" "Builds regional temperature and dew point products." "Airflow DAG" surface_visibility = component "surface_visibility" "Builds regional surface visibility products." "Airflow DAG" wind_agl = component "wind_agl" "Builds regional wind and wind gust products." "Airflow DAG" airports -> air_pressure_qnh "Gets data from" airports -> air_temperature_2m_agl "Gets data from" airports -> cloud "Gets data from" airports -> nowcast "Gets data from" airports -> nwp_cloud "Gets data from" airports -> surface_visibility "Gets data from" airports -> wind_agl "Gets data from" cloud -> nwp_cloud "Gets data from" nowcast -> process_gfs "Gets data from" nwp_cloud -> process_gfs "Gets data from" nwp_cloud -> process_icon_eu "Gets data from" rdt -> process_gfs "Gets data from" } wxdb = container "wxdb" "Stores weather-processing metadata used by wxprocessor deployments." "PostgreSQL" "Database,DataStore" argoworkflows = container "argoworkflows" "Provides artifact repository and workflow infrastructure used by the processor charts." "Argo Workflows" "Infrastructure,Orchestration" apiSystem.apigateway -> airserver "Routes weather API requests to" "HTTP" apiSystem.apigateway -> nwcgeo "Exposes exported NWC/GEO products from" "HTTP, files" airserver -> wxprocessor_glb "Gets data from" "via apigateway" airserver -> wxprocessor "Gets data from" "via apigateway" nwcgeo -> nwcgeo_aux "Gets data from" nwpprocessor -> airserver "Publishes datasets to" nwcprocessor -> airserver "Publishes datasets to" nwcprocessor -> nwpprocessor "Reads NWP datasets from" "S3" wxprocessor_glb -> wxprocessor_aux "Gets data from" wxprocessor -> wxprocessor_aux "Gets data from" wxprocessor -> nwcgeo "Gets data from" "via apigateway; netCDF" wxprocessor_aux -> wxdb "Reads and writes metadata in" "PostgreSQL" wxprocessor_glb -> wxdb "Reads and writes metadata in" "PostgreSQL" wxprocessor -> wxdb "Reads and writes metadata in" "PostgreSQL" argoworkflows -> nwpprocessor "Runs scheduled workflows for" argoworkflows -> nwcprocessor "Runs scheduled workflows for" nwpprocessor -> weatherDataProviders "Downloads forecast source datasets from" wxprocessor_aux -> weatherDataProviders "Downloads NWP and observation source datasets from" nwcgeo_aux -> weatherDataProviders "Downloads auxiliary weather inputs from" nwcgeo -> weatherDataProviders "Processes satellite-derived source inputs from" wxprocessor_glb.air_pressure_qnh_glb -> wxprocessor_aux.prepare_gfs "Gets data from" wxprocessor_glb.temperature_2m_agl_glb -> wxprocessor_aux.prepare_gfs "Gets data from" wxprocessor_glb.visibility_glb -> wxprocessor_aux.prepare_gfs "Gets data from" wxprocessor_glb.clouds -> wxprocessor_aux.prepare_gfs "Gets data from" wxprocessor_glb.wind -> wxprocessor_aux.prepare_gfs "Gets data from" wxprocessor_glb.airports -> wxprocessor_aux.download_metar "Gets observation data from" wxprocessor_glb.airports -> wxprocessor_aux.download_taf "Gets forecast data from" wxprocessor.process_icon_eu -> wxprocessor_aux.download_icon_eu "Gets data from" wxprocessor.process_gtg -> wxprocessor_aux.download_gtg "Gets data from" wxprocessor.nowcast -> wxprocessor_aux.download_nwcgeo "Gets data from" wxprocessor.rdt -> wxprocessor_aux.download_nwcgeo "Gets data from" wxprocessor.airports -> wxprocessor_aux.download_metar "Gets observation data from" wxprocessor.airports -> wxprocessor_aux.download_taf "Gets forecast data from" wxprocessor.air_pressure_qnh -> wxprocessor_aux.download_metar "Gets observation data from" wxprocessor.air_temperature_2m_agl -> wxprocessor_aux.download_metar "Gets observation data from" wxprocessor.surface_visibility -> wxprocessor_aux.download_metar "Gets observation data from" wxprocessor.wind_agl -> wxprocessor_aux.download_metar "Gets observation data from" } archiveSystem = softwareSystem "Archive System" "Archives weather data." { minio = container "minio" "Short-term and long-term object storage archive." "MinIO" "FileSystem,DataStore" } user -> apiSystem.apigateway "Sends requests to" "HTTPS" weatherSystem.nwpprocessor -> archiveSystem.minio "Stores datasets in" "S3" weatherSystem.nwcprocessor -> archiveSystem.minio "Stores datasets in" "S3" logSystem = softwareSystem "Log System" "Collects, transforms and stores logs." { elastic = container "elastic" "Transforms logs via ingestion pipelines and stores them." "Elasticsearch" "Infrastructure,Database" kibana = container "kibana" "Allows logs analysis and advanced visualization." "Kibana" "Infrastructure" kibana -> elastic "Gets logs from" } apiSystem.apigateway -> logSystem.elastic "Sends logs to" "Filebeat" monitoringSystem = softwareSystem "Monitoring System" "Monitors the infrastructure." { alertmanager = container "alertmanager" "Handles alerts sent by other applications." "Alertmanager" "Infrastructure" logshipper = container "logshipper" "Collects logs and is the current best match for the historical API exporter role." "Helm deployment" "Infrastructure" prometheus = container "prometheus" "Collects and monitors various metrics in real-time." "Prometheus" "Infrastructure" victoriametrics = container "victoriametrics" "Long-term metrics storage." "VictoriaMetrics" "Database,DataStore" grafana = container "grafana" "Allows metrics visualization." "Grafana" "Infrastructure" nodeexporter = container "nodeexporter" "Provides hardware and OS metrics on each node." "prometheus-node-exporter" "Infrastructure" alertmanager -> devopsStaff "Sends alerts to" grafana -> prometheus "Gets metrics from" prometheus -> alertmanager "Sends alerts to" prometheus -> logshipper "Gets metrics from" prometheus -> victoriametrics "Stores metrics to" prometheus -> nodeexporter "Gets metrics from" } monitoringSystem.logshipper -> weatherSystem.airserver "Checks API status" "via apigateway" deploymentEnvironment "Primary" { primaryClusterGroup = deploymentGroup "primary" primary1Group = deploymentGroup "primary1" primary2Group = deploymentGroup "primary2" primary3Group = deploymentGroup "primary3" primary4Group = deploymentGroup "primary4" primary5Group = deploymentGroup "primary5" nwcgeo1Group = deploymentGroup "nwcgeo1" deploymentNode "primary" "" "Hetzner, k0s" { containerInstance weatherSystem.nwpprocessor primaryClusterGroup containerInstance weatherSystem.wxdb primaryClusterGroup containerInstance apiSystem.tlscontroller primaryClusterGroup containerInstance monitoringSystem.logshipper primaryClusterGroup deploymentNode "primary1" "" "AX102, Debian 12" { deploymentNode "apigateway" "" "Kubernetes DaemonSet" { containerInstance apiSystem.apigateway primary1Group } deploymentNode "airserver" "" "Kubernetes Deployment" { containerInstance weatherSystem.airserver primary1Group } deploymentNode "nodeexporter" "" "System package" { containerInstance monitoringSystem.nodeexporter primary1Group } } deploymentNode "primary2" "" "AX102, Debian 12" { deploymentNode "apigateway" "" "Kubernetes DaemonSet" { containerInstance apiSystem.apigateway primary2Group } deploymentNode "wxprocessor-aux2" "" "Kubernetes Deployment" { containerInstance weatherSystem.wxprocessor_aux primary2Group } deploymentNode "wxprocessor-af1" "" "Kubernetes Deployment" { containerInstance weatherSystem.wxprocessor primary2Group } deploymentNode "wxprocessor-in1" "" "Kubernetes Deployment" { containerInstance weatherSystem.wxprocessor primary2Group } deploymentNode "nodeexporter" "" "System package" { containerInstance monitoringSystem.nodeexporter primary2Group } } deploymentNode "primary3" "" "AX101, Debian 12" { deploymentNode "apigateway" "" "Kubernetes DaemonSet" { containerInstance apiSystem.apigateway primary3Group } deploymentNode "wxprocessor-aux3" "" "Kubernetes Deployment" { containerInstance weatherSystem.wxprocessor_aux primary3Group } deploymentNode "wxprocessor-glb" "" "Kubernetes Deployment" { containerInstance weatherSystem.wxprocessor_glb primary3Group } deploymentNode "wxprocessor-sa1" "" "Kubernetes Deployment" { containerInstance weatherSystem.wxprocessor primary3Group } deploymentNode "nodeexporter" "" "System package" { containerInstance monitoringSystem.nodeexporter primary3Group } } deploymentNode "primary4" "" "AX102, Debian 12" { deploymentNode "apigateway" "" "Kubernetes DaemonSet" { containerInstance apiSystem.apigateway primary4Group } deploymentNode "wxprocessor-aux4" "" "Kubernetes Deployment" { containerInstance weatherSystem.wxprocessor_aux primary4Group } deploymentNode "wxprocessor-eu1" "" "Kubernetes Deployment" { containerInstance weatherSystem.wxprocessor primary4Group } deploymentNode "wxprocessor-oc2" "" "Kubernetes Deployment" { containerInstance weatherSystem.wxprocessor primary4Group } deploymentNode "nodeexporter" "" "System package" { containerInstance monitoringSystem.nodeexporter primary4Group } } deploymentNode "primary5" "" "AX102, Debian 12" { deploymentNode "apigateway" "" "Kubernetes DaemonSet" { containerInstance apiSystem.apigateway primary5Group } deploymentNode "wxprocessor-aux1" "" "Kubernetes Deployment" { containerInstance weatherSystem.wxprocessor_aux primary5Group } deploymentNode "wxprocessor-na1" "" "Kubernetes Deployment" { containerInstance weatherSystem.wxprocessor primary5Group } deploymentNode "nodeexporter" "" "System package" { containerInstance monitoringSystem.nodeexporter primary5Group } } } deploymentNode "nwcgeo1" "" "Single-node cluster" { containerInstance weatherSystem.nwcgeo_aux nwcgeo1Group containerInstance weatherSystem.nwcgeo nwcgeo1Group } } } views { systemlandscape { include * autoLayout } systemcontext apiSystem { include * autoLayout } systemcontext archiveSystem { include * autoLayout } systemcontext logSystem { include * autoLayout } systemcontext monitoringSystem { include * autoLayout } systemcontext weatherSystem { include * autoLayout } container apiSystem { include * autoLayout } container archiveSystem { include * autoLayout } container logSystem { include * autoLayout } container monitoringSystem { include * autoLayout } container weatherSystem { include * autoLayout } component weatherSystem.wxprocessor_aux { include * autoLayout } component weatherSystem.wxprocessor_glb { include * autoLayout } component weatherSystem.wxprocessor { include * autoLayout } deployment * "Primary" { title "Primary deployment topology" include * autoLayout tb } dynamic apiSystem "requestFlow" { title "Customer API request flow" user -> apiSystem.apigateway "Requests weather data from" "HTTPS" apiSystem.apigateway -> weatherSystem.airserver "Routes API request to" "HTTP" autoLayout } dynamic weatherSystem "processingFlow" { title "Forecast and nowcast publication flow" weatherDataProviders -> weatherSystem.nwpprocessor "Provides forecast source datasets to" weatherSystem.argoworkflows -> weatherSystem.nwpprocessor "Starts scheduled workflow for" weatherSystem.nwpprocessor -> archiveSystem.minio "Stores processed forecast datasets in" "S3" weatherSystem.nwpprocessor -> weatherSystem.airserver "Publishes forecast datasets to" weatherSystem.nwcprocessor -> weatherSystem.nwpprocessor "Reads forecast datasets from" "S3" weatherSystem.argoworkflows -> weatherSystem.nwcprocessor "Starts scheduled workflow for" weatherSystem.nwcprocessor -> archiveSystem.minio "Stores processed nowcast datasets in" "S3" weatherSystem.nwcprocessor -> weatherSystem.airserver "Publishes nowcast datasets to" weatherDataProviders -> weatherSystem.wxprocessor_aux "Provides NWP and observation inputs to" weatherSystem.wxprocessor_aux -> weatherSystem.wxprocessor "Provides prepared regional inputs to" weatherSystem.wxprocessor -> weatherSystem.wxdb "Stores processing metadata in" "PostgreSQL" weatherSystem.airserver -> weatherSystem.wxprocessor "Reads regional products from" "via apigateway" autoLayout } theme default styles { element "Runtime" { background #0B7285 color #FFFFFF } element "Infrastructure" { background #5C677D color #FFFFFF } element "Orchestration" { background #7C6F64 color #FFFFFF } element "DataStore" { background #2F9E44 color #FFFFFF } element "External" { background #F1F3F5 color #343A40 border Dashed } element "Database" { shape Cylinder } element "FileSystem" { shape Folder } } } }