@startuml
title Object counters pipeline, with archive
actor browser
node pergamon {
database prometheus
component prometheus_server
prometheus_server --> prometheus
}
node moma {
component webapp
}
node "storage(saam|storage1)" as storage_server {
component "swh-storage" as swhstoragegunicorn
}
node counters {
component redis {
collections "HyperLogLog collections" as hll
note left of hll: One collection per\nobject type
}
component counter_journal_client as cjc {
}
component counter as gunicorncounter {
boundary "/metrics" as metrics
boundary "/counter?type=<type>" as counterapi
boundary "/counter_history/refresh_cache?\nfile=<name>&counters=<list>" as refresh_cache
boundary "/counter_history/<cache file>" as counters_history
}
component nginx {
boundary 50XX as counter
}
component cron as counters_cron {
}
file "/usr/local/share/swh-data/archive.json" as archive_json
file "/usr/local/share/swh-data/counters.json" as counters_json
cjc --> hll
counter --> metrics
counter --> counterapi
gunicorncounter --> hll
refresh_cache --> counters_json: generate\ncache\nfile
refresh_cache --> archive_json: read\narchive\nfile
counters_history --> counters_json: read\ncached\ndata
counter --> counters_history
counter --> refresh_cache
counters_cron -left-> counter: curl\n/counter_history/refresh_cache?\nfile=counters.json&\ncounters=<list>
note left of counters_cron: Refresh the cache file periodically
}
node journal {
queue content as topic_content
queue origin as topic_origin
queue revision as topic_revision
queue "..." as topic_others
topic_others -[hidden]right-> topic_revision
}
webapp --> counter: /api/1/stat/counters\n->\n/counter/<type>
browser --> webapp
topic_content --> cjc
topic_origin --> cjc
topic_revision --> cjc
topic_others --> cjc
swhstoragegunicorn --> topic_content
swhstoragegunicorn --> topic_origin
swhstoragegunicorn --> topic_revision
swhstoragegunicorn --> topic_others
prometheus_server --> counter : /metrics
refresh_cache --> prometheus_server: Read live data
webapp --> counter: /counter_history/<cache file>
@enduml