Configuration Settings

Configuration can be done either by using a toml configuration file, environnement variable or secret files.

The configuration framework is based on the pydantic settings package which provides strong validation for configuration data.

Environment variables

The mapping of configuration values follows the following pattern:

Environment variable names are case-insensitive.
Environment variables must be prefixed with CONF_
Nested settings are separated by __ (double underscore)
List and dictionnaries are populated from environnement by treating the environnement variable’s value as JSON-encoded strinf.

Example:

Consider the following the toml configuration:

[logging]
level = "DEBUG"

[worker]
service_name = "MyService"
broker_host = "rabbitmq"
broker_backend = "redis:6370/0"

[processing]
workdir = "/qgis-workdir"

[processing.plugins]
paths = ["/qgis-plugins"]

[processing.projects.search_paths]
'/' = "/qgis-projects"

And the corresponding configuration with environment variables:

CONF_LOGGING__LEVEL=DEBUG
CONF_WORKER__SERVICE_NAME=MyService
CONF_WORKER__BROKER_HOST=rabbitmq
CONF_WORKER__BACKEND_HOST=redis:6379/0
CONF_PROCESSING__WORKDIR=/qgis-workdir
CONF_PROCESSING__PLUGINS__PATHS='["/qgis-plugins"]'
CONF_PROCESSING__PROJECTS__SEARCH_PATHS: '{"/":"/qgis-projects"}'

Secret files

Instead of using exposed environment variables or configuration files, values may be stored in files that contains a single value and where the name of the file is the configuration.

A common usecase is to allow for storing sensitive values in Docker encrypted secret files.

Configuration precedence

Configuration precedence is (by decreasing priority):

Configuration file
Environment variables
Secret files
Default values

Worker configuration

[logging]
level = "INFO"


# Worker configuration
# 
# Configure celery worker settings
# 
[worker]
#
# Celery amqp broker host
broker_host = "localhost"
broker_use_tls = false
#broker_user =   	# Optional
#broker_password =   	# Optional
#
# Celery redis backend host
backend_host = "localhost:6379/0"
backend_use_tls = false
#backend_password =   	# Optional
#
# Task hard time limit in seconds.
# The worker processing the task will be killed
# and replaced with a new one when this is exceeded.
# 
task_time_limit = 3600
#
# Grace period to add to the 'task_time_limit' value.
# The SoftTimeLimitExceeded exception will be raised
# when the 'task_time_limit' is exceeded.
# 
task_time_grace_period = 60
#
# Time (in seconds), for when after stored task tombstones will
# be deleted
# 
result_expires = 86400
#
# Concurrency
#
# The number of concurrent worker processes executing tasks.
#concurrency =   	# Optional
#
# Autoscale
#
# Activate concurrency autoscaling
#max_concurrency =   	# Optional
#
# Processes life cycle
#
# Maximum number of tasks a pool worker process can execute
# before it's replaced with a new one. Default is no limit.
# 
#max_tasks_per_child =   	# Optional
#
# Maximum consumed memory
#
# Maximum amount of resident memory, in kilobytes,
# that may be consumed by a worker before it will
# be replaced by a new worker.
# 
#max_memory_per_child =   	# Optional
#
# Name of the service
#
# Name used as location service name
# for initializing Celery worker.
# 
#service_name =   	# Required
#
# Service short title
title = ""
#
# Service description
description = ""
#
# Cleanup interval
#
# Interval is seconds between two cleanup of expired jobs.
# The minimun is 300s (5mn).
# 
cleanup_interval = 3600
#
# Reload watch file
#
# The file to watch for reloading processing plugins.
# When the the modified time of the file is changed, processing
# providers are reloaded.
# The restart is graceful, all running jobs are terminated normally.
# 
#reload_monitor =   	# Optional
#
# Hide presence versions
#
# Hide version details in presence.
# This may be useful when you do not want to
# display versions of libraries and OS for security
# reasons.
# 
hide_presence_versions = false

#
[worker.broker_tls]
#
# CA file
#cafile =   	# Optional
#
# TLS  certificat
#
# Path to the TLS cert file
#certfile =   	# Optional
#
# TLS key file
#
# Path to the TLS key file
#keyfile =   	# Optional

#
[worker.backend_tls]
#
# CA file
#cafile =   	# Optional
#
# TLS  certificat
#
# Path to the TLS cert file
#certfile =   	# Optional
#
# TLS key file
#
# Path to the TLS key file
#keyfile =   	# Optional

#
[worker.security]
#cert_store =   	# Required
#keyfile =   	# Required
#certfile =   	# Required

#
[worker.scheduler]
#
# Enable scheduler
#
# Enable embedded scheduler.
# Prefer scheduler as a service if more
# than one worker node is used.
# 
enabled = false
#
# Max interval
#
# Max seconds to sleep between schedule iterations.
#max_interval =   	# Optional
#
# Scheduler database path
#
# Path to the schedule database.
# Defaults to `celerybeat-schedule` (from Celery doc).
# 
#database =   	# Optional

#
# Service related links
#
[[worker.links]]
#rel =   	# Optional
#mime_type =   	# Optional
title = ""
#description =   	# Optional
#length =   	# Optional
templated = false
#hreflang =   	# Optional
#href =   	# Required


# Configure storage for processing data
[storage]
#
# Storage module
#
# The module implementing storage accesses for
# job's files.
# 
storage_class = "qjazz_processes.worker.storages.local.LocalStorage"
config = {}

#
# Callbacks
#
# Define a mapping between a uri scheme and a handler for that
# scheme.
# 
#
# Example:
#
# [callbacks.https]
# handler = "qjazz_processes.callbacks.Http",
# 
[callbacks.'key']
#
# Enable callback
enabled = true
#
# Callback module
#
# The callback handler import string
#handler =   	# Required
#
# Callback configuration
config = {}


[processing]
#
# Working directory
#
# Parent working directory where processes are executed.
# Each processes will create a working directory for storing
# result files and logs.
# 
#workdir =   	# Required
#
# Internal qgis providers exposed
#
# List of exposed QGIS processing internal providers.
# NOTE: It is not recommended exposing all providers like
# `qgis` or `native`, instead provide your own wrapping
# algorithm, script or model.
# 
exposed_providers = ["script","model"]
#
# Expose deprecated algorithms
#
# Expose algorithm wich have the `Deprecated`
# flag set.
# 
expose_deprecated_algorithms = true
#
# Default vector file extension
#
# Define the default vector file extensions for vector destination
# parameters. If not specified, then the QGIS default value is used.
# 
default_vector_file_ext = "fgb"
#
# Default raster file extension
#
# Define the default raster file extensions for raster destination
# parameters. If not specified, then the QGIS default value is used.
# 
#default_raster_file_ext =   	# Optional
#
# Force ellipsoid imposed by the source project
#
# Force the ellipsoid from the src project into the destination project.
# This only apply if the src project has a valid CRS.
# 
adjust_ellipsoid = false
#
# Set default CRS
#
# Set the CRS to use when no source map is specified.
# For more details on supported formats see the GDAL method
# 'GdalSpatialReference::SetFromUserInput()'
# 
default_crs = "urn:ogc:def:crs:OGC:1.3:CRS84"
#
# Advertised services urls
#
# Url template used for OGC services references.
advertised_services_url = "ows:$jobId/$name"
#
# Public download url
#
# Url template for downloading resources.
# This is the public base url that will be seen in
# referenced responses.
# This url will need to be translated by the front end
# executor to an effective download url.
# 
store_url = "${public_url}/jobs/$jobId/files/$resource"
#
# Use destination input as sink
#
# Allow input value as sink for destination layers.
# This allow value passed as input value to be interpreted as
# path or uri sink definition. This enable passing any string
# that QGIS may use a input source but without open options except for the
# 'layername=<name>' option.
# 
# NOTE: Running concurrent jobs with this option may result in unpredictable
# behavior.
# 
# For that reason it is considered as an UNSAFE OPTION and you should never enable
# this option if you are exposing the service publicly.
# 
# File path inputs prefixed with '/' will correspond to path located in the root
# directory specified by the `raw_destination_root_path` option.
# Otherwise, they will be stored in the job folder.
# 
raw_destination_input_sink = false
#
# Raw destination root path
#
# Specify the root directory for storing destination layers files when
# the `raw_destination_input_sink` option is enabled.
# If not specified, files will be stored in the job folder.
# 
#raw_destination_root_path =   	# Optional
#
# Project cache size
#
# The maximum number of projects in cache by process.
max_cached_projects = 10
#
# Qgis settings
#
# Qgis settings override.
# Use the syntax '<section>/<path>' for keys.
# Not that values defined here will override those
# from QGIS3.ini file."
# 
qgis_settings = {}
#
# Download timeout
#
# Download timeout is seconds for reference values
download_timeout = 20.0

#
# Projects configuration
#
# Projects and cache configuration
#
[processing.projects]
#
# Trust layer metadata
#
# Trust layer metadata.
# Improves layer load time by skipping expensive checks
# like primary key unicity, geometry type and
# srid and by using estimated metadata on layer load.
# Since QGIS 3.16
# 
trust_layer_metadata = false
#
# Disable GetPrint requests
#
# Don't load print layouts.
# Improves project read time if layouts are not required,
# and allows projects to be safely read in background threads
# (since print layouts are not thread safe).
# 
disable_getprint = false
#
# Force read only mode
#
# Force layers to open in read only mode."
# This is the default, unless you have really good reason
# to open layers in write mode.
# Note that some handlers (like S3) do not support opening layers
# in write mode.
# 
force_readonly_layers = true
#
# Ignore bad layers
#
# Allow projects to be loaded with event if it contains
# layers that cannot be loaded.
# Note that the 'dont_resolve_layers flag' trigger automatically
# this option.
# 
ignore_bad_layers = false
#
# Disable OWS advertised urls
#
# Disable ows urls defined in projects.
# This may be necessary because Qgis projects
# urls override proxy urls.
disable_advertised_urls = false
#
# Scheme mapping definitions
#
# Defines mapping betweeen location base path and storage handler root url.
# Resource path relative to location will be joined the the root url path.
# In the case of Qgis storage, the handler is responsible for transforming
# the result url into a comprehensive format for the corresponding
# QgsProjectStorage implementation.
# This is handled by the default storage implementation for Qgis native
# project storage.
# In case of custom QgsProjectStorage, if the scheme does not allow passing
# project as path component, it is possible to specify a custom resolver function.
# 
search_paths = {}
#
# Allow direct path resolution
#
# Allow direct path resolution if there is
# no matching from the search paths.
# Uri are directly interpreted as valid Qgis project's path.
# WARNING: allowing this may be a security vulnerabilty."
# 
allow_direct_path_resolution = false

#
# Project storage Handler configurations
#
# Configure storage handlers.
# The name will be used as scheme for project's search path
# configuration.
# 
#
# Example:
#
# [projects.search_paths]
# "/public/location1/" = "postgres1://?dbname=mydatabase1"
# "/public/location2/" = "postgres1://?dbname=mydatabase2"
# 
# [projects.handlers.postgres1]
# handler_class = qjazz_cache.handlers.postgresql.PostgresHandler
# 
# [projects.handlers.postgres1.config]
# uri = "postgresql://user@host/?schema=myschema"
# 
#
[processing.projects.handlers.'key']
#handler =   	# Required
config = {}

#
# Plugin configuration
#
[processing.plugins]
#
# Plugin paths
#
# The list of search paths for plugins.
# Qgis plugins found will be loaded according to
# the 'install' list.
# If the list is empty, the 'QGIS_PLUGINPATH'
# variable will be checked.
paths = []
#
# Installable plugins
#
# The list of installable plugins.
# Note: if the plugin directory contains other plugins
# plugins not in the list will NOT be loaded !
# The Plugins will be installed at startup
# if the 'install_mode' is set to 'auto'.
# Note that an empty list means what it is:
# i.e, *no* installed plugins.
#install =   	# Optional
#
# Plugin installation mode
#
# If set to 'auto', plugins installation
# will be checked at startup. Otherwise,
# Installation will be done from already available
# plugins.
install_mode = "external"
#
# Enable processing scripts
#
# Enable publication of processing scripts
enable_scripts = true
#
# Extra builtins providers
#
# Load extra builtin processing providers
# such as 'grass' and 'otb'.
extra_builtin_providers = []
#
# Path to plugin manager executable
#
# The absolute path to the qgis-plugin_manager executable
# that will be used for installing plugin in automatic mode.
plugin_manager = "/usr/local/bin/qgis-plugin-manager"

#
# TLS Certificates
#
# TLS credentials to use for references inputs
#
[processing.certificats]
#
# CA file
#cafile =   	# Optional
#
# TLS  certificat
#
# Path to the TLS cert file
#certfile =   	# Optional
#
# TLS key file
#
# Path to the TLS key file
#keyfile =   	# Optional

#
# Qgis network
#
[processing.network]
#
# Transfer timeout in ms
#
# Transfers are aborted if no bytes are transferred before
# the timeout expires.
# If set to 0, the timeout is disobled.
# Default value is set to 10000 milliseconds.
# 
transfer_timeout = 10000
#
# Trace network activity
trace = false
#
# Global cache policy
#
# Set a global cache policy for all requests"
# If set, this will override requests cache policy".
# 
#cache_policy =   	# Optional

#
# Domain policies
#
# Set per domain policy
#
[processing.network.domain_policy.'key']
#
# Cache load control
#
# Override QNetworkRequest::CacheLoadControl for request.
#cache_policy =   	# Optional
#
# Transfer timeout in ms
#transfer_timeout =   	# Optional

Server configuration

[logging]
level = "INFO"


# OAPI configuration
[oapi]
title = "Qjazz-Processes"
description = "Publish Qgis processing algorithms as OGC api processes"


# Defining job realm allow filtering job's requests by a token that is
# set by the client when requesting task execution (see description below).
# 
[job_realm]
#
# Enable job realm header
#
# When enabled, use the 'X-Job-Realm' http header
# as a client identification token for retrieving jobs status and results.
# 
enabled = false
#
# Admininistrator realm jobs tokens
#
# Define catch all tokens for listing and retrieve status and results
# for all jobs.
# 
admin_tokens = []


# Configure access policy
[access_policy]
#
# Access policy module
#
# The module implementing the access policy for
# processes execution.
# 
policy_class = "qjazz_processes.server.policies.default.DefaultAccessPolicy"
config = {}


[http]
#
# Interfaces to listen to
listen = ["127.0.0.1",9180]
#
# Use tls
use_tls = false
#
# CORS origin
#
# Allows to specify origin for CORS. If set 'all' will set
# Access-Control-Allow-Origin to '*'; 'same-origin' return
# the same value as the 'Origin' request header.
# A url may may be specified, restricting allowed origin to
# this url.
# 
cross_origin = "all"
#
# Service update interval
#
# Interval in seconds between update of available services
update_interval = 30
#
# Backend request timeout
timeout = 20

#
# TLS configuration
#
[http.tls]
#
# CA file
#cafile =   	# Optional
#
# TLS  certificat
#
# Path to the TLS cert file
#certfile =   	# Optional
#
# TLS key file
#
# Path to the TLS key file
#keyfile =   	# Optional

#
[http.proxy]
#
# Enabled Forwarded headers
#
# Enable proxy headers resolution.
# Include support for 'Forwarded' headers
# and 'X-Forwarded' headers if allow_x_headers is
# enabled."
# 
enable = false
#
# Support for 'X-Forwarded' headers
allow_x_headers = false


[executor]
#
# Message expiration timeout
#
# The amount of time an execution message
# can wait on queue before beeing processed
# with asynchronous response.
# 
message_expiration_timeout = 600

#
[executor.celery]
#
# Celery amqp broker host
broker_host = "localhost"
broker_use_tls = false
#broker_user =   	# Optional
#broker_password =   	# Optional
#
# Celery redis backend host
backend_host = "localhost:6379/0"
backend_use_tls = false
#backend_password =   	# Optional
#
# Task hard time limit in seconds.
# The worker processing the task will be killed
# and replaced with a new one when this is exceeded.
# 
task_time_limit = 3600
#
# Grace period to add to the 'task_time_limit' value.
# The SoftTimeLimitExceeded exception will be raised
# when the 'task_time_limit' is exceeded.
# 
task_time_grace_period = 60
#
# Time (in seconds), for when after stored task tombstones will
# be deleted
# 
result_expires = 86400
#
# Concurrency
#
# The number of concurrent worker processes executing tasks.
#concurrency =   	# Optional
#
# Autoscale
#
# Activate concurrency autoscaling
#max_concurrency =   	# Optional
#
# Processes life cycle
#
# Maximum number of tasks a pool worker process can execute
# before it's replaced with a new one. Default is no limit.
# 
#max_tasks_per_child =   	# Optional
#
# Maximum consumed memory
#
# Maximum amount of resident memory, in kilobytes,
# that may be consumed by a worker before it will
# be replaced by a new worker.
# 
#max_memory_per_child =   	# Optional

#
[executor.celery.broker_tls]
#
# CA file
#cafile =   	# Optional
#
# TLS  certificat
#
# Path to the TLS cert file
#certfile =   	# Optional
#
# TLS key file
#
# Path to the TLS key file
#keyfile =   	# Optional

#
[executor.celery.backend_tls]
#
# CA file
#cafile =   	# Optional
#
# TLS  certificat
#
# Path to the TLS cert file
#certfile =   	# Optional
#
# TLS key file
#
# Path to the TLS key file
#keyfile =   	# Optional

#
[executor.celery.security]
#cert_store =   	# Required
#keyfile =   	# Required
#certfile =   	# Required

#
[executor.celery.scheduler]
#
# Enable scheduler
#
# Enable embedded scheduler.
# Prefer scheduler as a service if more
# than one worker node is used.
# 
enabled = false
#
# Max interval
#
# Max seconds to sleep between schedule iterations.
#max_interval =   	# Optional
#
# Scheduler database path
#
# Path to the schedule database.
# Defaults to `celerybeat-schedule` (from Celery doc).
# 
#database =   	# Optional


# The storage configuration is used for configuring the
# connections to storage backends used by workers.
# 
[storage]
#
# Allow insecure downloads
#
# If set to false, only TLS encrypted downloads are allowed
allow_insecure_connection = true
#
# Download chunksize
chunksize = 65536
#
# Download url expiration
#
# Download url expiration in seconds
download_url_expiration = 3600

#
# TLS certifificats
#
# Certificats required for TLS downloads connections
#
[storage.tls]
#
# CA file
#cafile =   	# Optional
#
# TLS  certificat
#
# Path to the TLS cert file
#certfile =   	# Optional
#
# TLS key file
#
# Path to the TLS key file
#keyfile =   	# Optional


#[store]
#
# S3/Minio endpoint
#endpoint = "localhost:9000"
#cafile =   	# Optional
#access_key =   	# Required
#secret_key =   	# Required
#
# Enable TLS
#enable_tls = false
#
# Check certificat
#check_certificat = true
#region =   	# Optional
#
# Access TTL
#
# Lifetime given to a job store access since the job was scheduled
# for execution; the access credentials will no be
# longer valid after this time.
# 
#access_ttl = 86400

Callbacks

Callbacks requirements are specified by OGC standards.

It allows to set up a push based mechanism for processes results to others services.

Originally, only http POST requests are considered but Qjazz processes support arbitrary uri schemes.

Callbacks handlers are declared in worker configuration under the [callbacks."<scheme,...>"] section along with their configuration and the import string to the class implementing the callback support.

Qjazz implement natively ‘http’ and ‘mailto’ scheme callbacks.

HTTP Callback configuration

[callbacks]

#
[callbacks.mailto]
#
# Enable callback
enabled = true
handler = "qjazz_processes.callbacks.MailTo"

#
[callbacks.mailto.config]
#
# SMTP host
#smtp_host =   	# Required
#
# SMTP port
smtp_port = 587
#
# SMTP login
#smtp_login =   	# Optional
#
# SMTP password
smtp_password = ""
#
# TLS/SSL
smtp_tls = false
#
# From address
#mail_from =   	# Required
#
# Format
#
# The format of the e-mail body
body_format = "plain"
#
# Attach results
#
# Send job results as attachment
send_results_as_attachment = false
#
# Request timeout
#
# The request timeout value in seconds
timeout = 5
#
# Debug mode
debug = false

#
# Subject and body to set on success notification
# If a subject is provided then it will override the configuration value.
# 
#
[callbacks.mailto.config.content_success]
subject = "[Qjazz:$service] Job $process successfull"
body = '''
The job $jobid ($process) has been executed with success.
'''

#
# Subject and body to set on failed notification.
# If a subject is provided then it will override the configuration value.
# 
#
[callbacks.mailto.config.content_failed]
subject = "[QJazz:$service] Job $process failed"
body = '''
The job $jobid ($process) has failed.
'''

#
# Subject and body to set on inProgresss notification.
# If a subject is provided then it will override the configuration value.
# 
#
[callbacks.mailto.config.content_in_progress]
subject = "[QJazz:$service] Job $process started"
body = '''
The job $jobid ($process) has started.
'''

#
[callbacks.mailto.config.acl]
#
# Authorization order
#
# Set the order of evaluation of allow and deny directives:
# - Allow: allow by default  except thoses in deny then
#   put back those in deny with the allow directive.
# - Deny: deny by default  except thoses in allow then deny
#   those in allow with the deny directive.
# 
order = "Allow"
#
# Allowed addresses
#
# List of allowed hosts. An host may be a IP addresse at IP range
# in CIDR format or a FQDN or FQDN suffix starting with a dot (and
# an optional '*').
# 
#
# Example:
#
# allow = [
#     "foo.bar.com",
#     "*.mydomain.com",
#     "192.168.0.0/24",
#     "192.168.1.2",
# ]
# 
allow = []
#
# Forbidden addresses
#
# List of forbidden hosts in the same format as for 'allow' list.
deny = []

Note

For serving both http and https use [callbacks."http,https"].

MailTo Callback configuration

Allow sending e-mail with callbacks.

[callbacks]

#
[callbacks.mailto]
#
# Enable callback
enabled = true
handler = "qjazz_processes.callbacks.MailTo"

#
[callbacks.mailto.config]
#
# SMTP host
#smtp_host =   	# Required
#
# SMTP port
smtp_port = 587
#
# SMTP login
#smtp_login =   	# Optional
#
# SMTP password
smtp_password = ""
#
# TLS/SSL
smtp_tls = false
#
# From address
#mail_from =   	# Required
#
# Format
#
# The format of the e-mail body
body_format = "plain"
#
# Attach results
#
# Send job results as attachment
send_results_as_attachment = false
#
# Request timeout
#
# The request timeout value in seconds
timeout = 5
#
# Debug mode
debug = false

#
# Subject and body to set on success notification
# If a subject is provided then it will override the configuration value.
# 
#
[callbacks.mailto.config.content_success]
subject = "[Qjazz:$service] Job $process successfull"
body = '''
The job $jobid ($process) has been executed with success.
'''

#
# Subject and body to set on failed notification.
# If a subject is provided then it will override the configuration value.
# 
#
[callbacks.mailto.config.content_failed]
subject = "[QJazz:$service] Job $process failed"
body = '''
The job $jobid ($process) has failed.
'''

#
# Subject and body to set on inProgresss notification.
# If a subject is provided then it will override the configuration value.
# 
#
[callbacks.mailto.config.content_in_progress]
subject = "[QJazz:$service] Job $process started"
body = '''
The job $jobid ($process) has started.
'''

#
[callbacks.mailto.config.acl]
#
# Authorization order
#
# Set the order of evaluation of allow and deny directives:
# - Allow: allow by default  except thoses in deny then
#   put back those in deny with the allow directive.
# - Deny: deny by default  except thoses in allow then deny
#   those in allow with the deny directive.
# 
order = "Allow"
#
# Allowed addresses
#
# List of allowed hosts. An host may be a IP addresse at IP range
# in CIDR format or a FQDN or FQDN suffix starting with a dot (and
# an optional '*').
# 
#
# Example:
#
# allow = [
#     "foo.bar.com",
#     "*.mydomain.com",
#     "192.168.0.0/24",
#     "192.168.1.2",
# ]
# 
allow = []
#
# Forbidden addresses
#
# List of forbidden hosts in the same format as for 'allow' list.
deny = []