DBT Macro: Render Write Options
DBT macro name: trase_duckdb.render_write_options
DBT details
Description
No description
Details
Not referenced by any model or test.
{% macro render_write_options(config) -%}
{% set options = config.get('options', {}) %}
{% if options is not mapping %}
{% do exceptions.raise_compiler_error("The options argument must be a dictionary") %}
{% endif %}
{% for k in options %}
{% set _ = options.update({k: render(options[k])}) %}
{% endfor %}
{# legacy top-level write options #}
{% if config.get('format') %}
{% set _ = options.update({'format': render(config.get('format'))}) %}
{% endif %}
{% if config.get('delimiter') %}
{% set _ = options.update({'delimiter': render(config.get('delimiter'))}) %}
{% endif %}
{# trase defaults for external parquet writes, unless the model overrides them.
- parquet_version V2: better encodings (see header).
- codec zstd: DuckDB defaults to Snappy; zstd gives smaller files (less to
transfer) and decompresses just as fast, so it is the better default for
our S3-read-heavy access pattern. Unlike the V2 encodings, zstd is
supported by every Parquet reader, so there is no compatibility risk. #}
{% if (options.get('format', '') | string | lower) == 'parquet' %}
{% if 'parquet_version' not in options %}
{% set _ = options.update({'parquet_version': 'V2'}) %}
{% endif %}
{% if 'codec' not in options and 'compression' not in options %}
{% set _ = options.update({'codec': 'zstd'}) %}
{% endif %}
{# When the (effective) codec is zstd, default to a higher compression level
than DuckDB's default of 3. zstd decompresses at ~the same speed
regardless of level, so a higher level means smaller files (less to pull
from S3 -> faster reads) at the cost of slower writes only, which suits
our write-once / read-many external layer. 12 is a good balance; raising
it toward ~19 squeezes a little more at a steep write-time/memory cost. #}
{% set _codec = (options.get('codec') or options.get('compression') or '') | string | lower %}
{% if _codec == 'zstd' and 'compression_level' not in options %}
{% set _ = options.update({'compression_level': 12}) %}
{% endif %}
{% endif %}
{% do return(options) %}
{%- endmacro %}