Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix materialized view creation #194

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 36 additions & 12 deletions lib/active_record/connection_adapters/clickhouse/schema_creation.rb
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,19 @@ def assign_database_to_subquery!(subquery)
"#{current_database}.#{match[:table_name].sub('.', '')}"
end

def add_materialized_to_clause!(create_sql, options)
if !options.to
create_sql << " ENGINE = Memory()"
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So it stores this in the RAM?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. If the create_table definition does not include a to value but the view: materialized k/v pair is set so that we need to make a to clauses, we will default to storing the data in memory (since we don't know what tables we can use to store it, and Null() engine wouldn't make sense).

else
target_table = options.to.split('.').last
table_structure = @conn.execute("DESCRIBE TABLE #{target_table}")['data']
column_definitions = table_structure.map do |field|
"`#{field[0]}` #{field[1]}"
end
create_sql << "TO #{options.to} (#{column_definitions.join(', ')}) "
end
end

def add_to_clause!(create_sql, options)
# If you do not specify a database explicitly, ClickHouse will use the "default" database.
return unless options.to
Expand All @@ -97,23 +110,34 @@ def visit_TableDefinition(o)
create_sql = +"CREATE#{table_modifier_in_create(o)} #{o.view ? "VIEW" : "TABLE"} "
create_sql << "IF NOT EXISTS " if o.if_not_exists
create_sql << "#{quote_table_name(o.name)} "
add_as_clause!(create_sql, o) if o.as && !o.view
add_to_clause!(create_sql, o) if o.materialized

statements = o.columns.map { |c| accept c }
statements << accept(o.primary_keys) if o.primary_keys
# Add column definitions for regular tables only
if !o.view && o.columns.present?
statements = o.columns.map { |c| accept c }
statements << accept(o.primary_keys) if o.primary_keys

if supports_indexes_in_create?
indexes = o.indexes.map do |expression, options|
accept(@conn.add_index_options(o.name, expression, **options))
if supports_indexes_in_create?
indexes = o.indexes.map do |expression, options|
accept(@conn.add_index_options(o.name, expression, **options))
end
statements.concat(indexes)
end
statements.concat(indexes)

create_sql << "(#{statements.join(', ')})"
end

create_sql << "(#{statements.join(', ')})" if statements.present?
# Attach options for only table or materialized view without TO section
add_table_options!(create_sql, o) if !o.view || o.view && o.materialized && !o.to
add_as_clause!(create_sql, o) if o.as && o.view
# Add TO clause for materialized views before AS clause
add_materialized_to_clause!(create_sql, o) if o.materialized && o.view

# Add AS clause for all views
add_as_clause!(create_sql, o) if o.as

# Add TO clause for regular views (non-materialized) after AS clause
add_to_clause!(create_sql, o) if o.to && !o.materialized

# Add table options for regular tables
add_table_options!(create_sql, o) if !o.view

create_sql
end

Expand Down
44 changes: 44 additions & 0 deletions spec/single/materialized_view_spec.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
require 'spec_helper'

RSpec.describe 'Materialized Views' do
before do
ActiveRecord::Schema.define do
create_table "events", id: false, options: "Log", force: :cascade do |t|
t.integer "quantity", default: -> { "CAST(1, 'Int8')" }, null: false
t.string "name", null: false
t.date "created_at", null: false
end
end
end

after do
ActiveRecord::Schema.define do
drop_table :events if table_exists?(:events)
drop_table :aggregated_events_mv if table_exists?(:aggregated_events_mv)
drop_table :aggregated_events if table_exists?(:aggregated_events)
end
end

it 'creates a materialized view with TO clause and column definitions' do
database = ActiveRecord::Base.connection_db_config.database

ActiveRecord::Schema.define do
create_table "aggregated_events", id: false, options: "SummingMergeTree ORDER BY (name, date) SETTINGS index_granularity = 8192", force: :cascade do |t|
t.string "name", null: false
t.date "date", null: false
t.integer "total_quantity", limit: 8, null: false
t.integer "event_count", limit: 8, null: false
end

create_table "aggregated_events_mv", view: true, materialized: true, to: "#{database}.aggregated_events", id: false, as: "SELECT name, created_at AS date, sum(quantity) AS total_quantity, count() AS event_count FROM #{database}.events GROUP BY name, created_at", force: :cascade do |t|
end
end

# Verify the view was created correctly
result = ActiveRecord::Base.connection.do_system_execute(
"SHOW CREATE TABLE #{database}.aggregated_events_mv"
)['data'].first.first

expect(result.squish).to eq('CREATE MATERIALIZED VIEW default.aggregated_events_mv TO default.aggregated_events ( `name` String, `date` Date, `total_quantity` UInt64, `event_count` UInt64 ) AS SELECT name, created_at AS date, sum(quantity) AS total_quantity, count() AS event_count FROM default.events GROUP BY name, created_at')
end
end