How come bigquery is now using a stored procedure and execute immediate on my incremental table scripts? It seems there is this new approach with execute immediate and specifying column names on insert. I can't find any release notes or anything on it though. How / where is the source code / release notes for this change?
Prior to 19th September the boiler plate looks like this: (Stripping out schema/table names)
BEGIN
CREATE SCHEMA IF NOT EXISTS `table-ref` OPTIONS(location="australia-southeast1");
EXCEPTION WHEN ERROR THEN
IF NOT CONTAINS_SUBSTR(@@error.message, "already exists: dataset") AND
NOT CONTAINS_SUBSTR(@@error.message, "too many dataset metadata update operations") AND
NOT CONTAINS_SUBSTR(@@error.message, "User does not have bigquery.datasets.create permission")
THEN
RAISE USING MESSAGE = @@error.message;
END IF;
END;
BEGIN
DECLARE dataform_table_type DEFAULT (
SELECT ANY_VALUE(table_type)
FROM `table-ref`
WHERE table_name = 'table-name'
);
IF dataform_table_type IS NOT NULL AND dataform_table_type != 'BASE TABLE' THEN
IF dataform_table_type = 'BASE TABLE' THEN
DROP TABLE IF EXISTS `table-ref`;
ELSEIF dataform_table_type = "VIEW" THEN
DROP VIEW IF EXISTS `table-ref`;
ELSEIF dataform_table_type = 'MATERIALIZED VIEW' THEN
DROP MATERIALIZED VIEW IF EXISTS `table-ref`;
END IF;
END IF;
IF dataform_table_type IS NOT NULL THEN
BEGIN
INSERT INTO `table-ref`
(
BEGIN
CREATE SCHEMA IF NOT EXISTS `schema-ref` OPTIONS(location="australia-southeast1");
EXCEPTION WHEN ERROR THEN
IF NOT CONTAINS_SUBSTR(@@error.message, "already exists: dataset") AND
NOT CONTAINS_SUBSTR(@@error.message, "too many dataset metadata update operations") AND
NOT CONTAINS_SUBSTR(@@error.message, "User does not have bigquery.datasets.create permission")
THEN
RAISE USING MESSAGE = @@error.message;
END IF;
END;
BEGIN
DECLARE dataform_table_type DEFAULT (
SELECT ANY_VALUE(table_type)
FROM `schema-ref.INFORMATION_SCHEMA.TABLES`
WHERE table_name = 'table-ref'
);
IF dataform_table_type IS NOT NULL AND dataform_table_type != 'BASE TABLE' THEN
IF dataform_table_type = 'BASE TABLE' THEN
DROP TABLE IF EXISTS `table-ref`;
ELSEIF dataform_table_type = "VIEW" THEN
DROP VIEW IF EXISTS `table-ref`;
ELSEIF dataform_table_type = 'MATERIALIZED VIEW' THEN
DROP MATERIALIZED VIEW IF EXISTS `table-ref`;
END IF;
END IF;
IF dataform_table_type IS NOT NULL THEN
BEGIN
DECLARE dataform_columns ARRAY<STRING>;
DECLARE dataform_columns_list STRING;
SET dataform_columns = (
SELECT
ARRAY_AGG(DISTINCT "`" || column_name || "`")
FROM `table-ref.INFORMATION_SCHEMA.COLUMNS`
WHERE table_name = 'table-ref'
);
SET dataform_columns_list = (
SELECT
STRING_AGG(column)
FROM UNNEST(dataform_columns) AS column);
EXECUTE IMMEDIATE
"""
CREATE OR REPLACE PROCEDURE `procedure_ref`()
BEGIN
INSERT INTO `table-ref`
(""" || dataform_columns_list || """)
SELECT """ || dataform_columns_list || """
FROM (