Get hands-on experience with 20+ free Google Cloud products and $300 in free credit for new customers.

Dataform Incremental Table: Execute Immediate Approach

How come bigquery is now using a stored procedure and execute immediate on my incremental table scripts? It seems there is this new approach with execute immediate and specifying column names on insert. I can't find any release notes or anything on it though. How / where is the source code / release notes for this change?

Prior to 19th September the boiler plate looks like this: (Stripping out schema/table names)

 

 

BEGIN
  CREATE SCHEMA IF NOT EXISTS `table-ref` OPTIONS(location="australia-southeast1");
EXCEPTION WHEN ERROR THEN
  IF NOT CONTAINS_SUBSTR(@@error.message, "already exists: dataset") AND
    NOT CONTAINS_SUBSTR(@@error.message, "too many dataset metadata update operations") AND
    NOT CONTAINS_SUBSTR(@@error.message, "User does not have bigquery.datasets.create permission")
  THEN
    RAISE USING MESSAGE = @@error.message;
  END IF;
END;
    BEGIN
      DECLARE dataform_table_type DEFAULT (
  SELECT ANY_VALUE(table_type)
  FROM `table-ref`
  WHERE table_name = 'table-name'
);
      IF dataform_table_type IS NOT NULL AND dataform_table_type != 'BASE TABLE' THEN
  IF dataform_table_type = 'BASE TABLE' THEN
    DROP TABLE IF EXISTS `table-ref`;
  ELSEIF dataform_table_type = "VIEW" THEN
    DROP VIEW IF EXISTS `table-ref`;
  ELSEIF dataform_table_type = 'MATERIALIZED VIEW' THEN
    DROP MATERIALIZED VIEW IF EXISTS `table-ref`;
  END IF;
END IF;
      IF dataform_table_type IS NOT NULL THEN
        BEGIN
          
          INSERT INTO `table-ref`
          (
            

 

 

Now on September 20th it looks like this

 

 

BEGIN
  CREATE SCHEMA IF NOT EXISTS `schema-ref` OPTIONS(location="australia-southeast1");
EXCEPTION WHEN ERROR THEN
  IF NOT CONTAINS_SUBSTR(@@error.message, "already exists: dataset") AND
    NOT CONTAINS_SUBSTR(@@error.message, "too many dataset metadata update operations") AND
    NOT CONTAINS_SUBSTR(@@error.message, "User does not have bigquery.datasets.create permission")
  THEN
    RAISE USING MESSAGE = @@error.message;
  END IF;
END;
    BEGIN
      DECLARE dataform_table_type DEFAULT (
  SELECT ANY_VALUE(table_type)
  FROM `schema-ref.INFORMATION_SCHEMA.TABLES`
  WHERE table_name = 'table-ref'
);
      IF dataform_table_type IS NOT NULL AND dataform_table_type != 'BASE TABLE' THEN
  IF dataform_table_type = 'BASE TABLE' THEN
    DROP TABLE IF EXISTS `table-ref`;
  ELSEIF dataform_table_type = "VIEW" THEN
    DROP VIEW IF EXISTS `table-ref`;
  ELSEIF dataform_table_type = 'MATERIALIZED VIEW' THEN
    DROP MATERIALIZED VIEW IF EXISTS `table-ref`;
  END IF;
END IF;
      IF dataform_table_type IS NOT NULL THEN
        BEGIN
          DECLARE dataform_columns ARRAY<STRING>;
          DECLARE dataform_columns_list STRING;
          SET dataform_columns = (
            SELECT
              ARRAY_AGG(DISTINCT "`" || column_name || "`")
            FROM `table-ref.INFORMATION_SCHEMA.COLUMNS`
            WHERE table_name = 'table-ref'
          );
          SET dataform_columns_list = (
            SELECT
              STRING_AGG(column)
            FROM UNNEST(dataform_columns) AS column);

          EXECUTE IMMEDIATE
          """
            CREATE OR REPLACE PROCEDURE `procedure_ref`()
            BEGIN
              

              INSERT INTO `table-ref`
              (""" || dataform_columns_list || """)
              SELECT """ || dataform_columns_list || """
              FROM (

 

 

0 5 1,846
5 REPLIES 5